{ "kind": "separate-prospective-open-source-instrumentation-batches", "batches": [ { "batch": "eslint-1", "scoreDirectionExplicit": false, "sourceCommit": "12cbff4125db3f6446f72fa0c48c46b92b120427", "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "21333", "project": null, "arm": "memex", "historyRecords": 4, "attemptOutcome": "INCOMPLETE", "status": "turn_limit", "changedFileCount": 0, "modelCalls": 8, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "21333", "project": null, "arm": "keyword", "historyRecords": 4, "attemptOutcome": "INCOMPLETE", "status": "empty_or_truncated", "changedFileCount": 0, "modelCalls": 7, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "21443", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "INCOMPLETE", "status": "turn_limit", "changedFileCount": 0, "modelCalls": 8, "toolCalls": 11, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "21443", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "INCOMPLETE", "status": "turn_limit", "changedFileCount": 0, "modelCalls": 8, "toolCalls": 10, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "21436", "project": null, "arm": "keyword", "historyRecords": 6, "attemptOutcome": "INCOMPLETE", "status": "turn_limit", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 11, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "21436", "project": null, "arm": "memex", "historyRecords": 6, "attemptOutcome": "INCOMPLETE", "status": "turn_limit", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "INCOMPLETE": 6 }, "reviews": [ { "blind": "B01", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 0 } ], "registeredAdjudication": null } ], "posthocNormalization": { "normalizationStatuses": { "valid_unchanged": 2, "posthoc_normalized": 2 }, "ratings": { "B01:deepseek-flash": { "status": "valid_unchanged", "correctness": 1, "sideEffects": 1 }, "B01:gpt-6-astra": { "status": "posthoc_normalized", "correctness": 1, "sideEffects": 1 }, "B02:deepseek-flash": { "status": "posthoc_normalized", "correctness": 2, "sideEffects": 1 }, "B02:gpt-6-astra": { "status": "valid_unchanged", "correctness": 2, "sideEffects": 0 } }, "adjudications": { "B02": { "status": "VALID", "correctness": 2, "sideEffects": 1 } } }, "usage": { "claude-haiku-4-5": { "started": 48, "completed": 48, "reportedPromptTokens": 298855, "reportedCompletionTokens": 5601, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 811740, "responseBytes": 49210 }, "deepseek-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 2720, "reportedCompletionTokens": 318, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 9792, "responseBytes": 2360 }, "gpt-6-astra": { "started": 2, "completed": 2, "reportedPromptTokens": 11245, "reportedCompletionTokens": 657, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 9718, "responseBytes": 2263 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "lodash-interrupted", "scoreDirectionExplicit": false, "sourceCommit": "2b5e6f7399a7b48005140b5d5c6bc6c0e62919a8", "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "6195", "project": null, "arm": "keyword", "historyRecords": 6, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 0, "modelCalls": 17, "toolCalls": 32, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "6195", "project": null, "arm": "memex", "historyRecords": 6, "attemptOutcome": "INTERRUPTED", "status": null }, { "slot": 3, "issue": "5750", "project": null, "arm": "keyword", "historyRecords": 7, "attemptOutcome": "NOT_RUN", "status": null }, { "slot": 4, "issue": "5750", "project": null, "arm": "memex", "historyRecords": 7, "attemptOutcome": "NOT_RUN", "status": null }, { "slot": 5, "issue": "5881", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": null }, { "slot": 6, "issue": "5881", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": null } ], "outcomes": { "CHECKS_NONZERO": 1, "INTERRUPTED": 1, "NOT_RUN": 4 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": null } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } } ], "posthocNormalization": null, "usage": { "claude-haiku-4-5": { "started": 27, "completed": 26, "reportedPromptTokens": 138409, "reportedCompletionTokens": 3132, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 1, "requestBytes": 403284, "responseBytes": 27889 }, "deepseek-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 2393, "reportedCompletionTokens": 1196, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 5675, "responseBytes": 3443 }, "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 7883, "reportedCompletionTokens": 114, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 5638, "responseBytes": 740 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "lodash-repair", "scoreDirectionExplicit": false, "sourceCommit": "2b5e6f7399a7b48005140b5d5c6bc6c0e62919a8", "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "5750", "project": null, "arm": "keyword", "historyRecords": 7, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 0, "modelCalls": 17, "toolCalls": 30, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "5750", "project": null, "arm": "memex", "historyRecords": 7, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 0, "modelCalls": 9, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "5881", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 4, "issue": "5881", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "CHECKS_NONZERO": 1, "UNKNOWN": 1, "NOT_RUN": 2 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } } ], "posthocNormalization": null, "usage": { "claude-haiku-4-5": { "started": 27, "completed": 27, "reportedPromptTokens": 193679, "reportedCompletionTokens": 3440, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 557154, "responseBytes": 27978 }, "deepseek-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 2215, "reportedCompletionTokens": 975, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 4909, "responseBytes": 2414 }, "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 5520, "reportedCompletionTokens": 109, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 4872, "responseBytes": 849 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "da5aaf97518b5a549ca691198dfd7dbb23d5672f52bfd93b9a9ab242abb78660", "at": "2026-10-31T13:25:58.350452+08:00", "label": "02:8", "model": "claude-haiku-4-5", "status": "error", "httpStatus": null, "errorType": "RemoteDisconnected", "errorCode": null, "elapsedSeconds": 3.776 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "eslint-2", "scoreDirectionExplicit": false, "sourceCommit": "12cbff4125db3f6446f72fa0c48c46b92b120427", "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "21441", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "INCOMPLETE", "status": "empty_or_truncated", "changedFileCount": 0, "modelCalls": 17, "toolCalls": 25, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "21441", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "INCOMPLETE", "status": "empty_or_truncated", "changedFileCount": 0, "modelCalls": 17, "toolCalls": 31, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "19931", "project": null, "arm": "memex", "historyRecords": 7, "attemptOutcome": "INCOMPLETE", "status": "empty_or_truncated", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "19931", "project": null, "arm": "keyword", "historyRecords": 7, "attemptOutcome": "INCOMPLETE", "status": "empty_or_truncated", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "INCOMPLETE": 4 }, "reviews": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 0 } }, { "blind": "B02", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": null } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 0 } } ], "posthocNormalization": null, "usage": { "claude-haiku-4-5": { "started": 54, "completed": 54, "reportedPromptTokens": 695682, "reportedCompletionTokens": 7019, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1950931, "responseBytes": 59144 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 9960, "reportedCompletionTokens": 4378, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 23957, "responseBytes": 12100 }, "gpt-6-astra": { "started": 4, "completed": 4, "reportedPromptTokens": 19265, "reportedCompletionTokens": 475, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 23809, "responseBytes": 3096 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "lodash-model-check", "scoreDirectionExplicit": false, "sourceCommit": "2b5e6f7399a7b48005140b5d5c6bc6c0e62919a8", "plannedAssignments": 2, "runs": [ { "slot": 1, "issue": "5881", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "5881", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 2 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "claude-haiku-4-5", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "claude-haiku-4-5", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 15, "completed": 15, "reportedPromptTokens": 78928, "reportedCompletionTokens": 1562, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 174642, "responseBytes": 12153 }, "deepseek-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 1426, "reportedCompletionTokens": 178, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 5928, "responseBytes": 1779 }, "claude-haiku-4-5": { "started": 2, "completed": 2, "reportedPromptTokens": 1595, "reportedCompletionTokens": 2, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 5864, "responseBytes": 1322 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "mixed-3", "scoreDirectionExplicit": true, "sourceCommit": { "pino": "b29dbe97820630e7128e464c50263b427965ec99", "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 10, "runs": [ { "slot": 1, "issue": "2057", "project": null, "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2057", "project": null, "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "19847", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 3, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "19847", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 17, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "20293", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "20293", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 19, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 7, "issue": "19427", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 8, "issue": "19427", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 18, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 9, "issue": "20047", "project": null, "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 18, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 10, "issue": "20047", "project": null, "arm": "memex", "historyRecords": 3, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 3, "toolCalls": 4, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 7, "CHECKS_NONZERO": 2, "INCOMPLETE": 1 }, "reviews": [ { "blind": "B01", "slot": 8, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B03", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } }, { "blind": "B04", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "B05", "slot": 7, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "B06", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B07", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B08", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B09", "slot": 9, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 109, "completed": 109, "reportedPromptTokens": 1479467, "reportedCompletionTokens": 14193, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 2749926, "responseBytes": 84329 }, "deepseek-flash": { "started": 9, "completed": 9, "reportedPromptTokens": 15819, "reportedCompletionTokens": 7667, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 45688, "responseBytes": 32032 }, "gpt-6.1-sol": { "started": 9, "completed": 9, "reportedPromptTokens": 123554, "reportedCompletionTokens": 1595, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 45355, "responseBytes": 9298 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "prettier-4", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "19968", "project": "prettier", "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 21, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "19968", "project": "prettier", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 4, "modelCalls": 13, "toolCalls": 19, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "19955", "project": "prettier", "arm": "memex", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "19955", "project": "prettier", "arm": "keyword", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 4 }, "reviews": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B03", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B04", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 52, "completed": 52, "reportedPromptTokens": 840966, "reportedCompletionTokens": 7253, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1680861, "responseBytes": 47231 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 9125, "reportedCompletionTokens": 573, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 30206, "responseBytes": 4788 }, "gpt-6.1-sol": { "started": 4, "completed": 4, "reportedPromptTokens": 62224, "reportedCompletionTokens": 1309, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 30058, "responseBytes": 4654 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "eslint-3", "scoreDirectionExplicit": true, "sourceCommit": { "eslint": "12cbff4125db3f6446f72fa0c48c46b92b120427" }, "plannedAssignments": 2, "runs": [ { "slot": 1, "issue": "21122", "project": "eslint", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "21122", "project": "eslint", "arm": "memex", "historyRecords": 5, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "CHECKS_NONZERO": 2 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 26, "completed": 26, "reportedPromptTokens": 309462, "reportedCompletionTokens": 3040, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 607676, "responseBytes": 21018 }, "deepseek-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 2917, "reportedCompletionTokens": 233, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 10978, "responseBytes": 2113 }, "gpt-6.1-sol": { "started": 2, "completed": 2, "reportedPromptTokens": 29459, "reportedCompletionTokens": 253, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 10904, "responseBytes": 2326 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "mixed-4", "scoreDirectionExplicit": true, "sourceCommit": { "eslint": "12cbff4125db3f6446f72fa0c48c46b92b120427", "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 8, "runs": [ { "slot": 1, "issue": "21361", "project": null, "arm": "keyword", "historyRecords": 2, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 7, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "21361", "project": null, "arm": "memex", "historyRecords": 2, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 7, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "21422", "project": null, "arm": "memex", "historyRecords": 2, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 3, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "21422", "project": null, "arm": "keyword", "historyRecords": 2, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 0, "modelCalls": 6, "toolCalls": 10, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "20306", "project": null, "arm": "keyword", "historyRecords": 1, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 6, "issue": "20306", "project": null, "arm": "memex", "historyRecords": 1, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 7, "issue": "20307", "project": null, "arm": "memex", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 8, "issue": "20307", "project": null, "arm": "keyword", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "PASS": 3, "UNKNOWN": 1, "NOT_RUN": 4 }, "reviews": [ { "blind": "B01", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "B02", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 32, "completed": 32, "reportedPromptTokens": 280602, "reportedCompletionTokens": 3382, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 722761, "responseBytes": 22611 }, "deepseek-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 4203, "reportedCompletionTokens": 371, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 14573, "responseBytes": 3214 }, "gpt-6.1-sol": { "started": 3, "completed": 3, "reportedPromptTokens": 17086, "reportedCompletionTokens": 702, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 14462, "responseBytes": 2527 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "19d2a81ea4236020683039fdd66c55aa0ca2806b378c5778bcd9ee7f121bcdf0", "at": "2026-10-31T14:35:28.955280+08:00", "label": "04:5", "model": "gpt-6-astra", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 120.856 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "prettier-interrupted", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "18080", "project": "prettier", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 0, "modelCalls": 1, "toolCalls": 0, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "18080", "project": "prettier", "arm": "memex", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 3, "issue": "10851", "project": "prettier", "arm": "memex", "historyRecords": 4, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 4, "issue": "10851", "project": "prettier", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 5, "issue": "12649", "project": "prettier", "arm": "memex", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 6, "issue": "12649", "project": "prettier", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "UNKNOWN": 1, "NOT_RUN": 5 }, "reviews": [], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 0, "reportedCompletionTokens": 0, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 4058, "responseBytes": 0 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "f85eb11efb332b8e7c5fe081be350582b36398049f0e263441d1385ec6dded58", "at": "2026-10-31T14:38:06.205441+08:00", "label": "01:0", "model": "gpt-6-astra", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 120.908 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "prettier-route", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 10, "runs": [ { "slot": 1, "issue": "8182", "project": "prettier", "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "8182", "project": "prettier", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 11, "toolCalls": 21, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "20307", "project": "prettier", "arm": "memex", "historyRecords": 2, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "20307", "project": "prettier", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 17, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "12649", "project": "prettier", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 24, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "12649", "project": "prettier", "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 19, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 7, "issue": "20306", "project": "prettier", "arm": "keyword", "historyRecords": 1, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 23, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 8, "issue": "20306", "project": "prettier", "arm": "memex", "historyRecords": 1, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 9, "issue": "10851", "project": "prettier", "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 18, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 10, "issue": "10851", "project": "prettier", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 0, "modelCalls": 13, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 9, "CHECKS_NONZERO": 1 }, "reviews": [ { "blind": "B01", "slot": 10, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "B03", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B04", "slot": 9, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "B05", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "error", "correctness": null, "sideEffects": null } }, { "blind": "B06", "slot": 7, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B07", "slot": 8, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B08", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B09", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B10", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6.1-sol": { "started": 126, "completed": 126, "reportedPromptTokens": 1970633, "reportedCompletionTokens": 14910, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 4377557, "responseBytes": 99870 }, "deepseek-flash": { "started": 10, "completed": 10, "reportedPromptTokens": 15279, "reportedCompletionTokens": 1243, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 52352, "responseBytes": 10921 }, "gpt-6-astra": { "started": 10, "completed": 10, "reportedPromptTokens": 48646, "reportedCompletionTokens": 2383, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 51982, "responseBytes": 8949 } }, "failures": [ { "log": "adjudication-calls.jsonl", "logSha256": "9d6fd599188197e5570f3ec68a4fbf4d2a621f625ee19d9537454cf527f7b596", "at": "2026-10-31T15:04:14.005901+08:00", "label": "B05", "model": "gpt-6.1-sol", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 120.794 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "prettier-six", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 12, "runs": [ { "slot": 1, "issue": "9112", "project": "prettier", "arm": "memex", "historyRecords": 1, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 19, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "9112", "project": "prettier", "arm": "keyword", "historyRecords": 1, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 19, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "20174", "project": "prettier", "arm": "keyword", "historyRecords": 1, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 25, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "20174", "project": "prettier", "arm": "memex", "historyRecords": 1, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 1, "toolCalls": 0, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "18066", "project": "prettier", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "18066", "project": "prettier", "arm": "memex", "historyRecords": 2, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 1, "modelCalls": 12, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 7, "issue": "9955", "project": "prettier", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 8, "issue": "9955", "project": "prettier", "arm": "memex", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 9, "issue": "17213", "project": "prettier", "arm": "memex", "historyRecords": 1, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 10, "issue": "17213", "project": "prettier", "arm": "keyword", "historyRecords": 1, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 11, "issue": "15537", "project": "prettier", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 12, "issue": "15537", "project": "prettier", "arm": "memex", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "CHECKS_NONZERO": 4, "INCOMPLETE": 1, "UNKNOWN": 1, "NOT_RUN": 6 }, "reviews": [ { "blind": "B01", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } }, { "blind": "B03", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } }, { "blind": "B05", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } } ], "posthocNormalization": null, "usage": { "gpt-6.1-sol": { "started": 65, "completed": 65, "reportedPromptTokens": 838193, "reportedCompletionTokens": 9719, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 2144009, "responseBytes": 49137 }, "deepseek-flash": { "started": 5, "completed": 5, "reportedPromptTokens": 8927, "reportedCompletionTokens": 539, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 28128, "responseBytes": 4928 }, "gpt-6-astra": { "started": 5, "completed": 5, "reportedPromptTokens": 30750, "reportedCompletionTokens": 512, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 27943, "responseBytes": 4193 }, "claude-haiku-4-5": { "started": 3, "completed": 3, "reportedPromptTokens": 6453, "reportedCompletionTokens": 709, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 18418, "responseBytes": 5155 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "d21003720cf2e3221fcff3bc399b64dea5fbae89c28d81eb74c9dc065583a322", "at": "2026-10-31T15:33:30.976043+08:00", "label": "06:11", "model": "gpt-6.1-sol", "status": "http_error", "httpStatus": 503, "errorType": null, "errorCode": "system_cpu_overloaded", "elapsedSeconds": 1.752 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "prettier-unstarted", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "9955", "project": "prettier", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 1, "checkErrorPresent": false }, { "slot": 2, "issue": "9955", "project": "prettier", "arm": "memex", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 3, "issue": "17213", "project": "prettier", "arm": "memex", "historyRecords": 1, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 4, "issue": "17213", "project": "prettier", "arm": "keyword", "historyRecords": 1, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 5, "issue": "15537", "project": "prettier", "arm": "memex", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 6, "issue": "15537", "project": "prettier", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "UNKNOWN": 1, "NOT_RUN": 5 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 0 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 0 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6.1-sol": { "started": 13, "completed": 13, "reportedPromptTokens": 131592, "reportedCompletionTokens": 1627, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 350295, "responseBytes": 8500 }, "deepseek-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 1685, "reportedCompletionTokens": 78, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 5342, "responseBytes": 831 }, "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 5961, "reportedCompletionTokens": 114, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 5305, "responseBytes": 810 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "caaff5c269c13bb0647bc01a4208f10fc548463ff51a25b0bc47375366ed9c0e", "at": "2026-10-31T15:39:31.519180+08:00", "label": "01:final", "model": "gpt-6.1-sol", "status": "http_error", "httpStatus": 503, "errorType": null, "errorCode": "system_cpu_overloaded", "elapsedSeconds": 1.63 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "prettier-lowcost", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "17213", "project": "prettier", "arm": "memex", "historyRecords": 1, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 0, "modelCalls": 13, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "17213", "project": "prettier", "arm": "keyword", "historyRecords": 1, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 0, "modelCalls": 13, "toolCalls": 13, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "15537", "project": "prettier", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 0, "modelCalls": 13, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "15537", "project": "prettier", "arm": "memex", "historyRecords": 2, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 0, "modelCalls": 13, "toolCalls": 22, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "CHECKS_NONZERO": 4 }, "reviews": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 }, { "model": "claude-haiku-4-5", "status": "INVALID", "eventStatus": "http_error", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "gpt-6-astra", "status": "MISSING" }, { "model": "claude-haiku-4-5", "status": "MISSING" } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 3, "ratings": [ { "model": "gpt-6-astra", "status": "MISSING" }, { "model": "claude-haiku-4-5", "status": "MISSING" } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 4, "ratings": [ { "model": "gpt-6-astra", "status": "MISSING" }, { "model": "claude-haiku-4-5", "status": "MISSING" } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "deepseek-flash": { "started": 52, "completed": 52, "reportedPromptTokens": 348529, "reportedCompletionTokens": 3991, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1212457, "responseBytes": 37690 }, "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 5840, "reportedCompletionTokens": 125, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 4745, "responseBytes": 707 }, "claude-haiku-4-5": { "started": 1, "completed": 1, "reportedPromptTokens": 0, "reportedCompletionTokens": 0, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 4750, "responseBytes": 143 } }, "failures": [ { "log": "review-calls.jsonl", "logSha256": "4c29ca3d0112c1e37ebe280b7ed20a46ebdeecaa1bc48b3c00dde29a1665fc22", "at": "2026-10-31T15:46:53.712180+08:00", "label": "B01:claude-haiku-4-5", "model": "claude-haiku-4-5", "status": "http_error", "httpStatus": 503, "errorType": null, "errorCode": "system_cpu_overloaded", "elapsedSeconds": 1.141 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "prettier-gateway", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "16126", "project": "prettier", "arm": "memex", "historyRecords": 8, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "16126", "project": "prettier", "arm": "keyword", "historyRecords": 8, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 24, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "17487", "project": "prettier", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "17487", "project": "prettier", "arm": "memex", "historyRecords": 4, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 3, "toolCalls": 4, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 3, "INCOMPLETE": 1 }, "reviews": [ { "blind": "B01", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "claude-haiku-4-5", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "claude-haiku-4-5", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "claude-haiku-4-5", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 35, "completed": 35, "reportedPromptTokens": 322356, "reportedCompletionTokens": 4588, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 804846, "responseBytes": 27914 }, "deepseek-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 5370, "reportedCompletionTokens": 455, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 17488, "responseBytes": 3561 }, "claude-haiku-4-5": { "started": 3, "completed": 3, "reportedPromptTokens": 6209, "reportedCompletionTokens": 3, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 17392, "responseBytes": 1992 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "dayjs-gateway", "scoreDirectionExplicit": true, "sourceCommit": { "dayjs": "436bde0bcded312781cbe45dc2b0ef079a36d8e3" }, "plannedAssignments": 12, "runs": [ { "slot": 1, "issue": "2511", "project": "dayjs", "arm": "memex", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 9, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2511", "project": "dayjs", "arm": "keyword", "historyRecords": 6, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 3, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "1923", "project": "dayjs", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 7, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "1923", "project": "dayjs", "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 6, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "2956", "project": "dayjs", "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 6, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "2956", "project": "dayjs", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 9, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 7, "issue": "2951", "project": "dayjs", "arm": "memex", "historyRecords": 8, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 2, "toolCalls": 3, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 8, "issue": "2951", "project": "dayjs", "arm": "keyword", "historyRecords": 8, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 8, "modelCalls": 13, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 9, "issue": "1271", "project": "dayjs", "arm": "keyword", "historyRecords": 19, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 10, "issue": "1271", "project": "dayjs", "arm": "memex", "historyRecords": 19, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 11, "issue": "2479", "project": "dayjs", "arm": "keyword", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 9, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 12, "issue": "2479", "project": "dayjs", "arm": "memex", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 25, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 10, "INCOMPLETE": 2 }, "reviews": [ { "blind": "B01", "slot": 11, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B02", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B03", "slot": 12, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 9, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B05", "slot": 8, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B06", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B07", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B08", "slot": 10, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B09", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B10", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 0 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 82, "completed": 82, "reportedPromptTokens": 803168, "reportedCompletionTokens": 8528, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1471662, "responseBytes": 65050 }, "deepseek-flash": { "started": 10, "completed": 10, "reportedPromptTokens": 15102, "reportedCompletionTokens": 1202, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 49819, "responseBytes": 10310 }, "qwen3.8-flash": { "started": 10, "completed": 10, "reportedPromptTokens": 15598, "reportedCompletionTokens": 15853, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 49469, "responseBytes": 64118 }, "gemini-3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 8968, "reportedCompletionTokens": 2784, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 27404, "responseBytes": 2807 } }, "failures": [], "qwenReportedOverLimit": [ { "at": "2026-10-31T10:59:28.853267+00:00", "label": "B07:qwen3.8-flash#attempt-1", "requestedMaxTokens": 700, "reportedCompletionTokens": 2907, "reportedReasoningTokens": 2791, "logSha256": "b8534c60a82e92dc82451420ac1e84f225dbcc2cbda558c3af105aedc7276396" }, { "at": "2026-10-31T11:00:05.055751+00:00", "label": "B08:qwen3.8-flash#attempt-1", "requestedMaxTokens": 700, "reportedCompletionTokens": 6526, "reportedReasoningTokens": 6392, "logSha256": "b8534c60a82e92dc82451420ac1e84f225dbcc2cbda558c3af105aedc7276396" }, { "at": "2026-10-31T11:01:18.986922+00:00", "label": "B09:qwen3.8-flash#attempt-1", "requestedMaxTokens": 700, "reportedCompletionTokens": 1919, "reportedReasoningTokens": null, "logSha256": "b8534c60a82e92dc82451420ac1e84f225dbcc2cbda558c3af105aedc7276396" }, { "at": "2026-10-31T11:01:36.573521+00:00", "label": "B10:qwen3.8-flash#attempt-1", "requestedMaxTokens": 700, "reportedCompletionTokens": 1657, "reportedReasoningTokens": null, "logSha256": "b8534c60a82e92dc82451420ac1e84f225dbcc2cbda558c3af105aedc7276396" } ], "solverReportedOverLimit": [] }, { "batch": "moment-mathjs", "scoreDirectionExplicit": true, "sourceCommit": { "moment": "c2a703a4d84aa57ee2722ff8fb4a8016279fb3c4", "mathjs": "4f5d049e8d6790d6774acbf73e774c19f5e20db7" }, "plannedAssignments": 8, "runs": [ { "slot": 1, "issue": "3980", "project": "moment", "arm": "memex", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 13, "toolCalls": 17, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "3980", "project": "moment", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 3, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "3614", "project": "mathjs", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "3614", "project": "mathjs", "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 9, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "3600", "project": "mathjs", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 12, "toolCalls": 11, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "3600", "project": "mathjs", "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 12, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 7, "issue": "4641", "project": "moment", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 2, "toolCalls": 4, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 8, "issue": "4641", "project": "moment", "arm": "memex", "historyRecords": 5, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 5, "INCOMPLETE": 2, "CHECKS_NONZERO": 1 }, "reviews": [ { "blind": "B01", "slot": 8, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 0 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B02", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B03", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B04", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "error", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B05", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "MISSING" }, { "model": "qwen3.8-flash", "status": "MISSING" } ], "registeredAdjudication": null }, { "blind": "B06", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "MISSING" }, { "model": "qwen3.8-flash", "status": "MISSING" } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 66, "completed": 66, "reportedPromptTokens": 947584, "reportedCompletionTokens": 10463, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1290487, "responseBytes": 58375 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 5618, "reportedCompletionTokens": 632, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 20469, "responseBytes": 4875 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 3406, "reportedCompletionTokens": 7929, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 20333, "responseBytes": 36745 }, "gemini-3.8-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 3921, "reportedCompletionTokens": 6132, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 15063, "responseBytes": 2926 } }, "failures": [ { "log": "review-calls.jsonl", "logSha256": "fba596ed5d8179a71da451a6f41bdd6433083f1d3195e5f8cab3fa88b903f766", "at": "2026-10-31T11:35:46.724530+00:00", "label": "B04:qwen3.8-flash#attempt-1", "model": "qwen3.8-flash", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 120.825 } ], "qwenReportedOverLimit": [ { "at": "2026-10-31T11:34:28.058936+00:00", "label": "B02:qwen3.8-flash#attempt-1", "requestedMaxTokens": 2048, "reportedCompletionTokens": 6106, "reportedReasoningTokens": null, "logSha256": "fba596ed5d8179a71da451a6f41bdd6433083f1d3195e5f8cab3fa88b903f766" } ], "solverReportedOverLimit": [] }, { "batch": "dayjs-mathjs", "scoreDirectionExplicit": true, "sourceCommit": { "dayjs": "436bde0bcded312781cbe45dc2b0ef079a36d8e3", "mathjs": "4f5d049e8d6790d6774acbf73e774c19f5e20db7" }, "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "3062", "project": "dayjs", "arm": "memex", "historyRecords": 3, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 1, "checkErrorPresent": false }, { "slot": 2, "issue": "3062", "project": "dayjs", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 2, "toolCalls": 1, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "3528", "project": "mathjs", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 13, "deniedToolCalls": null, "publicTestCalls": 3, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "3528", "project": "mathjs", "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 9, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "3563", "project": "mathjs", "arm": "keyword", "historyRecords": 7, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "3563", "project": "mathjs", "arm": "memex", "historyRecords": 7, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 0, "modelCalls": 1, "toolCalls": 0, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "CHECKS_NONZERO": 1, "INCOMPLETE": 1, "PASS": 3, "UNKNOWN": 1 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 0 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 0 } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B03", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B04", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 37, "completed": 37, "reportedPromptTokens": 313537, "reportedCompletionTokens": 4367, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 638764, "responseBytes": 28188 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 3288, "reportedCompletionTokens": 501, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 13968, "responseBytes": 4319 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 3573, "reportedCompletionTokens": 14259, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 13832, "responseBytes": 68579 }, "gemini-3.8-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 2947, "reportedCompletionTokens": 4980, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 13031, "responseBytes": 3220 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "781e1ba88666f5ef4e020575f8770a29b1170a1f83e0e372f7df7315564e4389", "at": "2026-10-31T11:38:50.450650+00:00", "label": "06:0#attempt-1", "model": "gpt-6-astra", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 120.787 } ], "qwenReportedOverLimit": [ { "at": "2026-10-31T11:42:20.101594+00:00", "label": "B02:qwen3.8-flash#attempt-1", "requestedMaxTokens": 2048, "reportedCompletionTokens": 7405, "reportedReasoningTokens": null, "logSha256": "67f934cc3ee8a68a5072b29e8d9cb0578ecc759d2d467e542132fd76ad5d64be" }, { "at": "2026-10-31T11:44:14.150539+00:00", "label": "B04:qwen3.8-flash#attempt-1", "requestedMaxTokens": 2048, "reportedCompletionTokens": 6040, "reportedReasoningTokens": null, "logSha256": "67f934cc3ee8a68a5072b29e8d9cb0578ecc759d2d467e542132fd76ad5d64be" } ], "solverReportedOverLimit": [] }, { "batch": "prettier-late", "scoreDirectionExplicit": true, "sourceCommit": { "prettier": "bed4edbb1cf2a9db4004215b0ce5707cdb02b5d9" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "16925", "project": "prettier", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 23, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "16925", "project": "prettier", "arm": "memex", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 25, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "11982", "project": "prettier", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 9, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "11982", "project": "prettier", "arm": "memex", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 9, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 4 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B02", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B03", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B04", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 42, "completed": 42, "reportedPromptTokens": 704565, "reportedCompletionTokens": 5331, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1269172, "responseBytes": 33764 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 8120, "reportedCompletionTokens": 512, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 26005, "responseBytes": 4345 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 8312, "reportedCompletionTokens": 2296, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 25869, "responseBytes": 13474 }, "gemini-3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 9345, "reportedCompletionTokens": 5743, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 29613, "responseBytes": 4217 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "mathjs-late", "scoreDirectionExplicit": true, "sourceCommit": { "mathjs": "4f5d049e8d6790d6774acbf73e774c19f5e20db7" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "3644", "project": "mathjs", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 2, "modelCalls": 10, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "3644", "project": "mathjs", "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "1905", "project": "mathjs", "arm": "keyword", "historyRecords": 7, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 11, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "1905", "project": "mathjs", "arm": "memex", "historyRecords": 7, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 11, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 4 }, "reviews": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B04", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 34, "completed": 34, "reportedPromptTokens": 334463, "reportedCompletionTokens": 3356, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 708753, "responseBytes": 26274 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 3376, "reportedCompletionTokens": 489, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 13693, "responseBytes": 4146 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 3175, "reportedCompletionTokens": 615, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 13557, "responseBytes": 4127 }, "gemini-3.8-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 2114, "reportedCompletionTokens": 2410, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 8679, "responseBytes": 2528 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "validator-new", "scoreDirectionExplicit": true, "sourceCommit": { "validator": "9ff342479591ca5a43cb30000195bcf55c1bbed9" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "2295", "project": "validator", "arm": "memex", "historyRecords": 5, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 2, "toolCalls": 1, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2295", "project": "validator", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "1309", "project": "validator", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 13, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "1309", "project": "validator", "arm": "memex", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "INCOMPLETE": 1, "PASS": 3 }, "reviews": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B03", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 33, "completed": 33, "reportedPromptTokens": 316331, "reportedCompletionTokens": 4399, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 588368, "responseBytes": 27750 }, "deepseek-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 3137, "reportedCompletionTokens": 347, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 12268, "responseBytes": 3106 }, "qwen3.8-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 3020, "reportedCompletionTokens": 422, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 12166, "responseBytes": 3069 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "luxon-new", "scoreDirectionExplicit": true, "sourceCommit": { "luxon": "f427515a38f6a671f8de663e6bcc040ed81f114e" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "1366", "project": "luxon", "arm": "memex", "historyRecords": 12, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 12, "toolCalls": 13, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "1366", "project": "luxon", "arm": "keyword", "historyRecords": 12, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "1189", "project": "luxon", "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 10, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "1189", "project": "luxon", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 8, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 4 }, "reviews": [ { "blind": "B01", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 37, "completed": 37, "reportedPromptTokens": 414758, "reportedCompletionTokens": 3896, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 832062, "responseBytes": 28699 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 4974, "reportedCompletionTokens": 513, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 19179, "responseBytes": 4311 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 4694, "reportedCompletionTokens": 3975, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 19043, "responseBytes": 20941 } }, "failures": [], "qwenReportedOverLimit": [ { "at": "2026-10-31T12:14:53.972251+00:00", "label": "B02:qwen3.8-flash#attempt-1", "requestedMaxTokens": 2048, "reportedCompletionTokens": 3423, "reportedReasoningTokens": 3301, "logSha256": "b8cc0f92cc8f64b52e70f5c42003bd0ad094d8aac418149a42a59a53ac6e25fc" } ], "solverReportedOverLimit": [] }, { "batch": "lodash-new", "scoreDirectionExplicit": true, "sourceCommit": { "lodash": "2b5e6f7399a7b48005140b5d5c6bc6c0e62919a8" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "6191", "project": "lodash", "arm": "keyword", "historyRecords": 2, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 10, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "6191", "project": "lodash", "arm": "memex", "historyRecords": 2, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 10, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "5948", "project": "lodash", "arm": "memex", "historyRecords": 0, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "5948", "project": "lodash", "arm": "keyword", "historyRecords": 0, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 4 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B02", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "B03", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 35, "completed": 35, "reportedPromptTokens": 269600, "reportedCompletionTokens": 3471, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 479089, "responseBytes": 26972 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 3860, "reportedCompletionTokens": 525, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 15565, "responseBytes": 4414 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 3644, "reportedCompletionTokens": 696, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 15429, "responseBytes": 4706 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "yup-new", "scoreDirectionExplicit": true, "sourceCommit": { "yup": "ff31eee8a2b10c938bb3a22544d9e33957a8cf01" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "2150", "project": "yup", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 11, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2150", "project": "yup", "arm": "memex", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 11, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "660", "project": "yup", "arm": "keyword", "historyRecords": 11, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 7, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "660", "project": "yup", "arm": "memex", "historyRecords": 11, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 4 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B04", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 34, "completed": 34, "reportedPromptTokens": 360996, "reportedCompletionTokens": 3011, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 680702, "responseBytes": 24731 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 4380, "reportedCompletionTokens": 462, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 18308, "responseBytes": 4234 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 4267, "reportedCompletionTokens": 469, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 18172, "responseBytes": 3818 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "validator-chartjs", "scoreDirectionExplicit": true, "sourceCommit": { "validator": "9ff342479591ca5a43cb30000195bcf55c1bbed9", "chartjs": "71d44219e79e3ac41e6f9dc907022eebf574868d" }, "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "2497", "project": "validator", "arm": "keyword", "historyRecords": 12, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 7, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2497", "project": "validator", "arm": "memex", "historyRecords": 12, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 7, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "912", "project": "validator", "arm": "keyword", "historyRecords": 8, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 0, "modelCalls": 2, "toolCalls": 1, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "912", "project": "validator", "arm": "memex", "historyRecords": 8, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 5, "issue": "12191", "project": "chartjs", "arm": "keyword", "historyRecords": 0, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 6, "issue": "12191", "project": "chartjs", "arm": "memex", "historyRecords": 0, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "PASS": 2, "UNKNOWN": 1, "NOT_RUN": 3 }, "reviews": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 0 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 16, "completed": 16, "reportedPromptTokens": 98654, "reportedCompletionTokens": 3083, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 143453, "responseBytes": 11641 }, "deepseek-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 1755, "reportedCompletionTokens": 264, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6116, "responseBytes": 2126 }, "qwen3.8-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 1664, "reportedCompletionTokens": 383, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6048, "responseBytes": 2202 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "135ee22bbc39ccb75ca260800e31cd36cc2db798dde610b4beb1012f06646cd5", "at": "2026-10-31T12:52:16.056599+00:00", "label": "03:1#attempt-1", "model": "gpt-6-astra", "status": "error", "httpStatus": null, "errorType": "URLError", "errorCode": null, "elapsedSeconds": 5.003 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "chartjs-unstarted", "scoreDirectionExplicit": true, "sourceCommit": { "chartjs": "71d44219e79e3ac41e6f9dc907022eebf574868d" }, "plannedAssignments": 2, "runs": [ { "slot": 1, "issue": "12191", "project": "chartjs", "arm": "memex", "historyRecords": 0, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 9, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 2, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "12191", "project": "chartjs", "arm": "keyword", "historyRecords": 0, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 13, "deniedToolCalls": null, "publicTestCalls": 3, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 2 }, "reviews": [ { "blind": "B01", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 19, "completed": 19, "reportedPromptTokens": 139540, "reportedCompletionTokens": 1815, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 239808, "responseBytes": 13489 }, "deepseek-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 1597, "reportedCompletionTokens": 169, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6723, "responseBytes": 1771 }, "qwen3.8-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 1495, "reportedCompletionTokens": 274, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6655, "responseBytes": 1895 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "independent-27", "scoreDirectionExplicit": true, "sourceCommit": { "validator": "9ff342479591ca5a43cb30000195bcf55c1bbed9", "ramda": "f4b2e35ff433e18c751326b82acc3c6ab3585e18", "datefns": "a9739fb3e5384a810e90d7376446f6d84ecc5029", "mathjs": "4f5d049e8d6790d6774acbf73e774c19f5e20db7", "dayjs": "436bde0bcded312781cbe45dc2b0ef079a36d8e3" }, "plannedAssignments": 10, "runs": [ { "slot": 1, "issue": "4096", "project": "datefns", "arm": "memex", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "4096", "project": "datefns", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 7, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "3134", "project": "mathjs", "arm": "keyword", "historyRecords": 11, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 2, "toolCalls": 4, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "3134", "project": "mathjs", "arm": "memex", "historyRecords": 11, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 12, "toolCalls": 20, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "2137", "project": "ramda", "arm": "keyword", "historyRecords": 7, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 16, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "2137", "project": "ramda", "arm": "memex", "historyRecords": 7, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 5, "toolCalls": 15, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 7, "issue": "2430", "project": "validator", "arm": "memex", "historyRecords": 3, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 5, "toolCalls": 12, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 8, "issue": "2430", "project": "validator", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 14, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 9, "issue": "2281", "project": "dayjs", "arm": "keyword", "historyRecords": 9, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 10, "toolCalls": 25, "deniedToolCalls": null, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 10, "issue": "2281", "project": "dayjs", "arm": "memex", "historyRecords": 9, "attemptOutcome": "INCOMPLETE", "status": "tool_call_limit", "changedFileCount": 0, "modelCalls": 6, "toolCalls": 17, "deniedToolCalls": null, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 6, "INCOMPLETE": 4 }, "reviews": [ { "blind": "B01", "slot": 8, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B05", "slot": 9, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B06", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 76, "completed": 76, "reportedPromptTokens": 496965, "reportedCompletionTokens": 12574, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1401746, "responseBytes": 61542 }, "deepseek-flash": { "started": 6, "completed": 6, "reportedPromptTokens": 6819, "reportedCompletionTokens": 732, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 23813, "responseBytes": 6109 }, "qwen3.8-flash": { "started": 6, "completed": 6, "reportedPromptTokens": 6220, "reportedCompletionTokens": 1046, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 23609, "responseBytes": 6792 }, "gpt-6.1-sol": { "started": 2, "completed": 2, "reportedPromptTokens": 47598, "reportedCompletionTokens": 354, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 10516, "responseBytes": 2334 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "independent-28", "scoreDirectionExplicit": true, "sourceCommit": { "qs": "43baea4655cb9b543fb1c5cc0db95cde42852140", "express": "1c6f1b0fe4000977b9a7b4f34097e4d527dd9824", "cheerio": "70b0dcefb8514298d255ca7ca0ec26b7bc63475a", "mathjs": "4f5d049e8d6790d6774acbf73e774c19f5e20db7" }, "plannedAssignments": 8, "runs": [ { "slot": 1, "issue": "3620", "project": "mathjs", "arm": "memex", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 11, "toolCalls": 15, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "3620", "project": "mathjs", "arm": "keyword", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 8, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "581", "project": "cheerio", "arm": "keyword", "historyRecords": 3, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 0, "modelCalls": 11, "toolCalls": 18, "deniedToolCalls": 0, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "581", "project": "cheerio", "arm": "memex", "historyRecords": 3, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 5, "issue": "432", "project": "qs", "arm": "memex", "historyRecords": 6, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 6, "issue": "432", "project": "qs", "arm": "keyword", "historyRecords": 6, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 7, "issue": "7350", "project": "express", "arm": "keyword", "historyRecords": 8, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 8, "issue": "7350", "project": "express", "arm": "memex", "historyRecords": 8, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "PASS": 2, "UNKNOWN": 1, "NOT_RUN": 5 }, "reviews": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 29, "completed": 29, "reportedPromptTokens": 139458, "reportedCompletionTokens": 4912, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 602518, "responseBytes": 20400 }, "deepseek-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 1452, "reportedCompletionTokens": 185, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6243, "responseBytes": 1788 }, "qwen3.8-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 1357, "reportedCompletionTokens": 216, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6175, "responseBytes": 1734 }, "gpt-6.1-sol": { "started": 1, "completed": 1, "reportedPromptTokens": 23257, "reportedCompletionTokens": 148, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 3735, "responseBytes": 1469 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "7b6aa2135c97aadf76bd0faf57b40af150e8dab020fcd2127d0c9490e1883ba5", "at": "2026-10-31T14:37:29.060872+00:00", "label": "03:10#attempt-1", "model": "gpt-6-astra", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 120.798 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "setup-stop-29", "scoreDirectionExplicit": false, "sourceCommit": { "koa": "b786e7b456782c2cff1d6549cd4c8dba3feeb262", "dayjs": "436bde0bcded312781cbe45dc2b0ef079a36d8e3", "datefns": "a9739fb3e5384a810e90d7376446f6d84ecc5029", "express": "1c6f1b0fe4000977b9a7b4f34097e4d527dd9824" }, "plannedAssignments": 8, "runs": [ { "slot": 1, "issue": "7362", "project": "express", "arm": "memex", "historyRecords": 7, "attemptOutcome": "UNKNOWN", "status": "setup_failure", "changedFileCount": 0, "modelCalls": 0, "toolCalls": 0, "deniedToolCalls": 0, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 2, "issue": "7362", "project": "express", "arm": "keyword", "historyRecords": 7, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 3, "issue": "1134", "project": "datefns", "arm": "memex", "historyRecords": 9, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 4, "issue": "1134", "project": "datefns", "arm": "keyword", "historyRecords": 9, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 5, "issue": "2179", "project": "dayjs", "arm": "keyword", "historyRecords": 9, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 6, "issue": "2179", "project": "dayjs", "arm": "memex", "historyRecords": 9, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 7, "issue": "827", "project": "koa", "arm": "keyword", "historyRecords": 12, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 8, "issue": "827", "project": "koa", "arm": "memex", "historyRecords": 12, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "UNKNOWN": 1, "NOT_RUN": 7 }, "reviews": [], "posthocNormalization": null, "usage": {}, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "independent-30", "scoreDirectionExplicit": true, "sourceCommit": { "koa": "b786e7b456782c2cff1d6549cd4c8dba3feeb262", "dayjs": "436bde0bcded312781cbe45dc2b0ef079a36d8e3", "datefns": "a9739fb3e5384a810e90d7376446f6d84ecc5029", "express": "1c6f1b0fe4000977b9a7b4f34097e4d527dd9824" }, "plannedAssignments": 8, "runs": [ { "slot": 1, "issue": "7362", "project": "express", "arm": "keyword", "historyRecords": 7, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 8, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "7362", "project": "express", "arm": "memex", "historyRecords": 7, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 11, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "827", "project": "koa", "arm": "keyword", "historyRecords": 12, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "827", "project": "koa", "arm": "memex", "historyRecords": 12, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 16, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "1134", "project": "datefns", "arm": "keyword", "historyRecords": 9, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 3, "modelCalls": 13, "toolCalls": 34, "deniedToolCalls": 0, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "1134", "project": "datefns", "arm": "memex", "historyRecords": 9, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 13, "deniedToolCalls": 0, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 7, "issue": "2179", "project": "dayjs", "arm": "memex", "historyRecords": 9, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 10, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 8, "issue": "2179", "project": "dayjs", "arm": "keyword", "historyRecords": 9, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 12, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "CHECKS_NONZERO": 3, "PASS": 5 }, "reviews": [ { "blind": "B01", "slot": 7, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B02", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } }, { "blind": "B03", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": null } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } }, { "blind": "B04", "slot": 8, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B05", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "B06", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B07", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": null } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } }, { "blind": "B08", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 54, "completed": 54, "reportedPromptTokens": 377358, "reportedCompletionTokens": 17513, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 1274042, "responseBytes": 51238 }, "deepseek-flash": { "started": 8, "completed": 8, "reportedPromptTokens": 9026, "reportedCompletionTokens": 958, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 33437, "responseBytes": 8070 }, "qwen3.8-flash": { "started": 8, "completed": 8, "reportedPromptTokens": 8644, "reportedCompletionTokens": 1302, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 33165, "responseBytes": 8410 }, "gpt-6.1-sol": { "started": 6, "completed": 6, "reportedPromptTokens": 50706, "reportedCompletionTokens": 950, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 28424, "responseBytes": 5716 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [ { "at": "2026-10-31T15:00:49.259180+00:00", "label": "05:6#attempt-1", "model": "gpt-6-astra", "requestedMaxTokens": 1000, "reportedCompletionTokens": 2234, "logSha256": "069def2fddb7b4b1b1ed1cc6db378d0123452a51346dddc9c3f165ffbbf089ae" }, { "at": "2026-10-31T15:01:12.687943+00:00", "label": "05:7#attempt-1", "model": "gpt-6-astra", "requestedMaxTokens": 1000, "reportedCompletionTokens": 1402, "logSha256": "069def2fddb7b4b1b1ed1cc6db378d0123452a51346dddc9c3f165ffbbf089ae" }, { "at": "2026-10-31T15:06:02.610483+00:00", "label": "08:3#attempt-1", "model": "gpt-6-astra", "requestedMaxTokens": 1000, "reportedCompletionTokens": 2661, "logSha256": "069def2fddb7b4b1b1ed1cc6db378d0123452a51346dddc9c3f165ffbbf089ae" }, { "at": "2026-10-31T15:07:35.178569+00:00", "label": "08:4#attempt-1", "model": "gpt-6-astra", "requestedMaxTokens": 1000, "reportedCompletionTokens": 1383, "logSha256": "069def2fddb7b4b1b1ed1cc6db378d0123452a51346dddc9c3f165ffbbf089ae" } ] }, { "batch": "independent-31", "scoreDirectionExplicit": true, "sourceCommit": { "dayjs": "436bde0bcded312781cbe45dc2b0ef079a36d8e3", "validator": "9ff342479591ca5a43cb30000195bcf55c1bbed9", "mathjs": "4f5d049e8d6790d6774acbf73e774c19f5e20db7" }, "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "2442", "project": "dayjs", "arm": "memex", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 8, "toolCalls": 18, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2442", "project": "dayjs", "arm": "keyword", "historyRecords": 4, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "2036", "project": "validator", "arm": "keyword", "historyRecords": 11, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "2036", "project": "validator", "arm": "memex", "historyRecords": 11, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "3604", "project": "mathjs", "arm": "keyword", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 13, "toolCalls": 21, "deniedToolCalls": 0, "publicTestCalls": 0, "forcedFinal": true, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 6, "issue": "3604", "project": "mathjs", "arm": "memex", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 6 }, "reviews": [ { "blind": "B01", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 5, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } }, { "blind": "B05", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B06", "slot": 6, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 41, "completed": 41, "reportedPromptTokens": 392325, "reportedCompletionTokens": 7321, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 811221, "responseBytes": 37705 }, "deepseek-flash": { "started": 6, "completed": 6, "reportedPromptTokens": 6574, "reportedCompletionTokens": 982, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 25098, "responseBytes": 7335 }, "qwen3.8-flash": { "started": 6, "completed": 6, "reportedPromptTokens": 6853, "reportedCompletionTokens": 3071, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 24894, "responseBytes": 17115 }, "gpt-6.1-sol": { "started": 1, "completed": 1, "reportedPromptTokens": 6430, "reportedCompletionTokens": 226, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 8175, "responseBytes": 1246 } }, "failures": [], "qwenReportedOverLimit": [], "solverReportedOverLimit": [ { "at": "2026-10-31T15:19:08.409062+00:00", "label": "06:2#attempt-1", "model": "gpt-6-astra", "requestedMaxTokens": 1000, "reportedCompletionTokens": 2012, "logSha256": "ca218fb237307010f9aff3c9853aa31f5fbcf6f318f357929ef996106e341cdd" } ] }, { "batch": "independent-32", "scoreDirectionExplicit": true, "sourceCommit": { "underscore": "5c1e5f686f8500af9c9d15614ed60289d6d35d6b", "validator": "9ff342479591ca5a43cb30000195bcf55c1bbed9", "yup": "ff31eee8a2b10c938bb3a22544d9e33957a8cf01" }, "plannedAssignments": 6, "runs": [ { "slot": 1, "issue": "2433", "project": "validator", "arm": "memex", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 8, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2433", "project": "validator", "arm": "keyword", "historyRecords": 6, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "2214", "project": "underscore", "arm": "keyword", "historyRecords": 9, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 6, "toolCalls": 7, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "2214", "project": "underscore", "arm": "memex", "historyRecords": 9, "attemptOutcome": "UNKNOWN", "status": "model_failure", "changedFileCount": 0, "modelCalls": 4, "toolCalls": 12, "deniedToolCalls": 0, "publicTestCalls": 0, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 5, "issue": "2252", "project": "yup", "arm": "memex", "historyRecords": 13, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false }, { "slot": 6, "issue": "2252", "project": "yup", "arm": "keyword", "historyRecords": 13, "attemptOutcome": "NOT_RUN", "status": "NOT_RUN_AFTER_STOP", "changedFileCount": 0, "modelCalls": null, "toolCalls": null, "deniedToolCalls": null, "publicTestCalls": null, "forcedFinal": false, "acceptanceExit": null, "regressionExit": null, "checkErrorPresent": false } ], "outcomes": { "PASS": 3, "UNKNOWN": 1, "NOT_RUN": 2 }, "reviews": [ { "blind": "B01", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null }, { "blind": "B03", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } ], "registeredAdjudication": null } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 21, "completed": 21, "reportedPromptTokens": 102060, "reportedCompletionTokens": 2910, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 266492, "responseBytes": 15323 }, "deepseek-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 3374, "reportedCompletionTokens": 345, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 12672, "responseBytes": 3007 }, "qwen3.8-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 3668, "reportedCompletionTokens": 2114, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 12570, "responseBytes": 11676 } }, "failures": [ { "log": "solver-calls.jsonl", "logSha256": "29bae17aacffc20efbe32e9457f80fd55ebce4542cd6fa2a8ca1bf07ec139ec7", "at": "2026-10-31T15:27:50.571251+00:00", "label": "04:3#attempt-1", "model": "gpt-6-astra", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 120.869 } ], "qwenReportedOverLimit": [], "solverReportedOverLimit": [] }, { "batch": "independent-35", "scoreDirectionExplicit": true, "sourceCommit": { "yup": "ff31eee8a2b10c938bb3a22544d9e33957a8cf01", "mathjs": "4f5d049e8d6790d6774acbf73e774c19f5e20db7" }, "plannedAssignments": 4, "runs": [ { "slot": 1, "issue": "2252", "project": "yup", "arm": "memex", "historyRecords": 13, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 2, "issue": "2252", "project": "yup", "arm": "keyword", "historyRecords": 13, "attemptOutcome": "PASS", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 0, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 3, "issue": "3608", "project": "mathjs", "arm": "keyword", "historyRecords": 5, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 7, "toolCalls": 6, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false }, { "slot": 4, "issue": "3608", "project": "mathjs", "arm": "memex", "historyRecords": 5, "attemptOutcome": "CHECKS_NONZERO", "status": "completed", "changedFileCount": 1, "modelCalls": 5, "toolCalls": 4, "deniedToolCalls": 0, "publicTestCalls": 1, "forcedFinal": false, "acceptanceExit": 1, "regressionExit": 0, "checkErrorPresent": false } ], "outcomes": { "PASS": 2, "CHECKS_NONZERO": 2 }, "reviews": [ { "blind": "B01", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } ], "registeredAdjudication": null }, { "blind": "B02", "slot": 4, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 } }, { "blind": "B03", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": null }, { "blind": "B04", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } } ], "posthocNormalization": null, "usage": { "gpt-6-astra": { "started": 22, "completed": 22, "reportedPromptTokens": 165677, "reportedCompletionTokens": 3341, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 268642, "responseBytes": 15633 }, "deepseek-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 4762, "reportedCompletionTokens": 384, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 19722, "responseBytes": 3644 }, "qwen3.8-flash": { "started": 4, "completed": 4, "reportedPromptTokens": 4599, "reportedCompletionTokens": 5255, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 19586, "responseBytes": 25785 }, "gpt-6.1-sol": { "started": 2, "completed": 2, "reportedPromptTokens": 11070, "reportedCompletionTokens": 479, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 11289, "responseBytes": 2005 } }, "failures": [], "qwenReportedOverLimit": [ { "at": "2026-10-31T15:43:58.288683+00:00", "label": "B02:qwen3.8-flash#attempt-1", "requestedMaxTokens": 2048, "reportedCompletionTokens": 4827, "reportedReasoningTokens": 4755, "logSha256": "2258ac1a3df7ee47d88a5275a30fc86181c626c2dda8f3017a4b94c6bd8362f3" } ], "solverReportedOverLimit": [] } ], "newPilotSummary": { "distinctIssues": 30, "issuesWithAnyStartedArm": 30, "assignmentOutcomes": { "PASS": 48, "INCOMPLETE": 7, "CHECKS_NONZERO": 2, "UNKNOWN": 2, "NOT_RUN": 1 }, "bothArmsCompleteAndChecksPass": 21, "bothArmsCompleteWithTwoOriginalFormatValidReviews": 16, "originalReviewStatuses": { "VALID": 89, "INVALID": 7, "MISSING": 4 }, "originalComparableCandidateReviewPairs": 41, "originalScoreDisagreements": 28, "originalDisagreementAdjudications": { "INVALID": 19, "VALID": 9 }, "noHistoryControls": [ "chartjs#12191", "lodash#5948" ] }, "supplementaryCoverage": { "issuePairsWithOriginalOrSupplementaryRatings": 20, "noHistoryControlsIncluded": 2, "issuePairsExcludingNoHistoryControls": 18, "additionalIssuesAfterSupplementaryReviews": [ "dayjs#1271", "dayjs#2479", "mathjs#3614", "prettier#16126" ] }, "distinctIssueFollowupSummary": { "distinctIssues": 20, "assignmentOutcomes": { "PASS": 24, "INCOMPLETE": 4, "UNKNOWN": 2, "NOT_RUN": 5, "CHECKS_NONZERO": 5 }, "bothArmsCompleteWithTwoFormatValidReviews": 9, "bothArmsCompleteWithDisagreementsResolved": 9, "resolvedCorrectnessComparisons": { "KEYWORD_HIGHER": 1, "EQUAL": 7, "MEMEX_HIGHER": 1 }, "reviewStatuses": { "VALID": 52, "INVALID": 6 }, "disagreementAdjudications": { "VALID": 12 }, "issues": [ { "project": "cheerio", "issue": "581", "arms": { "keyword": "UNKNOWN", "memex": "NOT_RUN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "datefns", "issue": "1134", "arms": { "keyword": "PASS", "memex": "CHECKS_NONZERO" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 2, "memex": 0 }, "usedSupplementaryReviews": false }, { "project": "datefns", "issue": "4096", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false }, { "project": "dayjs", "issue": "2179", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false }, { "project": "dayjs", "issue": "2281", "arms": { "keyword": "PASS", "memex": "INCOMPLETE" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "dayjs", "issue": "2442", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "express", "issue": "7350", "arms": { "keyword": "NOT_RUN", "memex": "NOT_RUN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "express", "issue": "7362", "arms": { "keyword": "CHECKS_NONZERO", "memex": "CHECKS_NONZERO" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 0, "memex": 0 }, "usedSupplementaryReviews": false }, { "project": "koa", "issue": "827", "arms": { "keyword": "PASS", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 2, "memex": 2 }, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3134", "arms": { "keyword": "INCOMPLETE", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3604", "arms": { "keyword": "PASS", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 1, "memex": 1 }, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3608", "arms": { "keyword": "CHECKS_NONZERO", "memex": "CHECKS_NONZERO" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 0, "memex": 1 }, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3620", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false }, { "project": "qs", "issue": "432", "arms": { "memex": "NOT_RUN", "keyword": "NOT_RUN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "ramda", "issue": "2137", "arms": { "keyword": "PASS", "memex": "INCOMPLETE" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "underscore", "issue": "2214", "arms": { "keyword": "PASS", "memex": "UNKNOWN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "validator", "issue": "2036", "arms": { "keyword": "PASS", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "validator", "issue": "2430", "arms": { "memex": "INCOMPLETE", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "validator", "issue": "2433", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "yup", "issue": "2252", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false } ] }, "distinctIssueFollowupSupplementaryCoverage": { "distinctIssues": 20, "assignmentOutcomes": { "PASS": 24, "INCOMPLETE": 4, "UNKNOWN": 2, "NOT_RUN": 5, "CHECKS_NONZERO": 5 }, "bothArmsCompleteWithTwoFormatValidReviews": 12, "bothArmsCompleteWithDisagreementsResolved": 12, "resolvedCorrectnessComparisons": { "KEYWORD_HIGHER": 1, "EQUAL": 8, "MEMEX_HIGHER": 3 }, "reviewStatuses": { "VALID": 58 }, "disagreementAdjudications": { "VALID": 13, "INVALID": 1 }, "issues": [ { "project": "cheerio", "issue": "581", "arms": { "keyword": "UNKNOWN", "memex": "NOT_RUN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "datefns", "issue": "1134", "arms": { "keyword": "PASS", "memex": "CHECKS_NONZERO" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 2, "memex": 0 }, "usedSupplementaryReviews": false }, { "project": "datefns", "issue": "4096", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false }, { "project": "dayjs", "issue": "2179", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false }, { "project": "dayjs", "issue": "2281", "arms": { "keyword": "PASS", "memex": "INCOMPLETE" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "dayjs", "issue": "2442", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 1 }, "usedSupplementaryReviews": true }, { "project": "express", "issue": "7350", "arms": { "keyword": "NOT_RUN", "memex": "NOT_RUN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "express", "issue": "7362", "arms": { "keyword": "CHECKS_NONZERO", "memex": "CHECKS_NONZERO" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 0, "memex": 0 }, "usedSupplementaryReviews": false }, { "project": "koa", "issue": "827", "arms": { "keyword": "PASS", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 2, "memex": 2 }, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3134", "arms": { "keyword": "INCOMPLETE", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3604", "arms": { "keyword": "PASS", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 1, "memex": 1 }, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3608", "arms": { "keyword": "CHECKS_NONZERO", "memex": "CHECKS_NONZERO" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 0, "memex": 1 }, "usedSupplementaryReviews": false }, { "project": "mathjs", "issue": "3620", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false }, { "project": "qs", "issue": "432", "arms": { "memex": "NOT_RUN", "keyword": "NOT_RUN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "ramda", "issue": "2137", "arms": { "keyword": "PASS", "memex": "INCOMPLETE" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "underscore", "issue": "2214", "arms": { "keyword": "PASS", "memex": "UNKNOWN" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": true }, { "project": "validator", "issue": "2036", "arms": { "keyword": "PASS", "memex": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "keyword": 2, "memex": 2 }, "usedSupplementaryReviews": true }, { "project": "validator", "issue": "2430", "arms": { "memex": "INCOMPLETE", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": false, "bothArmsCompleteWithDisagreementsResolved": false, "effectiveCorrectness": {}, "usedSupplementaryReviews": false }, { "project": "validator", "issue": "2433", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 1 }, "usedSupplementaryReviews": true }, { "project": "yup", "issue": "2252", "arms": { "memex": "PASS", "keyword": "PASS" }, "bothArmsCompleteWithTwoFormatValidReviews": true, "bothArmsCompleteWithDisagreementsResolved": true, "effectiveCorrectness": { "memex": 2, "keyword": 2 }, "usedSupplementaryReviews": false } ] }, "posthocReviewsV34": { "kind": "posthoc-alternate-blind-reviews-v34", "selection": "Completed batch 31 and 32 attempts with valid DeepSeek and completed but format-invalid Qwen reviews; exclude missing or failed/timeout reviews.", "reviewer": "gpt-6.1-sol", "adjudicator": "gpt-6-astra", "candidates": [ { "blind": "S01", "source": "20261031-independent-batch31", "sourceBlind": "B03", "slot": 4, "issue": "2036", "arm": "memex", "firstReview": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, "supplementaryReview": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, "supplementaryAdjudication": { "status": "MISSING" } }, { "blind": "S02", "source": "20261031-independent-batch31", "sourceBlind": "B02", "slot": 1, "issue": "2442", "arm": "memex", "firstReview": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "supplementaryReview": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "supplementaryAdjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "S03", "source": "20261031-independent-batch31", "sourceBlind": "B01", "slot": 3, "issue": "2036", "arm": "keyword", "firstReview": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, "supplementaryReview": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, "supplementaryAdjudication": { "status": "MISSING" } }, { "blind": "S04", "source": "20261031-independent-batch32", "sourceBlind": "B02", "slot": 2, "issue": "2433", "arm": "keyword", "firstReview": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "supplementaryReview": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "supplementaryAdjudication": { "status": "MISSING" } }, { "blind": "S05", "source": "20261031-independent-batch32", "sourceBlind": "B01", "slot": 3, "issue": "2214", "arm": "keyword", "firstReview": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "supplementaryReview": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 }, "supplementaryAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "S06", "source": "20261031-independent-batch32", "sourceBlind": "B03", "slot": 1, "issue": "2433", "arm": "memex", "firstReview": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "supplementaryReview": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "supplementaryAdjudication": { "status": "MISSING" } } ], "usage": { "gpt-6.1-sol": { "started": 6, "completed": 6, "reportedPromptTokens": 121229, "reportedCompletionTokens": 1393, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 19260, "responseBytes": 5882 }, "gpt-6-astra": { "started": 2, "completed": 2, "reportedPromptTokens": 11948, "reportedCompletionTokens": 439, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 7782, "responseBytes": 2162 } }, "failures": [], "limitations": [ "Selected and model substituted after observing original review failures; originals remain unchanged.", "Adjudicator uses the solver model label, so is not independent of the solver.", "All AI model labels share one proxy with unverified provenance; no human review.", "Format validity does not prove correctness; tests are partial." ] }, "historicalResolutionCoverage": { "excludesNoHistoryControls": true, "beforeV33": { "pairsWithTwoValidRatingsPerArm": 18, "pairsWithValidDisagreementAdjudications": 14, "resolvedIssueIds": [ "dayjs#1271", "dayjs#1923", "dayjs#2479", "dayjs#2956", "luxon#1366", "mathjs#1905", "mathjs#3528", "mathjs#3614", "mathjs#3644", "prettier#16925", "validator#1309", "validator#2497", "yup#2150", "yup#660" ], "resolvedCorrectnessComparisons": { "EQUAL": 13, "KEYWORD_HIGHER": 1 }, "unresolvedCandidates": [ { "project": "prettier", "issue": "16126", "arm": "memex", "batch": "prettier-gateway", "slot": 1 }, { "project": "prettier", "issue": "16126", "arm": "keyword", "batch": "prettier-gateway", "slot": 2 }, { "project": "prettier", "issue": "11982", "arm": "memex", "batch": "prettier-late", "slot": 4 }, { "project": "luxon", "issue": "1189", "arm": "memex", "batch": "luxon-new", "slot": 3 }, { "project": "lodash", "issue": "6191", "arm": "keyword", "batch": "lodash-new", "slot": 1 } ] }, "includingV33": { "pairsWithTwoValidRatingsPerArm": 18, "pairsWithValidDisagreementAdjudications": 18, "resolvedIssueIds": [ "dayjs#1271", "dayjs#1923", "dayjs#2479", "dayjs#2956", "lodash#6191", "luxon#1189", "luxon#1366", "mathjs#1905", "mathjs#3528", "mathjs#3614", "mathjs#3644", "prettier#11982", "prettier#16126", "prettier#16925", "validator#1309", "validator#2497", "yup#2150", "yup#660" ], "resolvedCorrectnessComparisons": { "EQUAL": 17, "KEYWORD_HIGHER": 1 }, "unresolvedCandidates": [] }, "note": "Supplementary post-hoc ratings and adjudications are included where identified; format validity does not establish correctness." }, "posthocAdjudicationsV33": { "kind": "posthoc-five-invalid-disagreement-adjudications-v33", "selection": "Five disagreements in four otherwise fully rated historical pairs whose prior adjudications completed but failed the format check. No timed-out adjudication is replayed.", "model": "gpt-6.1-sol", "candidates": [ { "blind": "R01", "source": "20261031-batch15-posthoc-budgeted", "sourceBlind": "B02", "slot": 1, "issue": "16126", "arm": "memex", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "R02", "source": "20261031-lodash-batch23", "sourceBlind": "B01", "slot": 1, "issue": "6191", "arm": "keyword", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "R03", "source": "20261031-prettier-batch19", "sourceBlind": "B04", "slot": 4, "issue": "11982", "arm": "memex", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "R04", "source": "20261031-luxon-batch22", "sourceBlind": "B01", "slot": 3, "issue": "1189", "arm": "memex", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "R05", "source": "20261031-batch15-posthoc-budgeted", "sourceBlind": "B01", "slot": 2, "issue": "16126", "arm": "keyword", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "usage": { "gpt-6.1-sol": { "started": 5, "completed": 5, "reportedPromptTokens": 83031, "reportedCompletionTokens": 967, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 27351, "responseBytes": 5706 } }, "failures": [], "limitations": [ "Selected after observing earlier ratings and adjudications; all originals remain separate.", "Different AI model labels share one proxy with unverified provenance; no human review or proven independent provider.", "Format validity does not prove correctness; partial checks are not upstream acceptance." ] }, "posthocExplicitRubric": { "kind": "posthoc-explicit-rubric-blind-rereview", "selection": "Only attempts with a nonempty code diff; other attempts are not reviewed in this post-hoc pass.", "candidates": [ { "blind": "R01", "batch": "20261031-eslint-batch4", "slot": 3, "issue": "19931", "arm": "memex", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "adjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 } }, { "blind": "R02", "batch": "20261031-eslint-batch1", "slot": 5, "issue": "21436", "arm": "keyword", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } ], "adjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } }, { "blind": "R03", "batch": "20261031-lodash-batch5", "slot": 1, "issue": "5881", "arm": "memex", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "adjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "R04", "batch": "20261031-lodash-batch5", "slot": 2, "issue": "5881", "arm": "keyword", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, { "model": "gpt-6.1-sol", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "adjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "R05", "batch": "20261031-eslint-batch1", "slot": 6, "issue": "21436", "arm": "memex", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "adjudication": null }, { "blind": "R06", "batch": "20261031-eslint-batch4", "slot": 4, "issue": "19931", "arm": "keyword", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 1 }, { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } ], "adjudication": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 2 } } ], "usage": { "deepseek-flash": { "started": 6, "completed": 6, "reportedPromptTokens": 9999, "reportedCompletionTokens": 6412, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 25518, "responseBytes": 23029 }, "gpt-6-astra": { "started": 6, "completed": 6, "reportedPromptTokens": 33654, "reportedCompletionTokens": 843, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 27422, "responseBytes": 5420 }, "gpt-6.1-sol": { "started": 2, "completed": 2, "reportedPromptTokens": 28117, "reportedCompletionTokens": 311, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6534, "responseBytes": 2261 }, "claude-haiku-4-5": { "started": 3, "completed": 3, "reportedPromptTokens": 4967, "reportedCompletionTokens": 686, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 16717, "responseBytes": 4972 } }, "limitations": [ "Post-hoc selection, rubric and replacement of empty reviewer with gpt-6.1-sol; not a replacement for original ratings", "Two AI reviewers, not human labels; model outputs can reveal condition indirectly", "Adjudicator uses the solver model family", "Public issue and PR exposure may contaminate results", "No production, savings or independent acceptance claim" ] }, "posthocUnstartedAdjudication": { "kind": "posthoc-unstarted-disagreement-adjudication", "selection": "Only valid conflicting review pairs with no original adjudication file; excludes all failed, invalid, or completed original adjudications.", "model": "claude-haiku-4-5", "candidates": [ { "blind": "U01", "originalBlind": "B09", "slot": 6, "issue": "12649", "arm": "memex", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } }, { "blind": "U02", "originalBlind": "B06", "slot": 7, "issue": "20306", "arm": "keyword", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "U03", "originalBlind": "B07", "slot": 8, "issue": "20306", "arm": "memex", "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "U04", "originalBlind": "B08", "slot": 5, "issue": "12649", "arm": "keyword", "result": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "U05", "originalBlind": "B10", "slot": 4, "issue": "20307", "arm": "keyword", "result": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } } ], "usage": { "claude-haiku-4-5": { "started": 5, "completed": 5, "reportedPromptTokens": 8641, "reportedCompletionTokens": 1410, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 28832, "responseBytes": 9350 } }, "limitations": [ "Post-hoc model choice and selection after a timeout, not the original planned adjudication", "Distinct AI model name through the same proxy; not independent human review", "Short public issue histories and partial checks do not establish production value" ] }, "budgetedPosthocBatch15": { "kind": "posthoc-three-candidate-budgeted-blinded-rereview", "selection": "All three candidates from batch 15 with an original valid DeepSeek rating and an empty or truncated original Haiku rating; the unreviewable fourth attempt remains excluded.", "candidates": [ { "blind": "B01", "slot": 2, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null }, "issue": "16126", "arm": "keyword" }, { "blind": "B02", "slot": 1, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null }, "issue": "16126", "arm": "memex" }, { "blind": "B03", "slot": 3, "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } ], "registeredAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null }, "issue": "17487", "arm": "keyword" } ], "usage": { "deepseek-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 5370, "reportedCompletionTokens": 554, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 17488, "responseBytes": 4157 }, "qwen3.8-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 5338, "reportedCompletionTokens": 4438, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 17386, "responseBytes": 2494 }, "gemini-3.8-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 6158, "reportedCompletionTokens": 4831, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 20830, "responseBytes": 3761 } }, "limitations": [ "Post-hoc model replacement and candidate selection after observing empty reviews; original ratings remain unchanged.", "Two AI reviewers and a third AI adjudicator use the same proxy, not independent human assessment.", "Short public histories and partial tests do not establish production value.", "The increased reasoning-model budgets were selected after observing truncation and a separate synthetic probe; this is not the original registered assessment." ], "posthocReferenceNormalization": { "inputSha256": "2dfdf4541fdc2b013edf7bda358a3aeaa17f8254bf5497dbc27fb646be8a9c51", "scriptSha256": "b53c94d12497e2522960b76cda4e60034d5825558ab1cdd875b944af620b92af", "rows": { "B01#attempt-1": { "status": "POSTHOC_NORMALIZED", "correctness": 2, "sideEffects": 2, "changes": [ { "original": "candidateDiff", "normalized": "candidate" }, { "original": "acceptance", "normalized": "checks" }, { "original": "regressionControls", "normalized": "checks" } ] }, "B02#attempt-1": { "status": "POSTHOC_NORMALIZED", "correctness": 2, "sideEffects": 2, "changes": [ { "original": "candidateDiff", "normalized": "candidate" }, { "original": "acceptance", "normalized": "checks" }, { "original": "regressionControls", "normalized": "checks" } ] }, "B03#attempt-1": { "status": "POSTHOC_NORMALIZED", "correctness": 2, "sideEffects": 2, "changes": [ { "original": "candidateDiff", "normalized": "candidate" }, { "original": "acceptance", "normalized": "checks" }, { "original": "regressionControls", "normalized": "checks" } ] } } } }, "budgetedPosthocBatch16": { "kind": "posthoc-budgeted-dayjs-review-completion", "selection": "Re-review exactly the three empty or truncated original Qwen ratings; retain original valid DeepSeek and Qwen ratings. Adjudicate each valid score disagreement with a new Gemini call.", "candidates": [ { "blind": "R01", "slot": 6, "issue": "2956", "arm": "keyword", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "originalQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 0, "sideEffects": 0 }, "posthocQwen": { "status": "NOT_APPLICABLE" }, "posthocAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "R02", "slot": 1, "issue": "2511", "arm": "memex", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "originalQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "posthocQwen": { "status": "NOT_APPLICABLE" }, "posthocAdjudication": { "status": "MISSING" } }, { "blind": "R03", "slot": 4, "issue": "1923", "arm": "memex", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "originalQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "posthocQwen": { "status": "NOT_APPLICABLE" }, "posthocAdjudication": { "status": "MISSING" } }, { "blind": "R04", "slot": 12, "issue": "2479", "arm": "memex", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "originalQwen": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null }, "posthocQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "posthocAdjudication": { "status": "MISSING" } }, { "blind": "R05", "slot": 3, "issue": "1923", "arm": "keyword", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "originalQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "posthocQwen": { "status": "NOT_APPLICABLE" }, "posthocAdjudication": { "status": "MISSING" } }, { "blind": "R06", "slot": 8, "issue": "2951", "arm": "keyword", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "originalQwen": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null }, "posthocQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "posthocAdjudication": { "status": "MISSING" } }, { "blind": "R07", "slot": 11, "issue": "2479", "arm": "keyword", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, "originalQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "posthocQwen": { "status": "NOT_APPLICABLE" }, "posthocAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "R08", "slot": 5, "issue": "2956", "arm": "memex", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "originalQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, "posthocQwen": { "status": "NOT_APPLICABLE" }, "posthocAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "R09", "slot": 10, "issue": "1271", "arm": "memex", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "originalQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 }, "posthocQwen": { "status": "NOT_APPLICABLE" }, "posthocAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "R10", "slot": 9, "issue": "1271", "arm": "keyword", "originalDeepseek": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, "originalQwen": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null }, "posthocQwen": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 }, "posthocAdjudication": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } } ], "usage": { "qwen3.8-flash": { "started": 3, "completed": 3, "reportedPromptTokens": 4870, "reportedCompletionTokens": 3074, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 14697, "responseBytes": 3160 }, "gemini-3.8-flash": { "started": 5, "completed": 5, "reportedPromptTokens": 10581, "reportedCompletionTokens": 10220, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 33225, "responseBytes": 4551 } }, "limitations": [ "Selected after prior batches and some results were observed; not one pre-registered 30–50-task cohort.", "Six issues touch different functional areas, but common project and source exposure prevent an independence guarantee.", "Public issue discussions and PRs may disclose proposed solutions or appear in model training.", "Source references and issue-specific tests are partial and do not establish maintainer acceptance.", "Checks use Node source ESM hooks rather than the published compiled npm package.", "Retrieved contexts have different actual lengths and pre-seeded short issue discussions are not production sessions.", "Reviews and adjudications are AI through a common proxy; model provenance is unverified.", "Autonomous memory lifecycle, production effectiveness and cost savings are not established.", "Post-hoc selection and larger reasoning-model budgets were chosen after original truncations; original ratings and adjudications remain unchanged.", "Synthetic format probes do not guarantee valid real review responses." ] }, "fallbackAdjudicationsBatch16": { "kind": "posthoc-dayjs-alternate-adjudicator", "selection": "Five valid score disagreements after the original ratings and post-hoc Qwen replacements, each with an invalid post-hoc Gemini adjudication.", "model": "gpt-6-astra", "candidates": [ { "blind": "A01", "batch": null, "slot": 6, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "A02", "batch": null, "slot": 9, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "A03", "batch": null, "slot": 5, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "A04", "batch": null, "slot": 10, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "A05", "batch": null, "slot": 11, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } } ], "usage": { "gpt-6-astra": { "started": 5, "completed": 5, "reportedPromptTokens": 44366, "reportedCompletionTokens": 1603, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 33245, "responseBytes": 5651 } }, "limitations": [ "This alternate adjudicator and subset were chosen after observing failed adjudications.", "The adjudicator has the same model name as the solver, so it is not independent of the solver.", "All models share one proxy and provenance is unverified; AI format validity is not proof of factual correctness." ] }, "fallbackAdjudicationsBatches17to20": { "kind": "posthoc-alternate-adjudications-batches-17-20", "selection": "Only original valid-score disagreements in batches 17–20 with a completed but format-invalid Gemini adjudication. No missing or timed-out adjudication is retried.", "model": "gpt-6-astra", "candidates": [ { "blind": "F01", "batch": "17", "slot": 8, "result": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "F02", "batch": "18", "slot": 3, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "F03", "batch": "17", "slot": 4, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 } }, { "blind": "F04", "batch": "18", "slot": 5, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "F05", "batch": "20", "slot": 2, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "F06", "batch": "18", "slot": 4, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "F07", "batch": "19", "slot": 2, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } }, { "blind": "F08", "batch": "19", "slot": 3, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 2 } }, { "blind": "F09", "batch": "20", "slot": 4, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } }, { "blind": "F10", "batch": "17", "slot": 1, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 } }, { "blind": "F11", "batch": "19", "slot": 4, "result": { "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } }, { "blind": "F12", "batch": "19", "slot": 1, "result": { "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } } ], "usage": { "gpt-6-astra": { "started": 12, "completed": 12, "reportedPromptTokens": 113540, "reportedCompletionTokens": 2124, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 65078, "responseBytes": 12888 } }, "limitations": [ "The subset, output budget and alternate model were chosen after original failures.", "The adjudicator has the same model name as the solver and is not independent of it.", "All AI models use one proxy with unverified provenance; format validity does not prove factual support." ] }, "unstartedReviewsBatch17": { "kind": "posthoc-batch17-never-started-reviews", "selection": "Only B05 and B06, whose two original reviewer calls never started after the B04 timeout; no timed-out call is repeated.", "candidates": [ { "blind": "U01", "originalBlind": "B06", "slot": 3, "issue": "3614", "arm": "keyword", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 } ], "adjudication": { "model": "gpt-6-astra", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 2 } }, { "blind": "U02", "originalBlind": "B05", "slot": 5, "issue": "3600", "arm": "keyword", "ratings": [ { "model": "deepseek-flash", "status": "VALID", "eventStatus": "ok", "correctness": 1, "sideEffects": 1 }, { "model": "qwen3.8-flash", "status": "VALID", "eventStatus": "ok", "correctness": 2, "sideEffects": 1 } ], "adjudication": { "model": "gpt-6-astra", "status": "INVALID", "eventStatus": "ok", "correctness": null, "sideEffects": null } } ], "usage": { "deepseek-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 2589, "reportedCompletionTokens": 379, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 9377, "responseBytes": 2834 }, "qwen3.8-flash": { "started": 2, "completed": 2, "reportedPromptTokens": 2573, "reportedCompletionTokens": 411, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 9309, "responseBytes": 2582 }, "gpt-6-astra": { "started": 2, "completed": 2, "reportedPromptTokens": 8899, "reportedCompletionTokens": 490, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 13027, "responseBytes": 2521 } }, "failures": [], "limitations": [ "Post-hoc selection follows the observed timeout and prior results; original missing ratings remain missing.", "The Astra adjudicator has the solver model name and is not independent of it.", "Reviewers share a proxy of unverified provenance; format validity does not establish factual support." ] }, "diagnostics": { "reviewerRouteProbe": { "gpt-6.1-sol": { "started": 1, "completed": 1, "reportedPromptTokens": 4236, "reportedCompletionTokens": 9, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 215, "responseBytes": 336 } }, "alternateSolverToolProbe": { "gpt-6.1-sol": { "started": 1, "completed": 1, "reportedPromptTokens": 4436, "reportedCompletionTokens": 18, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 413, "responseBytes": 460 } }, "post503HealthProbe": { "gpt-6.1-sol": { "started": 1, "completed": 1, "reportedPromptTokens": 4272, "reportedCompletionTokens": 36, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 414, "responseBytes": 521 } }, "alternateSolverSyntheticSmoke": { "model": "gpt-6.1-sol", "status": "passed", "started": 5, "completed": 5, "unmatchedStarted": 0, "missingUsage": 0, "reportedPromptTokens": 23007, "reportedCompletionTokens": 149, "toolCalls": 4, "testsPassed": 1 }, "deepseekSyntheticSmoke": { "model": "deepseek-flash", "status": "passed", "started": 4, "completed": 4, "unmatchedStarted": 0, "missingUsage": 0, "reportedPromptTokens": 2822, "reportedCompletionTokens": 154, "toolCalls": 3, "testsPassed": 1 }, "gatewayTriage": { "deepseek-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 11, "reportedCompletionTokens": 1, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 175, "responseBytes": 0 }, "gpt-6.1-sol": { "started": 1, "completed": 1, "reportedPromptTokens": 0, "reportedCompletionTokens": 0, "missingUsage": 1, "failedResponses": 1, "unmatchedStarted": 0, "requestBytes": 138, "responseBytes": 0 }, "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 4229, "reportedCompletionTokens": 5, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 138, "responseBytes": 0 }, "claude-haiku-4-5": { "started": 1, "completed": 1, "reportedPromptTokens": 14, "reportedCompletionTokens": 4, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 143, "responseBytes": 0 } }, "gatewayTriageFailures": [ { "log": "calls.jsonl", "logSha256": "ac3bf64579a42f6529150e12bab30058bf87e8f413610d1652f9d6aa3a8df8af", "at": null, "label": null, "model": "gpt-6.1-sol", "status": "error", "httpStatus": null, "errorType": "TimeoutError", "errorCode": null, "elapsedSeconds": 35.884 } ], "qwenNoThinkingProbe": { "qwen3.8-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 1334, "reportedCompletionTokens": 148, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 5631, "responseBytes": 0 } }, "qwenReviewRouteProbe": { "qwen3.8-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 52, "reportedCompletionTokens": 48, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 326, "responseBytes": 520 } }, "geminiReviewRouteProbe": { "gemini-3.8-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 49, "reportedCompletionTokens": 266, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 329, "responseBytes": 810 } }, "reviewBudgetProbe": { "qwen3.8-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 1263, "reportedCompletionTokens": 213, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6246, "responseBytes": 1506 }, "gemini-3.8-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 1115, "reportedCompletionTokens": 231, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 6249, "responseBytes": 870 } }, "timeoutHealthProbe": { "deepseek-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 7, "reportedCompletionTokens": 2, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 153, "responseBytes": 444 }, "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 9, "reportedCompletionTokens": 5, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 116, "responseBytes": 320 }, "qwen3.8-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 385, "reportedCompletionTokens": 3, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 118, "responseBytes": 327 } }, "transportHealthProbe": { "deepseek-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 7, "reportedCompletionTokens": 2, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 153, "responseBytes": 444 }, "gpt-6-astra": { "started": 1, "completed": 1, "reportedPromptTokens": 4225, "reportedCompletionTokens": 5, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 116, "responseBytes": 372 }, "qwen3.8-flash": { "started": 1, "completed": 1, "reportedPromptTokens": 64, "reportedCompletionTokens": 25, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 118, "responseBytes": 477 } }, "v27JudgeRouteProbe": { "gpt-6.1-sol": { "started": 1, "completed": 1, "reportedPromptTokens": 4233, "reportedCompletionTokens": 9, "missingUsage": 0, "failedResponses": 0, "unmatchedStarted": 0, "requestBytes": 189, "responseBytes": 336 } } }, "limitations": [ "Different batches were selected or revised after prior results and are not one pre-registered cohort.", "The pre-seeded public issue discussions are not long production sessions.", "Tests are partial and reviewers are AI; label validity does not prove factual support.", "The numeric review direction was not specified in batches 1–5; their raw scores cannot be treated as a common scale.", "Unmatched started and failed requests may have unreported tokens or fees; only explicitly registered later batches retry explicit HTTP 503, not timeouts.", "Source exposure to public PRs or model pretraining cannot be excluded." ] }