{ "format": 1, "kind": "live-ab-codex-result", "generatedAt": "2026-10-29T15:27:12.492Z", "provider": "codex-cli", "model": "gpt-6.1-sol", "testedVersion": "0.14.1 + SQLite fix", "testedCommit": "9fb13bb80038eb79b239a96186ced2ccfd93eb31", "nativeReportSha256": "9849b3ba873a67e9336a7875cb96927f444770f1bb2375dd7fa5b52959a2bf2f", "bridgeExitCode": 0, "method": "codex exec --skip-git-repo-check --sandbox read-only --output-last-message; control/treatment generated by scripts/live-ab.mjs --emit-prompts", "aggregate": { "control": { "correct": 3, "n": 60, "accuracy": 0.05 }, "treatment": { "correct": 60, "n": 60, "accuracy": 1.0 }, "evictedControl": { "correct": 0, "n": 57, "accuracy": 0.0 }, "evictedTreatment": { "correct": 57, "n": 57, "accuracy": 1.0 } }, "seeds": [ { "seed": 1, "turns": 300, "tailCount": 20, "tailIds": [ "mem_e03250e2be", "mem_e7d7c42c52", "mem_16162f5bd6", "mem_3b2105ff96", "mem_a2dda0d3bb", "mem_13a0b19ba6", "mem_544e5f7f1c", "mem_8602bf2ea3", "mem_afaf78d740", "mem_84be8f458c", "mem_384a176aa4", "mem_1903c7cb18", "mem_0e7ee09249", "mem_cec00596f3", "mem_8461160893", "mem_d56ace8206", "mem_3208923a17", "mem_36d0318b4d", "mem_88e87d5b0c", "mem_6e56881022" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_eb1b7d0434", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_f64473f5cb", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_bb992c645e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_e3d42fea35", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_c9ff09288a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_f615810461", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_146c687336", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_3daeed6f0b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_6b4ef9275d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_1559042b33", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_027676dd06", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_70bb9fc86e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_468c33d098", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_1bd8f294e2", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_a79795a000", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_f868128c0f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_6d8b3807a2", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_73d1a66c63", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_a372f713f4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_13a0b19ba6", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_eb1b7d0434", "inTail": false, "evicted": true, "answer": "FACT-000-mkorf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_f64473f5cb", "inTail": false, "evicted": true, "answer": "FACT-015-03jmy", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_bb992c645e", "inTail": false, "evicted": true, "answer": "FACT-030-izkkh", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_e3d42fea35", "inTail": false, "evicted": true, "answer": "FACT-045-zbfww", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_c9ff09288a", "inTail": false, "evicted": true, "answer": "FACT-060-yv0n0", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_f615810461", "inTail": false, "evicted": true, "answer": "FACT-075-a4b5x", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_146c687336", "inTail": false, "evicted": true, "answer": "FACT-090-m28ly", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_3daeed6f0b", "inTail": false, "evicted": true, "answer": "FACT-105-py2zp", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_6b4ef9275d", "inTail": false, "evicted": true, "answer": "FACT-120-fbtzd", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_1559042b33", "inTail": false, "evicted": true, "answer": "FACT-135-ztagj", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_027676dd06", "inTail": false, "evicted": true, "answer": "FACT-150-ge14o", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_70bb9fc86e", "inTail": false, "evicted": true, "answer": "FACT-165-hlgwo", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_468c33d098", "inTail": false, "evicted": true, "answer": "FACT-180-50232", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_1bd8f294e2", "inTail": false, "evicted": true, "answer": "FACT-195-ej946", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_a79795a000", "inTail": false, "evicted": true, "answer": "FACT-210-8wsh3", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_f868128c0f", "inTail": false, "evicted": true, "answer": "FACT-225-5k815", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_6d8b3807a2", "inTail": false, "evicted": true, "answer": "FACT-240-hlxhw", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_73d1a66c63", "inTail": false, "evicted": true, "answer": "FACT-255-2fb7j", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_a372f713f4", "inTail": false, "evicted": true, "answer": "FACT-270-e7fcx", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_13a0b19ba6", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 2, "turns": 300, "tailCount": 20, "tailIds": [ "mem_0a5f3bfa47", "mem_10d1d2a3d7", "mem_b0ee469b1e", "mem_d3be79b73d", "mem_8aedbfdf7f", "mem_c78c91b522", "mem_2cbc5c596b", "mem_c186be31e9", "mem_e01959e0ae", "mem_e898913fc6", "mem_ca0ab08ca2", "mem_167cd2b7a2", "mem_3434fa3d3e", "mem_a7d27c648f", "mem_354e23a8e7", "mem_16328db12b", "mem_078b409194", "mem_052bc10a1e", "mem_5832a9c9df", "mem_fc407a4c36" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_3e0db0d146", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_e04cec5e54", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_c3d636fb29", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_620ca1ddbc", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_a988f6654d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_1d0fcc349a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_c28d284f5c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_1e4e19708d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_59d712d736", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_00d4fd7bad", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_ccc535b853", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_31b3ed9e25", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_00bf3f690d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_f45fe70a70", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_2280a986cd", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_b0556f9f26", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_2ba70f58a5", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_546dfcc434", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_a6e50aff94", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_c78c91b522", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_3e0db0d146", "inTail": false, "evicted": true, "answer": "FACT-000-qfl7m", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_e04cec5e54", "inTail": false, "evicted": true, "answer": "FACT-015-bp74k", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_c3d636fb29", "inTail": false, "evicted": true, "answer": "FACT-030-a9qrt", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_620ca1ddbc", "inTail": false, "evicted": true, "answer": "FACT-045-jd6u3", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_a988f6654d", "inTail": false, "evicted": true, "answer": "FACT-060-vidfn", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_1d0fcc349a", "inTail": false, "evicted": true, "answer": "FACT-075-mpk5r", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_c28d284f5c", "inTail": false, "evicted": true, "answer": "FACT-090-hyyqy", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_1e4e19708d", "inTail": false, "evicted": true, "answer": "FACT-105-cuzlu", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_59d712d736", "inTail": false, "evicted": true, "answer": "FACT-120-s4xt8", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_00d4fd7bad", "inTail": false, "evicted": true, "answer": "FACT-135-43s9i", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_ccc535b853", "inTail": false, "evicted": true, "answer": "FACT-150-saher", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_31b3ed9e25", "inTail": false, "evicted": true, "answer": "FACT-165-797as", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_00bf3f690d", "inTail": false, "evicted": true, "answer": "FACT-180-f9so7", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_f45fe70a70", "inTail": false, "evicted": true, "answer": "FACT-195-u25n2", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_2280a986cd", "inTail": false, "evicted": true, "answer": "FACT-210-kxe32", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_b0556f9f26", "inTail": false, "evicted": true, "answer": "FACT-225-52qt3", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_2ba70f58a5", "inTail": false, "evicted": true, "answer": "FACT-240-o77fl", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_546dfcc434", "inTail": false, "evicted": true, "answer": "FACT-255-d78oa", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_a6e50aff94", "inTail": false, "evicted": true, "answer": "FACT-270-yz7lh", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_c78c91b522", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 3, "turns": 300, "tailCount": 20, "tailIds": [ "mem_d738f0787e", "mem_de36f0aefd", "mem_ee06c1dd12", "mem_cfc44e699a", "mem_649117ef9a", "mem_819366c6bc", "mem_36a1b5d32a", "mem_b7b7bc865b", "mem_7b45c08fca", "mem_71fa3f97f4", "mem_980128e11f", "mem_f662119c13", "mem_749554ca1b", "mem_304229bdf9", "mem_60262018cc", "mem_03bcab98b4", "mem_219699574d", "mem_822d3be06d", "mem_a5640e8ffb", "mem_9d8fa58e78" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_94dcc59aa6", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_a15bc88d26", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_98227b7edc", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_a735112408", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_802feb32dd", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_c527f19bc0", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_87c3012863", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_e898465ae1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_aa5bd9f2c4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_e46d765e83", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_60856fb793", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_85c4b28c04", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_94e630540d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_93d9affc6f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_e052472de2", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_5dbd2d1ea1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_02211b0f7f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_3bef46793b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_a4aeb1f198", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_819366c6bc", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_94dcc59aa6", "inTail": false, "evicted": true, "answer": "FACT-000-pxewf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_a15bc88d26", "inTail": false, "evicted": true, "answer": "FACT-015-1e3tl", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_98227b7edc", "inTail": false, "evicted": true, "answer": "FACT-030-gf83p", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_a735112408", "inTail": false, "evicted": true, "answer": "FACT-045-2p3ua", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_802feb32dd", "inTail": false, "evicted": true, "answer": "FACT-060-rgus2", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_c527f19bc0", "inTail": false, "evicted": true, "answer": "FACT-075-hblyb", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_87c3012863", "inTail": false, "evicted": true, "answer": "FACT-090-hisus", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_e898465ae1", "inTail": false, "evicted": true, "answer": "FACT-105-5toyt", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_aa5bd9f2c4", "inTail": false, "evicted": true, "answer": "FACT-120-9h6hf", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_e46d765e83", "inTail": false, "evicted": true, "answer": "FACT-135-jw6fx", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_60856fb793", "inTail": false, "evicted": true, "answer": "FACT-150-70m0g", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_85c4b28c04", "inTail": false, "evicted": true, "answer": "FACT-165-30c3x", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_94e630540d", "inTail": false, "evicted": true, "answer": "FACT-180-pp2gq", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_93d9affc6f", "inTail": false, "evicted": true, "answer": "FACT-195-h5u81", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_e052472de2", "inTail": false, "evicted": true, "answer": "FACT-210-skod2", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_5dbd2d1ea1", "inTail": false, "evicted": true, "answer": "FACT-225-vo6pc", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_02211b0f7f", "inTail": false, "evicted": true, "answer": "FACT-240-bnezh", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_3bef46793b", "inTail": false, "evicted": true, "answer": "FACT-255-ouzrk", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_a4aeb1f198", "inTail": false, "evicted": true, "answer": "FACT-270-abmgy", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_819366c6bc", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 20, "total": 20 } } ], "calls": [ { "seed": 1, "label": "control", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 23367, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 185, "reasoning_output_tokens": 74, "total_tokens": 23552 }, "reasoningEffort": "max", "configurationUpdates": [], "sessionLogSha256": "d3ec2168dd631ac2297ab9f9f5179635b070ce0accaea47d2ec8ab3b5b3b1f52", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-rm66w", "stderrTailSha256": "80410579551c74104a0274941240e165578800d27a94e49721c6a0c3c1cbff83", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 1, "label": "treatment", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 29630, "cached_input_tokens": 0, "cache_write_input_tokens": 0, "output_tokens": 455, "reasoning_output_tokens": 235, "total_tokens": 30085 }, "reasoningEffort": "max", "configurationUpdates": [], "sessionLogSha256": "4bfa9ea2d633b99533c74875725a469b26d6b31b536cc3955ad89637a54fc168", "answerText": "Q0: FACT-000-mkorf\nQ1: FACT-015-03jmy\nQ2: FACT-030-izkkh\nQ3: FACT-045-zbfww\nQ4: FACT-060-yv0n0\nQ5: FACT-075-a4b5x\nQ6: FACT-090-m28ly\nQ7: FACT-105-py2zp\nQ8: FACT-120-fbtzd\nQ9: FACT-135-ztagj\nQ10: FACT-150-ge14o\nQ11: FACT-165-hlgwo\nQ12: FACT-180-50232\nQ13: FACT-195-ej946\nQ14: FACT-210-8wsh3\nQ15: FACT-225-5k815\nQ16: FACT-240-hlxhw\nQ17: FACT-255-2fb7j\nQ18: FACT-270-e7fcx\nQ19: FACT-285-rm66w", "stderrTailSha256": "991a9dfc1d25a2fd505d5da531ccc64ff085a993da5464c486905ad3e30d672f", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 2, "label": "control", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 29064, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 172, "reasoning_output_tokens": 61, "total_tokens": 29236 }, "reasoningEffort": "max", "configurationUpdates": [ "max" ], "sessionLogSha256": "52f845834f1f9a3134c7cadbc2d24c36a4134a648742b2397605a5cd51ff903a", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-1esyu", "stderrTailSha256": "df2090b4a38092386285df9e4ad7412f663b2afbb98df261256686e2183bab2d", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 2, "label": "treatment", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 29615, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 464, "reasoning_output_tokens": 247, "total_tokens": 30079 }, "reasoningEffort": "max", "configurationUpdates": [], "sessionLogSha256": "3177bfdc4d94435e00cbe2d5c6a97dcb4736d9c1c17a3722d75af121c498cc8b", "answerText": "Q0: FACT-000-qfl7m\nQ1: FACT-015-bp74k\nQ2: FACT-030-a9qrt\nQ3: FACT-045-jd6u3\nQ4: FACT-060-vidfn\nQ5: FACT-075-mpk5r\nQ6: FACT-090-hyyqy\nQ7: FACT-105-cuzlu\nQ8: FACT-120-s4xt8\nQ9: FACT-135-43s9i\nQ10: FACT-150-saher\nQ11: FACT-165-797as\nQ12: FACT-180-f9so7\nQ13: FACT-195-u25n2\nQ14: FACT-210-kxe32\nQ15: FACT-225-52qt3\nQ16: FACT-240-o77fl\nQ17: FACT-255-d78oa\nQ18: FACT-270-yz7lh\nQ19: FACT-285-1esyu", "stderrTailSha256": "dcc19455e613c2b6c3008349434eb4fbde19f6eaa2b8e3772c62e6b88e34cf22", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 3, "label": "control", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 23366, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 161, "reasoning_output_tokens": 51, "total_tokens": 23527 }, "reasoningEffort": "max", "configurationUpdates": [], "sessionLogSha256": "f5d5f92dd198205a563003248eeac1b3817b352f7af7624453608daa65ff1faf", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-o73dt", "stderrTailSha256": "cb38eac0de42c575758affb9a32875bd4938892a68566fd31532d1fd711f4b97", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 3, "label": "treatment", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 29646, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 445, "reasoning_output_tokens": 222, "total_tokens": 30091 }, "reasoningEffort": "max", "configurationUpdates": [], "sessionLogSha256": "a0665d8913ecfd4d13d11754f92f8a984107525b86d1448b8437e78ebedcbd32", "answerText": "Q0: FACT-000-pxewf\nQ1: FACT-015-1e3tl\nQ2: FACT-030-gf83p\nQ3: FACT-045-2p3ua\nQ4: FACT-060-rgus2\nQ5: FACT-075-hblyb\nQ6: FACT-090-hisus\nQ7: FACT-105-5toyt\nQ8: FACT-120-9h6hf\nQ9: FACT-135-jw6fx\nQ10: FACT-150-70m0g\nQ11: FACT-165-30c3x\nQ12: FACT-180-pp2gq\nQ13: FACT-195-h5u81\nQ14: FACT-210-skod2\nQ15: FACT-225-vo6pc\nQ16: FACT-240-bnezh\nQ17: FACT-255-ouzrk\nQ18: FACT-270-abmgy\nQ19: FACT-285-o73dt", "stderrTailSha256": "e07e96d51f08d6a7ff09426f14bd049e3d282592d6c37bbdcac88a79b5ffbaf3", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false } ], "usage": { "source": "Codex local session token_count total_token_usage; totals include reported cached input", "calls": 6, "totals": { "input_tokens": 164688, "cached_input_tokens": 65920, "cache_write_input_tokens": 0, "output_tokens": 1882, "reasoning_output_tokens": 890, "total_tokens": 166570 }, "billedCost": "UNKNOWN", "billingReceipt": "NOT_CAPTURED" }, "caveats": [ "Synthetic 300-turn fixtures; control sees tail titles and summaries, not full bodies.", "Treatment receives preselected search/open evidence; this is not a model-driven memory-tool benchmark.", "No model-issued tool calls appear in these six local session logs.", "Only diagnostic tails were captured by the bridge; they are redacted publicly and retained locally.", "The native report and local session logs remain local; billed cost is UNKNOWN." ] }