{ "format": 1, "kind": "live-ab-codex-result", "generatedAt": "2026-10-30T09:27:52.899Z", "provider": "codex-cli", "model": "gpt-6.1-sol", "testedVersion": "0.26.0", "testedCommit": "0b39791d144c8ebdf7a4f3ddc283550438b4c8d4", "nativeReportSha256": "27058af6a873c9aeaced3622af1ddf33391b389c5cd767ce28096f89862d9b99", "bridgeExitCode": 0, "method": "codex exec --skip-git-repo-check --sandbox read-only --output-last-message; control/treatment generated by scripts/live-ab.mjs --emit-prompts", "aggregate": { "control": { "correct": 3, "n": 60, "accuracy": 0.05 }, "treatment": { "correct": 60, "n": 60, "accuracy": 1.0 }, "evictedControl": { "correct": 0, "n": 57, "accuracy": 0.0 }, "evictedTreatment": { "correct": 57, "n": 57, "accuracy": 1.0 } }, "seeds": [ { "seed": 1, "turns": 300, "tailCount": 20, "tailIds": [ "mem_2889053973", "mem_2bd2cc6123", "mem_7206a0d7d8", "mem_5a13c14fec", "mem_468ecc98b0", "mem_678d3c6586", "mem_8235644ea9", "mem_7629589423", "mem_17cdc0db73", "mem_bc16eb5f73", "mem_621fbd3cf8", "mem_594762f056", "mem_c6d9ffc38a", "mem_1411daff64", "mem_99079723ec", "mem_7b5c10dd05", "mem_7188060113", "mem_f471a226f8", "mem_88272a4cec", "mem_d7cff47c76" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_c0f40725c7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_7a54a79c99", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_476974b102", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_465ad58ae7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_1ad93d7c5f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_325e770d3c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_4b43596355", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_c9fd4da76c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_a56de2cd1f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_47cb27acc4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_051359a6d5", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_30a7c07863", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_3854c96808", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_5e58bbc0d0", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_bc800ffbbf", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_59a5af4261", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_79be21c2a7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_0aea3e2309", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_4bdd485b8d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_678d3c6586", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_c0f40725c7", "inTail": false, "evicted": true, "answer": "FACT-000-mkorf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_7a54a79c99", "inTail": false, "evicted": true, "answer": "FACT-015-03jmy", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_476974b102", "inTail": false, "evicted": true, "answer": "FACT-030-izkkh", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_465ad58ae7", "inTail": false, "evicted": true, "answer": "FACT-045-zbfww", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_1ad93d7c5f", "inTail": false, "evicted": true, "answer": "FACT-060-yv0n0", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_325e770d3c", "inTail": false, "evicted": true, "answer": "FACT-075-a4b5x", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_4b43596355", "inTail": false, "evicted": true, "answer": "FACT-090-m28ly", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_c9fd4da76c", "inTail": false, "evicted": true, "answer": "FACT-105-py2zp", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_a56de2cd1f", "inTail": false, "evicted": true, "answer": "FACT-120-fbtzd", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_47cb27acc4", "inTail": false, "evicted": true, "answer": "FACT-135-ztagj", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_051359a6d5", "inTail": false, "evicted": true, "answer": "FACT-150-ge14o", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_30a7c07863", "inTail": false, "evicted": true, "answer": "FACT-165-hlgwo", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_3854c96808", "inTail": false, "evicted": true, "answer": "FACT-180-50232", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_5e58bbc0d0", "inTail": false, "evicted": true, "answer": "FACT-195-ej946", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_bc800ffbbf", "inTail": false, "evicted": true, "answer": "FACT-210-8wsh3", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_59a5af4261", "inTail": false, "evicted": true, "answer": "FACT-225-5k815", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_79be21c2a7", "inTail": false, "evicted": true, "answer": "FACT-240-hlxhw", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_0aea3e2309", "inTail": false, "evicted": true, "answer": "FACT-255-2fb7j", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_4bdd485b8d", "inTail": false, "evicted": true, "answer": "FACT-270-e7fcx", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_678d3c6586", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 2, "turns": 300, "tailCount": 20, "tailIds": [ "mem_287254c8ca", "mem_f0cfc8f30f", "mem_722176622e", "mem_4bf208d23b", "mem_303d894677", "mem_339682f3c2", "mem_2afad1088c", "mem_f2f5398673", "mem_1b240cb688", "mem_8d8edb51e7", "mem_d0724d2ada", "mem_70fd378d27", "mem_9d16ae194d", "mem_8baea5e456", "mem_694799b2d3", "mem_ca9fce58ce", "mem_5fd348c617", "mem_dabca6a174", "mem_3b23c939a4", "mem_e75dc5ee0d" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_d5be62b902", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_4c1f77e6b3", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_c34fb4ac73", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_183eef60b1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_114a8e7110", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_320673239f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_cf8182598f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_a292d4785a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_42ebcb7a94", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_c93e421208", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_98cbaaf0db", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_164c4952d1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_d6394b9c21", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_2d2af45526", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_6916621de0", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_5506865157", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_c2ef39cd2b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_de2cd64df8", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_86e37779d4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_339682f3c2", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_d5be62b902", "inTail": false, "evicted": true, "answer": "FACT-000-qfl7m", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_4c1f77e6b3", "inTail": false, "evicted": true, "answer": "FACT-015-bp74k", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_c34fb4ac73", "inTail": false, "evicted": true, "answer": "FACT-030-a9qrt", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_183eef60b1", "inTail": false, "evicted": true, "answer": "FACT-045-jd6u3", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_114a8e7110", "inTail": false, "evicted": true, "answer": "FACT-060-vidfn", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_320673239f", "inTail": false, "evicted": true, "answer": "FACT-075-mpk5r", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_cf8182598f", "inTail": false, "evicted": true, "answer": "FACT-090-hyyqy", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_a292d4785a", "inTail": false, "evicted": true, "answer": "FACT-105-cuzlu", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_42ebcb7a94", "inTail": false, "evicted": true, "answer": "FACT-120-s4xt8", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_c93e421208", "inTail": false, "evicted": true, "answer": "FACT-135-43s9i", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_98cbaaf0db", "inTail": false, "evicted": true, "answer": "FACT-150-saher", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_164c4952d1", "inTail": false, "evicted": true, "answer": "FACT-165-797as", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_d6394b9c21", "inTail": false, "evicted": true, "answer": "FACT-180-f9so7", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_2d2af45526", "inTail": false, "evicted": true, "answer": "FACT-195-u25n2", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_6916621de0", "inTail": false, "evicted": true, "answer": "FACT-210-kxe32", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_5506865157", "inTail": false, "evicted": true, "answer": "FACT-225-52qt3", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_c2ef39cd2b", "inTail": false, "evicted": true, "answer": "FACT-240-o77fl", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_de2cd64df8", "inTail": false, "evicted": true, "answer": "FACT-255-d78oa", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_86e37779d4", "inTail": false, "evicted": true, "answer": "FACT-270-yz7lh", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_339682f3c2", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 3, "turns": 300, "tailCount": 20, "tailIds": [ "mem_c8100fc942", "mem_2705d92f60", "mem_b9a037c0ce", "mem_a61001c105", "mem_736eed1bac", "mem_35d33efa4c", "mem_9062370821", "mem_fb7b98531b", "mem_b159b0cadc", "mem_4da8fd8c63", "mem_4491a36888", "mem_d70e52e103", "mem_d1a9fe3d09", "mem_7ca5371606", "mem_febd544e09", "mem_03020eac26", "mem_80a212e5aa", "mem_f905ef3fcb", "mem_97622cdf8e", "mem_2668aebb82" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_0edd99d733", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_7f053d2444", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_4abe58da09", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_e08e839cd1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_46e740bee6", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_f592c6dd16", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_d24fae3434", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_feaa6d5ae0", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_3777ab3e01", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_8354ee7b00", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_96d711f101", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_5c5e34585b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_737f2fb3a8", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_9416e39b30", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_94aeccf198", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_3a4d3df02f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_5f506a771c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_57cf21fe30", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_35b31dfc5c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_35d33efa4c", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_0edd99d733", "inTail": false, "evicted": true, "answer": "FACT-000-pxewf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_7f053d2444", "inTail": false, "evicted": true, "answer": "FACT-015-1e3tl", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_4abe58da09", "inTail": false, "evicted": true, "answer": "FACT-030-gf83p", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_e08e839cd1", "inTail": false, "evicted": true, "answer": "FACT-045-2p3ua", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_46e740bee6", "inTail": false, "evicted": true, "answer": "FACT-060-rgus2", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_f592c6dd16", "inTail": false, "evicted": true, "answer": "FACT-075-hblyb", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_d24fae3434", "inTail": false, "evicted": true, "answer": "FACT-090-hisus", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_feaa6d5ae0", "inTail": false, "evicted": true, "answer": "FACT-105-5toyt", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_3777ab3e01", "inTail": false, "evicted": true, "answer": "FACT-120-9h6hf", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_8354ee7b00", "inTail": false, "evicted": true, "answer": "FACT-135-jw6fx", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_96d711f101", "inTail": false, "evicted": true, "answer": "FACT-150-70m0g", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_5c5e34585b", "inTail": false, "evicted": true, "answer": "FACT-165-30c3x", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_737f2fb3a8", "inTail": false, "evicted": true, "answer": "FACT-180-pp2gq", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_9416e39b30", "inTail": false, "evicted": true, "answer": "FACT-195-h5u81", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_94aeccf198", "inTail": false, "evicted": true, "answer": "FACT-210-skod2", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_3a4d3df02f", "inTail": false, "evicted": true, "answer": "FACT-225-vo6pc", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_5f506a771c", "inTail": false, "evicted": true, "answer": "FACT-240-bnezh", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_57cf21fe30", "inTail": false, "evicted": true, "answer": "FACT-255-ouzrk", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_35b31dfc5c", "inTail": false, "evicted": true, "answer": "FACT-270-abmgy", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_35d33efa4c", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 20, "total": 20 } } ], "calls": [ { "seed": 1, "label": "control", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 23286, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 154, "reasoning_output_tokens": 43, "total_tokens": 23440 }, "reasoningEffort": "xhigh", "configurationUpdates": [], "sessionLogSha256": "fedaa1e4f218d6c5d225f858ae055ee5aa73a87c11654a4d7bbee23baedd2f6e", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-rm66w", "stderrTailSha256": "336e448a06fc86bd068b5261bef032d30e74812a2e50bfdb8fed85e33c1c266d", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 1, "label": "treatment", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 29571, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 364, "reasoning_output_tokens": 144, "total_tokens": 29935 }, "reasoningEffort": "xhigh", "configurationUpdates": [], "sessionLogSha256": "9c67c95cc7f0c2738527024ef9a9c613f1a993c38053827f74dbf8e21121564e", "answerText": "Q0: FACT-000-mkorf\nQ1: FACT-015-03jmy\nQ2: FACT-030-izkkh\nQ3: FACT-045-zbfww\nQ4: FACT-060-yv0n0\nQ5: FACT-075-a4b5x\nQ6: FACT-090-m28ly\nQ7: FACT-105-py2zp\nQ8: FACT-120-fbtzd\nQ9: FACT-135-ztagj\nQ10: FACT-150-ge14o\nQ11: FACT-165-hlgwo\nQ12: FACT-180-50232\nQ13: FACT-195-ej946\nQ14: FACT-210-8wsh3\nQ15: FACT-225-5k815\nQ16: FACT-240-hlxhw\nQ17: FACT-255-2fb7j\nQ18: FACT-270-e7fcx\nQ19: FACT-285-rm66w", "stderrTailSha256": "453fa643d4611c72ba032d1cf00f089ff0d2ee859eae4c0c001e1825824213d6", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 2, "label": "control", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 23286, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 150, "reasoning_output_tokens": 39, "total_tokens": 23436 }, "reasoningEffort": "xhigh", "configurationUpdates": [], "sessionLogSha256": "5c3f659171dc4417cb06476f42bd8972a87af76fb71d66206b63feb35a82ed2d", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-1esyu", "stderrTailSha256": "edbc2748239e82f9e46ae8a59bf6849deb916b69992bfc2f1ef1f8c01ac23956", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 2, "label": "treatment", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 29521, "cached_input_tokens": 21120, "cache_write_input_tokens": 0, "output_tokens": 355, "reasoning_output_tokens": 138, "total_tokens": 29876 }, "reasoningEffort": "xhigh", "configurationUpdates": [], "sessionLogSha256": "86e7848daf64abf3ef2bb03cf0bab41ce1999afa3b5453f51531a64762330ffd", "answerText": "Q0: FACT-000-qfl7m\nQ1: FACT-015-bp74k\nQ2: FACT-030-a9qrt\nQ3: FACT-045-jd6u3\nQ4: FACT-060-vidfn\nQ5: FACT-075-mpk5r\nQ6: FACT-090-hyyqy\nQ7: FACT-105-cuzlu\nQ8: FACT-120-s4xt8\nQ9: FACT-135-43s9i\nQ10: FACT-150-saher\nQ11: FACT-165-797as\nQ12: FACT-180-f9so7\nQ13: FACT-195-u25n2\nQ14: FACT-210-kxe32\nQ15: FACT-225-52qt3\nQ16: FACT-240-o77fl\nQ17: FACT-255-d78oa\nQ18: FACT-270-yz7lh\nQ19: FACT-285-1esyu", "stderrTailSha256": "268c9b2a6228779eda4b035dc89997e79fc563933b260f064ec76f0425863f8a", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 3, "label": "control", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 28982, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 146, "reasoning_output_tokens": 36, "total_tokens": 29128 }, "reasoningEffort": "xhigh", "configurationUpdates": [], "sessionLogSha256": "7945182275d8ec1a087d1694724ee3e609f7e64aae35bc3934727549b0067a7d", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-o73dt", "stderrTailSha256": "c8cb5c007390e1d789d610168c0b8e208a67a9b34fca01cdec6dce298e1cff4f", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false }, { "seed": 3, "label": "treatment", "processExitCode": 0, "sessionTaskComplete": true, "recordedToolCalls": 0, "usage": { "input_tokens": 29607, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 371, "reasoning_output_tokens": 148, "total_tokens": 29978 }, "reasoningEffort": "xhigh", "configurationUpdates": [], "sessionLogSha256": "fe070760dc6489ae09af4c1b9d353fb4e10525d9d8545e93ebea0bd18c32a9a1", "answerText": "Q0: FACT-000-pxewf\nQ1: FACT-015-1e3tl\nQ2: FACT-030-gf83p\nQ3: FACT-045-2p3ua\nQ4: FACT-060-rgus2\nQ5: FACT-075-hblyb\nQ6: FACT-090-hisus\nQ7: FACT-105-5toyt\nQ8: FACT-120-9h6hf\nQ9: FACT-135-jw6fx\nQ10: FACT-150-70m0g\nQ11: FACT-165-30c3x\nQ12: FACT-180-pp2gq\nQ13: FACT-195-h5u81\nQ14: FACT-210-skod2\nQ15: FACT-225-vo6pc\nQ16: FACT-240-bnezh\nQ17: FACT-255-ouzrk\nQ18: FACT-270-abmgy\nQ19: FACT-285-o73dt", "stderrTailSha256": "b840b262d69424e474d700eef81486c7eae6fda3dcbf99d523e1623662337c7b", "stderrTailPublished": "redacted; bridge captured only the final 1200 characters", "initializeWarningSeen": true, "analyticsWarningSeen": false } ], "usage": { "source": "Codex local session token_count total_token_usage; totals include reported cached input", "calls": 6, "totals": { "input_tokens": 164253, "cached_input_tokens": 87040, "cache_write_input_tokens": 0, "output_tokens": 1540, "reasoning_output_tokens": 548, "total_tokens": 165793 }, "billedCost": "UNKNOWN", "billingReceipt": "NOT_CAPTURED" }, "caveats": [ "Synthetic 300-turn fixtures; control sees tail titles and summaries, not full bodies.", "Treatment receives preselected search/open evidence; this is not a model-driven memory-tool benchmark.", "No model-issued tool calls appear in these six local session logs.", "Only diagnostic tails were captured by the bridge; they are redacted publicly and retained locally.", "The native report and local session logs remain local; billed cost is UNKNOWN." ] }