{ "format": 1, "kind": "live-ab-codex-reconstructed-after-fixture-failure", "generatedAt": "2026-10-29T12:57:31.983726+00:00", "provider": "codex-cli", "model": "gpt-6.1-sol", "testedVersion": "0.9.1 + tail fix", "testedCommit": "72ba8fc338e79e46121d44c8757594c773ed3b6b", "method": "The original bridge produced seed 1 and 2 answers, then exited before any seed 3 model call because fixture generation hit SQLite ERR_SQLITE_ERROR 5. Seed 3 was generated in a separate temporary root and only its two missing calls were run with the same Codex exec arguments. All six completions and outputs were checked against the local Codex session logs. Grades were reconstructed from actual plans and answers.", "initialBridgeFailure": { "exitCode": 1, "seed": 3, "stage": "fixture generation, archiveCold", "errorCode": "ERR_SQLITE_ERROR", "sqliteCode": 5, "message": "database is locked", "cause": "UNKNOWN", "modelCallsForSeed3BeforeFailure": 0, "failureTextSha256": "9ad0550e028b0f3cd590f561d0374b829cf6cd6e9d87726734ba18957b860da1", "failureTextNote": "The bridge printed the last 500 bytes of fixture stderr; the local copy and failed temporary database are preserved." }, "aggregate": { "control": { "correct": 3, "n": 60, "accuracy": 0.05 }, "treatment": { "correct": 60, "n": 60, "accuracy": 1.0 }, "evictedControl": { "correct": 0, "n": 57, "accuracy": 0.0 }, "evictedTreatment": { "correct": 57, "n": 57, "accuracy": 1.0 } }, "usage": { "source": "Codex local session token_count total_token_usage; totals include reported cached input", "calls": 6, "totals": { "input_tokens": 157473, "cached_input_tokens": 79104, "cache_write_input_tokens": 0, "output_tokens": 1838, "reasoning_output_tokens": 846, "total_tokens": 159311 }, "billedCost": "UNKNOWN", "billingReceipt": "NOT_CAPTURED" }, "seeds": [ { "seed": 1, "turns": 300, "tailCount": 20, "tailIds": [ "mem_fc9f4c3b46", "mem_69bd2b2c59", "mem_69d312b681", "mem_528f14cd4d", "mem_8d753b2d6f", "mem_ccc1fdff08", "mem_6f75a0f222", "mem_5720be0735", "mem_c4266807c5", "mem_42849b2a81", "mem_c7c2dc4456", "mem_862df8dbb4", "mem_c1eb6d5126", "mem_130edcbbd7", "mem_5a2c61178f", "mem_e5a2f13d11", "mem_c2c92f6044", "mem_5512cc12d0", "mem_ea0fe979fd", "mem_5d80bd930a" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_f132e96c1d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_3a8538bb48", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_752caf00ed", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_c4f33fe83a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_bdd50d94af", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_e94301b37b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_77531e8702", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_db7e8db342", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_d5ee3332fe", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_b74bc1d7fa", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_0dfcec91d6", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_5e226ce11b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_426949f2e1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_c712cf68ef", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_ff89c10ab7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_1cc8db9f87", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_bd9f4a9443", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_971cb5867f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_b249d3d208", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_ccc1fdff08", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_f132e96c1d", "inTail": false, "evicted": true, "answer": "FACT-000-mkorf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_3a8538bb48", "inTail": false, "evicted": true, "answer": "FACT-015-03jmy", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_752caf00ed", "inTail": false, "evicted": true, "answer": "FACT-030-izkkh", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_c4f33fe83a", "inTail": false, "evicted": true, "answer": "FACT-045-zbfww", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_bdd50d94af", "inTail": false, "evicted": true, "answer": "FACT-060-yv0n0", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_e94301b37b", "inTail": false, "evicted": true, "answer": "FACT-075-a4b5x", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_77531e8702", "inTail": false, "evicted": true, "answer": "FACT-090-m28ly", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_db7e8db342", "inTail": false, "evicted": true, "answer": "FACT-105-py2zp", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_d5ee3332fe", "inTail": false, "evicted": true, "answer": "FACT-120-fbtzd", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_b74bc1d7fa", "inTail": false, "evicted": true, "answer": "FACT-135-ztagj", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_0dfcec91d6", "inTail": false, "evicted": true, "answer": "FACT-150-ge14o", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_5e226ce11b", "inTail": false, "evicted": true, "answer": "FACT-165-hlgwo", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_426949f2e1", "inTail": false, "evicted": true, "answer": "FACT-180-50232", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_c712cf68ef", "inTail": false, "evicted": true, "answer": "FACT-195-ej946", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_ff89c10ab7", "inTail": false, "evicted": true, "answer": "FACT-210-8wsh3", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_1cc8db9f87", "inTail": false, "evicted": true, "answer": "FACT-225-5k815", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_bd9f4a9443", "inTail": false, "evicted": true, "answer": "FACT-240-hlxhw", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_971cb5867f", "inTail": false, "evicted": true, "answer": "FACT-255-2fb7j", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_b249d3d208", "inTail": false, "evicted": true, "answer": "FACT-270-e7fcx", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_ccc1fdff08", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 2, "turns": 300, "tailCount": 20, "tailIds": [ "mem_32a33cec81", "mem_9451b76e08", "mem_177fa8eea8", "mem_430e8b0568", "mem_aff5fbb17f", "mem_42d67439ae", "mem_f87a6e485d", "mem_136cfa3aa8", "mem_57b8d3567e", "mem_af5c250470", "mem_5c5075b9eb", "mem_6a2cca57c9", "mem_310e7f26e7", "mem_60214aedd3", "mem_3171b4dcf3", "mem_fc7f883c48", "mem_6f5ff723e1", "mem_faaec12f97", "mem_4057ae1a30", "mem_00134dfcca" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_4f892fffd5", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_d50215902f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_7af51b5210", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_7480d5ab92", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_17926893e1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_f33c2adfb0", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_f58f9f1aed", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_e689fcbbce", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_d6f43f9c12", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_cbc496b731", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_c3a3b1f7ce", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_dbe9a4538e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_cf42f4bbf6", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_fdb009738a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_e5b59c3dbc", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_0a702105fd", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_9a3725a7f3", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_54f09e59a1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_dc49ac2953", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_42d67439ae", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_4f892fffd5", "inTail": false, "evicted": true, "answer": "FACT-000-qfl7m", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_d50215902f", "inTail": false, "evicted": true, "answer": "FACT-015-bp74k", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_7af51b5210", "inTail": false, "evicted": true, "answer": "FACT-030-a9qrt", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_7480d5ab92", "inTail": false, "evicted": true, "answer": "FACT-045-jd6u3", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_17926893e1", "inTail": false, "evicted": true, "answer": "FACT-060-vidfn", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_f33c2adfb0", "inTail": false, "evicted": true, "answer": "FACT-075-mpk5r", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_f58f9f1aed", "inTail": false, "evicted": true, "answer": "FACT-090-hyyqy", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_e689fcbbce", "inTail": false, "evicted": true, "answer": "FACT-105-cuzlu", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_d6f43f9c12", "inTail": false, "evicted": true, "answer": "FACT-120-s4xt8", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_cbc496b731", "inTail": false, "evicted": true, "answer": "FACT-135-43s9i", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_c3a3b1f7ce", "inTail": false, "evicted": true, "answer": "FACT-150-saher", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_dbe9a4538e", "inTail": false, "evicted": true, "answer": "FACT-165-797as", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_cf42f4bbf6", "inTail": false, "evicted": true, "answer": "FACT-180-f9so7", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_fdb009738a", "inTail": false, "evicted": true, "answer": "FACT-195-u25n2", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_e5b59c3dbc", "inTail": false, "evicted": true, "answer": "FACT-210-kxe32", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_0a702105fd", "inTail": false, "evicted": true, "answer": "FACT-225-52qt3", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_9a3725a7f3", "inTail": false, "evicted": true, "answer": "FACT-240-o77fl", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_54f09e59a1", "inTail": false, "evicted": true, "answer": "FACT-255-d78oa", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_dc49ac2953", "inTail": false, "evicted": true, "answer": "FACT-270-yz7lh", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_42d67439ae", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 3, "turns": 300, "tailCount": 20, "tailIds": [ "mem_5d22bf8b5f", "mem_0f372e9c85", "mem_e9e14b7dc8", "mem_133068f2e3", "mem_20ed04a331", "mem_f9021bef4b", "mem_02112ee1ab", "mem_73c4e72d88", "mem_8a5df378a0", "mem_5cb27a332c", "mem_edb9896093", "mem_8e76742978", "mem_81d769a290", "mem_979a2eb08e", "mem_19c9c8baa6", "mem_109ef26c11", "mem_1b78183f1b", "mem_b304a16d24", "mem_6346698f02", "mem_6ebc03951e" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_b3e7c6c931", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_a60f53147d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_3e2cc6b270", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_af20042c23", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_3005fbf071", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_b4fffc489a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_8eecbbeb78", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_66423ec91f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_7fa076a056", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_a72818ecec", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_e379e4b23b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_766a58bd2a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_5d362e5446", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_11639fa080", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_9e02dc8d1d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_23892ad26c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_572057a79c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_d6b45e15be", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_8692a8cf8a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_f9021bef4b", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_b3e7c6c931", "inTail": false, "evicted": true, "answer": "FACT-000-pxewf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_a60f53147d", "inTail": false, "evicted": true, "answer": "FACT-015-1e3tl", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_3e2cc6b270", "inTail": false, "evicted": true, "answer": "FACT-030-gf83p", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_af20042c23", "inTail": false, "evicted": true, "answer": "FACT-045-2p3ua", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_3005fbf071", "inTail": false, "evicted": true, "answer": "FACT-060-rgus2", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_b4fffc489a", "inTail": false, "evicted": true, "answer": "FACT-075-hblyb", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_8eecbbeb78", "inTail": false, "evicted": true, "answer": "FACT-090-hisus", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_66423ec91f", "inTail": false, "evicted": true, "answer": "FACT-105-5toyt", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_7fa076a056", "inTail": false, "evicted": true, "answer": "FACT-120-9h6hf", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_a72818ecec", "inTail": false, "evicted": true, "answer": "FACT-135-jw6fx", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_e379e4b23b", "inTail": false, "evicted": true, "answer": "FACT-150-70m0g", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_766a58bd2a", "inTail": false, "evicted": true, "answer": "FACT-165-30c3x", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_5d362e5446", "inTail": false, "evicted": true, "answer": "FACT-180-pp2gq", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_11639fa080", "inTail": false, "evicted": true, "answer": "FACT-195-h5u81", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_9e02dc8d1d", "inTail": false, "evicted": true, "answer": "FACT-210-skod2", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_23892ad26c", "inTail": false, "evicted": true, "answer": "FACT-225-vo6pc", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_572057a79c", "inTail": false, "evicted": true, "answer": "FACT-240-bnezh", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_d6b45e15be", "inTail": false, "evicted": true, "answer": "FACT-255-ouzrk", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_8692a8cf8a", "inTail": false, "evicted": true, "answer": "FACT-270-abmgy", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_f9021bef4b", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 20, "total": 20 } } ], "calls": [ { "seed": 1, "label": "control", "processExitCode": null, "processExitCodeNote": "Not retained by interrupted original bridge", "sessionTaskComplete": true, "model": "gpt-6.1-sol", "recordedToolCalls": 0, "usage": { "input_tokens": 23125, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 173, "reasoning_output_tokens": 62, "total_tokens": 23298 }, "sessionLogSha256": "766e5503b70165eb356362ba5f7d367c145cc041c0f698e6a478f487c4e9c6cd", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-rm66w", "diagnosticPublished": "not retained by interrupted original bridge" }, { "seed": 1, "label": "treatment", "processExitCode": null, "processExitCodeNote": "Not retained by interrupted original bridge", "sessionTaskComplete": true, "model": "gpt-6.1-sol", "recordedToolCalls": 0, "usage": { "input_tokens": 29369, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 430, "reasoning_output_tokens": 210, "total_tokens": 29799 }, "sessionLogSha256": "fa368695bbdcf0a6cccb1ee94fe553f96832ae2ceb88dd38ae654354f924d900", "answerText": "Q0: FACT-000-mkorf\nQ1: FACT-015-03jmy\nQ2: FACT-030-izkkh\nQ3: FACT-045-zbfww\nQ4: FACT-060-yv0n0\nQ5: FACT-075-a4b5x\nQ6: FACT-090-m28ly\nQ7: FACT-105-py2zp\nQ8: FACT-120-fbtzd\nQ9: FACT-135-ztagj\nQ10: FACT-150-ge14o\nQ11: FACT-165-hlgwo\nQ12: FACT-180-50232\nQ13: FACT-195-ej946\nQ14: FACT-210-8wsh3\nQ15: FACT-225-5k815\nQ16: FACT-240-hlxhw\nQ17: FACT-255-2fb7j\nQ18: FACT-270-e7fcx\nQ19: FACT-285-rm66w", "diagnosticPublished": "not retained by interrupted original bridge" }, { "seed": 2, "label": "control", "processExitCode": null, "processExitCodeNote": "Not retained by interrupted original bridge", "sessionTaskComplete": true, "model": "gpt-6.1-sol", "recordedToolCalls": 0, "usage": { "input_tokens": 23125, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 183, "reasoning_output_tokens": 72, "total_tokens": 23308 }, "sessionLogSha256": "e10b2fc01ca16e1f984586d8f67303fc705508c3081d20440ac3aaf4f5653a63", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-1esyu", "diagnosticPublished": "not retained by interrupted original bridge" }, { "seed": 2, "label": "treatment", "processExitCode": null, "processExitCodeNote": "Not retained by interrupted original bridge", "sessionTaskComplete": true, "model": "gpt-6.1-sol", "recordedToolCalls": 0, "usage": { "input_tokens": 29394, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 412, "reasoning_output_tokens": 195, "total_tokens": 29806 }, "sessionLogSha256": "6e3eaecef1a29ed5903bca8b4094efc617052161a3e2af9310efced02e74cec3", "answerText": "Q0: FACT-000-qfl7m\nQ1: FACT-015-bp74k\nQ2: FACT-030-a9qrt\nQ3: FACT-045-jd6u3\nQ4: FACT-060-vidfn\nQ5: FACT-075-mpk5r\nQ6: FACT-090-hyyqy\nQ7: FACT-105-cuzlu\nQ8: FACT-120-s4xt8\nQ9: FACT-135-43s9i\nQ10: FACT-150-saher\nQ11: FACT-165-797as\nQ12: FACT-180-f9so7\nQ13: FACT-195-u25n2\nQ14: FACT-210-kxe32\nQ15: FACT-225-52qt3\nQ16: FACT-240-o77fl\nQ17: FACT-255-d78oa\nQ18: FACT-270-yz7lh\nQ19: FACT-285-1esyu", "diagnosticPublished": "not retained by interrupted original bridge" }, { "seed": 3, "label": "control", "processExitCode": 0, "processExitCodeNote": "Captured by recovery runner", "sessionTaskComplete": true, "model": "gpt-6.1-sol", "recordedToolCalls": 0, "usage": { "input_tokens": 23094, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 176, "reasoning_output_tokens": 66, "total_tokens": 23270 }, "sessionLogSha256": "6ca30468d0bd736a4eff102c6ef97caffc0501453f93e961795fe8ad655e3889", "answerText": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-o73dt", "diagnosticSha256": "e1113167fef4d1c0cde1435c5c41addbde69fe00f16c7bb16f1694ec282a2268", "diagnosticPublished": "redacted; original retained locally", "analyticsWarningSeen": false, "initializeWarningSeen": true }, { "seed": 3, "label": "treatment", "processExitCode": 0, "processExitCodeNote": "Captured by recovery runner", "sessionTaskComplete": true, "model": "gpt-6.1-sol", "recordedToolCalls": 0, "usage": { "input_tokens": 29366, "cached_input_tokens": 13184, "cache_write_input_tokens": 0, "output_tokens": 464, "reasoning_output_tokens": 241, "total_tokens": 29830 }, "sessionLogSha256": "b551ff651a7f3a49b4c7e57d604127a5f453f318c408c5e2756563da19a85d1e", "answerText": "Q0: FACT-000-pxewf\nQ1: FACT-015-1e3tl\nQ2: FACT-030-gf83p\nQ3: FACT-045-2p3ua\nQ4: FACT-060-rgus2\nQ5: FACT-075-hblyb\nQ6: FACT-090-hisus\nQ7: FACT-105-5toyt\nQ8: FACT-120-9h6hf\nQ9: FACT-135-jw6fx\nQ10: FACT-150-70m0g\nQ11: FACT-165-30c3x\nQ12: FACT-180-pp2gq\nQ13: FACT-195-h5u81\nQ14: FACT-210-skod2\nQ15: FACT-225-vo6pc\nQ16: FACT-240-bnezh\nQ17: FACT-255-ouzrk\nQ18: FACT-270-abmgy\nQ19: FACT-285-o73dt", "diagnosticSha256": "5716330aa6ac1d20f7eb7ec601dac1a54dea36baaf343b8f24879eec3e1e255d", "diagnosticPublished": "redacted; original retained locally", "analyticsWarningSeen": false, "initializeWarningSeen": true } ], "caveats": [ "Synthetic 300-turn fixtures; the control sees tail record titles and summaries, not full bodies.", "Treatment receives preselected search/open evidence; this is not a model-driven memory-tool benchmark.", "The first four CLI process exit codes were not saved when the original bridge exited; their local sessions contain task_complete and matching final answers.", "The two missing seed-3 calls were run separately without repeating earlier calls.", "Complete original bridge stderr for the first four calls was not retained. Seed-3 diagnostic text is kept locally and omitted publicly.", "The original failure and failed SQLite files are preserved locally; root cause and monetary cost are UNKNOWN." ] }