{ "format": 1, "kind": "live-ab-codex-result", "generatedAt": "2026-10-29T10:54:52.718Z", "provider": "codex-cli", "model": "gpt-6.1-terra", "method": "codex exec --skip-git-repo-check --sandbox read-only --output-last-message; control/treatment generated by scripts/live-ab.mjs --emit-prompts", "seeds": [ { "seed": 1, "turns": 300, "tailCount": 20, "tailIds": [ "mem_7358d32b6c", "mem_7eff739202", "mem_3d08f99ce2", "mem_9891f0ec6b", "mem_dd677d7f04", "mem_fe809f2e75", "mem_32f2f72ebb", "mem_496aecf488", "mem_6226f16ade", "mem_d9fa54cc15", "mem_61caf38da8", "mem_62cbe323c5", "mem_65753b944b", "mem_8ef4df6ac9", "mem_3a83fb972d", "mem_4ad39f8323", "mem_9b55b23dad", "mem_b2b19adfeb", "mem_b4310d449f", "mem_761fc7be41" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_af5bcdb148", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_3b1ef42c83", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_4ea47579f7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_e64032b427", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_2c3eb7f0fb", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_2e60b68c64", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_5bf2530247", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_d18e8910f7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_6fb53ea600", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_85e2824b63", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_d3d3e00e05", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_11eaaee129", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_cd85e1de7c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_b8256aae47", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_d4df0ab2d3", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_45a2f9cee8", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_a1ae38f9ad", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_fc7389e2bb", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_02b3893892", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_dd677d7f04", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_af5bcdb148", "inTail": false, "evicted": true, "answer": "FACT-000-mkorf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_3b1ef42c83", "inTail": false, "evicted": true, "answer": "FACT-015-03jmy", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_4ea47579f7", "inTail": false, "evicted": true, "answer": "FACT-030-izkkh", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_e64032b427", "inTail": false, "evicted": true, "answer": "FACT-045-zbfww", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_2c3eb7f0fb", "inTail": false, "evicted": true, "answer": "FACT-060-yv0n0", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_2e60b68c64", "inTail": false, "evicted": true, "answer": "FACT-075-a4b5x", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_5bf2530247", "inTail": false, "evicted": true, "answer": "FACT-090-m28ly", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_d18e8910f7", "inTail": false, "evicted": true, "answer": "FACT-105-py2zp", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_6fb53ea600", "inTail": false, "evicted": true, "answer": "FACT-120-fbtzd", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_85e2824b63", "inTail": false, "evicted": true, "answer": "FACT-135-ztagj", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_d3d3e00e05", "inTail": false, "evicted": true, "answer": "FACT-150-ge14o", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_11eaaee129", "inTail": false, "evicted": true, "answer": "FACT-165-hlgwo", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_cd85e1de7c", "inTail": false, "evicted": true, "answer": "FACT-180-50232", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_b8256aae47", "inTail": false, "evicted": true, "answer": "FACT-195-ej946", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_d4df0ab2d3", "inTail": false, "evicted": true, "answer": "FACT-210-8wsh3", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_45a2f9cee8", "inTail": false, "evicted": true, "answer": "FACT-225-5k815", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_a1ae38f9ad", "inTail": false, "evicted": true, "answer": "FACT-240-hlxhw", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_fc7389e2bb", "inTail": false, "evicted": true, "answer": "FACT-255-2fb7j", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_02b3893892", "inTail": false, "evicted": true, "answer": "FACT-270-e7fcx", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_dd677d7f04", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 2, "turns": 300, "tailCount": 20, "tailIds": [ "mem_569575e8e8", "mem_64708b232c", "mem_af556e728f", "mem_e07b46ad60", "mem_5c8297f5f4", "mem_9b1dc27368", "mem_b10bd0fbec", "mem_fc3eef2c59", "mem_976152dd8e", "mem_bd35d0437c", "mem_8b60c85928", "mem_a1c112d964", "mem_d2486898ec", "mem_de3295d1fa", "mem_4498783e41", "mem_4bf7171270", "mem_b309e405b3", "mem_382f51c330", "mem_672a6dfc56", "mem_7ca63661e6" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_883fdd50ad", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_f79cc6bf10", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_380a319cab", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_0903398bfc", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_780d32644b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_fe221edb0e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_6585a35054", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_f96b63a00d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_2e22dfa3b5", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_a35b8680b6", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_d02e4535f5", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_a08f8d6ba4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_c097a77378", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_9e214dfb21", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_ef56f7220d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_ddf84e6846", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_ac4e10db13", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_fcbbfe21d5", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_b335e58e69", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_5c8297f5f4", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_883fdd50ad", "inTail": false, "evicted": true, "answer": "FACT-000-qfl7m", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_f79cc6bf10", "inTail": false, "evicted": true, "answer": "FACT-015-bp74k", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_380a319cab", "inTail": false, "evicted": true, "answer": "FACT-030-a9qrt", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_0903398bfc", "inTail": false, "evicted": true, "answer": "FACT-045-jd6u3", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_780d32644b", "inTail": false, "evicted": true, "answer": "FACT-060-vidfn", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_fe221edb0e", "inTail": false, "evicted": true, "answer": "FACT-075-mpk5r", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_6585a35054", "inTail": false, "evicted": true, "answer": "FACT-090-hyyqy", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_f96b63a00d", "inTail": false, "evicted": true, "answer": "FACT-105-cuzlu", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_2e22dfa3b5", "inTail": false, "evicted": true, "answer": "FACT-120-s4xt8", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_a35b8680b6", "inTail": false, "evicted": true, "answer": "FACT-135-43s9i", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_d02e4535f5", "inTail": false, "evicted": true, "answer": "FACT-150-saher", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_a08f8d6ba4", "inTail": false, "evicted": true, "answer": "FACT-165-797as", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_c097a77378", "inTail": false, "evicted": true, "answer": "FACT-180-f9so7", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_9e214dfb21", "inTail": false, "evicted": true, "answer": "FACT-195-u25n2", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_ef56f7220d", "inTail": false, "evicted": true, "answer": "FACT-210-kxe32", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_ddf84e6846", "inTail": false, "evicted": true, "answer": "FACT-225-52qt3", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_ac4e10db13", "inTail": false, "evicted": true, "answer": "FACT-240-o77fl", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_fcbbfe21d5", "inTail": false, "evicted": true, "answer": "FACT-255-d78oa", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_b335e58e69", "inTail": false, "evicted": true, "answer": "FACT-270-yz7lh", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_5c8297f5f4", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 3, "turns": 300, "tailCount": 20, "tailIds": [ "mem_1979436b14", "mem_6a8d37268f", "mem_cc4d45443c", "mem_38f68c6fbf", "mem_5da1db3a22", "mem_df1345f40f", "mem_2d91c0f663", "mem_85e56e4485", "mem_f4e0a712d4", "mem_fd85f4c506", "mem_3ef5d21e54", "mem_664e0a6353", "mem_aa22f2c010", "mem_f03e37dc6d", "mem_5118a22bd4", "mem_83ae17c242", "mem_8432f7bda2", "mem_caa7f06573", "mem_53e017644a", "mem_faf40b7add" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_cdf2cc86fe", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_b7b19584eb", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_a6cd8c1f41", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_f3e4432f34", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_614f8ebf91", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_0cac3b9a70", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_8e7249cf6a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_fa68d4acc8", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_cbc8d42ec3", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_0c1487ce7a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_a4939a7605", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_a2cecf7ecc", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_6e38067113", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_973b6e888e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_4601f47f26", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_555c55cf78", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_745226d5c7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_4c97e8b27c", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_8f4aa71aa4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_38f68c6fbf", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_cdf2cc86fe", "inTail": false, "evicted": true, "answer": "FACT-000-pxewf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_b7b19584eb", "inTail": false, "evicted": true, "answer": "FACT-015-1e3tl", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_a6cd8c1f41", "inTail": false, "evicted": true, "answer": "FACT-030-gf83p", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_f3e4432f34", "inTail": false, "evicted": true, "answer": "FACT-045-2p3ua", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_614f8ebf91", "inTail": false, "evicted": true, "answer": "FACT-060-rgus2", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_0cac3b9a70", "inTail": false, "evicted": true, "answer": "FACT-075-hblyb", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_8e7249cf6a", "inTail": false, "evicted": true, "answer": "FACT-090-hisus", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_fa68d4acc8", "inTail": false, "evicted": true, "answer": "FACT-105-5toyt", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_cbc8d42ec3", "inTail": false, "evicted": true, "answer": "FACT-120-9h6hf", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_0c1487ce7a", "inTail": false, "evicted": true, "answer": "FACT-135-jw6fx", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_a4939a7605", "inTail": false, "evicted": true, "answer": "FACT-150-70m0g", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_a2cecf7ecc", "inTail": false, "evicted": true, "answer": "FACT-165-30c3x", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_6e38067113", "inTail": false, "evicted": true, "answer": "FACT-180-pp2gq", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_973b6e888e", "inTail": false, "evicted": true, "answer": "FACT-195-h5u81", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_4601f47f26", "inTail": false, "evicted": true, "answer": "FACT-210-skod2", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_555c55cf78", "inTail": false, "evicted": true, "answer": "FACT-225-vo6pc", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_745226d5c7", "inTail": false, "evicted": true, "answer": "FACT-240-bnezh", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_4c97e8b27c", "inTail": false, "evicted": true, "answer": "FACT-255-ouzrk", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_8f4aa71aa4", "inTail": false, "evicted": true, "answer": "FACT-270-abmgy", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_38f68c6fbf", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 20, "total": 20 } } ], "aggregate": { "control": { "correct": 3, "n": 60, "accuracy": 0.05 }, "treatment": { "correct": 60, "n": 60, "accuracy": 1 }, "evictedControl": { "correct": 0, "n": 57, "accuracy": 0 }, "evictedTreatment": { "correct": 57, "n": 57, "accuracy": 1 } }, "raw": [ { "seed": 1, "label": "control", "status": 0, "stderrTail": "r_8W0IaPa7Kv_bZGaSodTA49eAfCGzl1njcNn1sQ9PlNpK9KzLeIohR6YAuUgGIIA.Nt6QFwqh7XgM5gqx7nwu8BmUUqC_s35Lo49WMtub5ZvCI7fErqJnz4w',mdrd: 'CoBfEgrefBRFdT21wYu_LZzuCyfQwYd_GFk83Rj9aaY-1793271302-1.2.1.1-28Ptr1rCfJnehjZKwXVZG3dWx6Ru1FIZoFV5fE_rbfGwHM44zZ0.wNgtWuUn2Dt6FHRT_JIYZ_exMzpTjwiGpX4.NXZ8UVxvHcjaoeRgYI3ZJqKMpa.6qlFiM5Y4qt0EB8Kx7UgMqAd0JdMfEPdnRwC3L8.D_GkYgHPWg7AvNzTnPk7pVgsAyG.5R7y1Ya2Napn59fWvIhGvQNsCiWHEaZbyaxYBxbzHMUvpt5_K7iUgDj7.xr6u0GXjE5rE1C6yWzB7Mwvv5uDc2k82C7A4G56OK2TP0oWsVC4sXCetnRFJn9sas1DfNJZ4HsGM3hhSG8NGt0c92R7BgDlQRArp2zJX4EG7M3saVkfa5YjVm.pUwOSx89QNEyBeghL44wMUdEEiy03AoGStz1PQ.rcTzqV28j5qo9GR.RyqVNxj2h4eQg85Gb_IZswAaINRP9bw8NqbwxwdhIYqi8ZpTw28OWbSZ0_dItgl805UCbtq_rj7nOxcWuC38y7ggTufjXgDJUsP.7MTkUqx2BPeNmfXpHa3boeXcpzutn.HWFx7EjMEf7ifvzJnZQp57inZn9TRm9mmWqcFt2XKuEGybhRK.trWLNnw5pnib1cmztGUlbTRz89zifPZtqKw.JZIU8c7fyeYexaL4vk1ae0cqynrza6WPkLx2qiEItPGZaUgPdQuYnzYW.ExdOMJ... (truncated 2462 bytes)\")\ncodex\nQ0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-rm66w\ntokens used\n10,000\n", "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-rm66w" }, { "seed": 1, "label": "treatment", "status": 0, "stderrTail": "viKDyXkTWecnuG49aY7akOkOsQAZ_vtlp2cddbWA-1793271337-1.2.1.1-x3XVaw.obwPlT2Dd9SzRX22tet1Yg6ybOIZ.5sAFTthg_RyIyk8OT80wWIC1CsN2qCtBg9Gcu_JX.aoSEyE8AYok5zi2pPvR3YIQatGfsEd7WNgHkYMLWN9KeP553NHhkbj02yHUjNeM0nCUa4Qmh.lcQR6S6s87RvNBHCh_09TiFU1GQZ6VAMAIJkMPObE8rVRSHbPNyCHO8Yve1oUx7_kgtb6c3M_mMJkL7aLnc.R78xD8WO0hp0k8fE0hJFIXiSo4zNbhzH_ukRuFpWOlxKBlyBZ.82o4L7tjGqWaWUCt7FDtYVYwjdUQ2ooswX7Ck3c_eUcuG3wi5vcKYo3ySXs5K7uIkmW8j0pjmdAK0T_kBUnFqbAsBMf4pGlDFaDnTA8siahtja6_zmR8K_6Hwe6jFVUaOPumthT5UUtnCHdNh5Jr7DNCqLVsoqW4XHqzbyTLfbEky5XBN4VQ.DCqcEO7fwSc2NlSS9siBXUauPCz7liq2D.QjbTLkE3eNeHeW68dVT.ubZdo_Nb6HQ.igBa9Cnm8h6enpryEiKUZOIPm_VCpavWl6Rnmly15ATeS82liZrTfCxyPYqX14twnZO8cYraKqwllXIqmk_OgClbvyZ9tikU1S6.CVLM_6X0QfHimCO8apnSaP7oiyxtb9JB2CwAfJVoe_sn5HslMCTuEcGiUttE18mMb... (truncated 2462 bytes)\")\ncodex\nQ0: FACT-000-mkorf\nQ1: FACT-015-03jmy\nQ2: FACT-030-izkkh\nQ3: FACT-045-zbfww\nQ4: FACT-060-yv0n0\nQ5: FACT-075-a4b5x\nQ6: FACT-090-m28ly\nQ7: FACT-105-py2zp\nQ8: FACT-120-fbtzd\nQ9: FACT-135-ztagj\nQ10: FACT-150-ge14o\nQ11: FACT-165-hlgwo\nQ12: FACT-180-50232\nQ13: FACT-195-ej946\nQ14: FACT-210-8wsh3\nQ15: FACT-225-5k815\nQ16: FACT-240-hlxhw\nQ17: FACT-255-2fb7j\nQ18: FACT-270-e7fcx\nQ19: FACT-285-rm66w\ntokens used\n16,571\n", "textTail": "Q0: FACT-000-mkorf\nQ1: FACT-015-03jmy\nQ2: FACT-030-izkkh\nQ3: FACT-045-zbfww\nQ4: FACT-060-yv0n0\nQ5: FACT-075-a4b5x\nQ6: FACT-090-m28ly\nQ7: FACT-105-py2zp\nQ8: FACT-120-fbtzd\nQ9: FACT-135-ztagj\nQ10: FACT-150-ge14o\nQ11: FACT-165-hlgwo\nQ12: FACT-180-50232\nQ13: FACT-195-ej946\nQ14: FACT-210-8wsh3\nQ15: FACT-225-5k815\nQ16: FACT-240-hlxhw\nQ17: FACT-255-2fb7j\nQ18: FACT-270-e7fcx\nQ19: FACT-285-rm66w" }, { "seed": 2, "label": "control", "status": 0, "stderrTail": "iOyLhXWgffbeMlE126w05tdEpH6ZHUhJUX.6XPbUqdX1PGs1dUyGk708LlNOjrf1.wHxV8ugdlynYP4t2hZg.WsKEUAyDsSz9hvw6S9eSkt5ZO9nZwMsHEqY4g',mdrd: 'MMCyM._flQUqdG7xsqJXPjcGMFwV7h0wlXB0CHVZPlA-1793271379-1.2.1.1-N6giXm8auQ6fuf2y7jGBETZPdTS2gNDREjodiqsaF6jt05ewObJwIWljbK90OME2FSGToBxyGkdt.RJW4DJuhwi4cm8.65GK8Lj5UN0ozuIQJt4ndLv6cTWOAyQTLCrjTwjtUVGoiePb6UyJQU3yp2Pc7F8jc6G3rWPGnQqv_G1DKW_3jwfzxubnp.pQE5eWR_fx.pWH.3EH8YcIn7bqpXuyJ_OHz2xWL69OKjeOLRx1.UpqoKpM0aifl8ysousVf3sxNB3NbR22bZ0EatU0oospn8ZzMCOaiMuWCx4HHQQbLv5Hm1Y0DrgBP0rkajDJnb8LuKCuMMQeS73mdAHfR3KxNWZlj7hUyqBzz8Lg6PfdRl62wyEnlaqZ8LZcZ14HBsMj3d2BfH1tya4pKPDIxNI2F9Q2W4SfKe7Z8eKirLnHKz0ZHi.sPMsgy3m0AXJfIlYCEw5D4iEP1qP0edYHpz.Qceoxwv5YtMXiSPFVPZdThOc79RN3nh7BQRe5snXJVWK8.ST17ZHhWX.Yonhv49iMZCYX.B52vO2yQOOdwtbK6v3qDJRag5h.MTewEHZm0s_gdzBlWoY4RO2oaBLotuahB1rRVbmeaKEN70kzWeLdxDn3T2awUHy6bXMN6rSYzBEZDAEVKsplsv8dblJlU3uHYGex3FfKlVXiBKttQaXqPY7_VYoa5Wbl... (truncated 2462 bytes)\")\ncodex\nQ0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-1esyu\ntokens used\n2,220\n", "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-1esyu" }, { "seed": 2, "label": "treatment", "status": 0, "stderrTail": "5eV_GGmQCjLQslSocGdCTWg6YvBpeTUZx5XN47U0-1793271419-1.2.1.1-XbBvEYLDPBAjBn90VV0.dZzV4Xm0Rs9PkWv58GJ5jmwTmklub8ZtgifnkY2TnS9_bsD3cvJnRhWNSvGKL5HiOC4iX_XJlKDWmRd21.2O65AcVKfiUhXaOl9pDV3gs9v5zGSAGxOtNe.ssVnoHgmPkN3twLsrsyuK6BEpxG6IZTK3VUBqYXOF8JBtR_nYV.KH2iFjBxz_H0KtCtsv3W.dMoo.YRVHMwPfAlXk6rlEYJEoedTLw85V9TdHKADRczgsyEvdhwtpgpsCuOedjjpJ3a.h2.i7IZHL8Y5JkWjkrjVV15bM_nr_WI6hocj.teul_vboyN0zyrolCLcr2e61QD_qIATa1CgAocQhDLa5Eq6yZwhPAoHf2fuuohq4D.crwt73ZLxUJAh51KGxdG3XT7v9hAXrxCS4Wvp0_7367P5BEJiU8PVntEY6bBhRnrjy9E2t87W3AFFme7zXvY9ZiAgQ0J3B4NIUNA7x_xxmQVDwPPWkaiK2_OtR5a_Ae6ZuFoqCQ4dUg.kFSUtS5YHhZyxY8IRyCBCBpYkLeIOctuZZPctCZjb1Aguh7LfpfVDTjJavc7G45jkv_artcE.tDEBBJJQiz_QjyfA3XTOY36Q_y4iRjOFjjlSXCSxLTgBLVoFDqjUA3bl8JdvHhr2_G7O2fdGIy8WlPNQVGNMkZY4zvoSdys4xz1Li... (truncated 2462 bytes)\")\ncodex\nQ0: FACT-000-qfl7m\nQ1: FACT-015-bp74k\nQ2: FACT-030-a9qrt\nQ3: FACT-045-jd6u3\nQ4: FACT-060-vidfn\nQ5: FACT-075-mpk5r\nQ6: FACT-090-hyyqy\nQ7: FACT-105-cuzlu\nQ8: FACT-120-s4xt8\nQ9: FACT-135-43s9i\nQ10: FACT-150-saher\nQ11: FACT-165-797as\nQ12: FACT-180-f9so7\nQ13: FACT-195-u25n2\nQ14: FACT-210-kxe32\nQ15: FACT-225-52qt3\nQ16: FACT-240-o77fl\nQ17: FACT-255-d78oa\nQ18: FACT-270-yz7lh\nQ19: FACT-285-1esyu\ntokens used\n20,404\n", "textTail": "Q0: FACT-000-qfl7m\nQ1: FACT-015-bp74k\nQ2: FACT-030-a9qrt\nQ3: FACT-045-jd6u3\nQ4: FACT-060-vidfn\nQ5: FACT-075-mpk5r\nQ6: FACT-090-hyyqy\nQ7: FACT-105-cuzlu\nQ8: FACT-120-s4xt8\nQ9: FACT-135-43s9i\nQ10: FACT-150-saher\nQ11: FACT-165-797as\nQ12: FACT-180-f9so7\nQ13: FACT-195-u25n2\nQ14: FACT-210-kxe32\nQ15: FACT-225-52qt3\nQ16: FACT-240-o77fl\nQ17: FACT-255-d78oa\nQ18: FACT-270-yz7lh\nQ19: FACT-285-1esyu" }, { "seed": 3, "label": "control", "status": 0, "stderrTail": "wCH2a9tL2Ndu31_lQRh4aRTqL4JpGwXp5YWWdqZef.7.3ZZ8w4GugTcUoOoUv0lCd3eaofzJ2Bb32e66gSD5UOvV9QR0YNOT8.kpPRPtEiDeaRXjNTTrMJ7Tg',mdrd: 'k1b3AOPCbA7P3mGctfTkkAPka0DsmsWvIEVB2By4QC0-1793271454-1.2.1.1-EhNlASfEmHjBc5HhuLqZKmcwsiINKHr8qbCY1ybkBzBfrh6WiHBSaJ80WhOsBJwzSxAYliIT6qCMJj3ihNgeS8y7wvv96lu7ux01_TMrLcFCiEImvbc_ddNcUGOeGPLInepsqzj_SGTX6giImw0yNNrT7i9gx1NTWUmhN2h.auFsJu_wPxcZdlYEXQuUbMdrI9No69akjgL_otXrG247uAhAwTGOhZe20vruQWMnY7Gr.Mriu74GNfgRcSziYwdjPmzLHe0.M.4MT1.8M0nw._CtO_SzXnA4_LyGassa_M1vN5Zh4PZOdOMJ5dv0WGuwafl66vLJA4RhaT2MUpQWeqLTZ.kOD99ud8d4qExFJnKaLu3TzaHvJgZHXK6zYZ9yU6188VU9pgd5O1vloC0xjOdXMdRAZxLCmQOqeF_j9npeeje5pj4T4Ns_qbTiFFFowea2ry_fw95zfiqAw8nB6SE.I.6XGR6.xCj9_P.LnmuFcvqAn0kWAQV_VwA.k.ESxEVVTHLl0z2LKCmyK44qpUOlbUXt0BkzjcMVCCy3iAenmxrTCmgL5nHLZovRI3SCXrdqEn7laECfhfJFUTBY2Mvl3cBiFZ5pFnZfMZ_tmJKDPIzaKKc.FNE_1C8GMTLqk6hJqVTlr_QnXuHPwmFQ9W5rgGL..ZKbkSRXF56Jty0tMvX1efFMkMKO... (truncated 2462 bytes)\")\ncodex\nQ0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-o73dt\ntokens used\n10,005\n", "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-o73dt" }, { "seed": 3, "label": "treatment", "status": 0, "stderrTail": "orded then?\nQ15 (turn 225, topic router): what durable value was recorded then?\nQ16 (turn 240, topic gateway): what durable value was recorded then?\nQ17 (turn 255, topic router): what durable value was recorded then?\nQ18 (turn 270, topic gateway): what durable value was recorded then?\nQ19 (turn 285, topic router): what durable value was recorded then?\n\nAnswer each question on its own line exactly as \"Q: \" from the evidence. Do not use tools. Do not explain.\n2026-10-29T10:58:10.268808Z ERROR codex_models_manager::manager: failed to refresh available models: request timed out\n2026-10-29T10:58:14.896629Z ERROR codex_api::endpoint::responses_websocket: failed to connect to websocket: IO error: tls handshake eof, url: wss://chatgpt.com/backend-api/codex/responses\ncodex\nQ0: FACT-000-pxewf\nQ1: FACT-015-1e3tl\nQ2: FACT-030-gf83p\nQ3: FACT-045-2p3ua\nQ4: FACT-060-rgus2\nQ5: FACT-075-hblyb\nQ6: FACT-090-hisus\nQ7: FACT-105-5toyt\nQ8: FACT-120-9h6hf\nQ9: FACT-135-jw6fx\nQ10: FACT-150-70m0g\nQ11: FACT-165-30c3x\nQ12: FACT-180-pp2gq\nQ13: FACT-195-h5u81\nQ14: FACT-210-skod2\nQ15: FACT-225-vo6pc\nQ16: FACT-240-bnezh\nQ17: FACT-255-ouzrk\nQ18: FACT-270-abmgy\nQ19: FACT-285-o73dt\ntokens used\n16,752\n", "textTail": "Q0: FACT-000-pxewf\nQ1: FACT-015-1e3tl\nQ2: FACT-030-gf83p\nQ3: FACT-045-2p3ua\nQ4: FACT-060-rgus2\nQ5: FACT-075-hblyb\nQ6: FACT-090-hisus\nQ7: FACT-105-5toyt\nQ8: FACT-120-9h6hf\nQ9: FACT-135-jw6fx\nQ10: FACT-150-70m0g\nQ11: FACT-165-30c3x\nQ12: FACT-180-pp2gq\nQ13: FACT-195-h5u81\nQ14: FACT-210-skod2\nQ15: FACT-225-vo6pc\nQ16: FACT-240-bnezh\nQ17: FACT-255-ouzrk\nQ18: FACT-270-abmgy\nQ19: FACT-285-o73dt" } ], "caveats": [ "Control/treatment prompts are synthetic fixtures; do not include private data.", "The Codex agent has its own tools; the prompt asks it not to use them. Review the raw replies before publishing.", "Usage/token counts are not captured by this bridge.", "Tail membership is saved explicitly; evicted subset is verified.", "Contributed from a local Codex CLI run; no keys or private prompts are included." ], "contributedBy": "Codex CLI (gpt-6.1-terra)", "bridge": "scripts/codex-ab.mjs", "verifiedTail": true, "seedsRun": "3 seeds x 20 probes" }