{
  "format": 1,
  "kind": "live-ab-codex-result",
  "generatedAt": "2026-10-29T11:05:43.568Z",
  "provider": "codex-cli",
  "model": "gpt-6.1-sol",
  "method": "codex exec --skip-git-repo-check --sandbox read-only --output-last-message; control/treatment generated by scripts/live-ab.mjs --emit-prompts",
  "seeds": [
    {
      "seed": 1,
      "turns": 300,
      "tailCount": 20,
      "tailIds": [
        "mem_212fa91ace",
        "mem_c7bed43a8b",
        "mem_f506dbc1ae",
        "mem_432432bb34",
        "mem_e826444fef",
        "mem_4ce1671669",
        "mem_87c8497987",
        "mem_7e968f4b82",
        "mem_0673f9a4ee",
        "mem_fc21458e21",
        "mem_d62bbbb3cb",
        "mem_b34b3d080f",
        "mem_99afd33a5b",
        "mem_1817b8b371",
        "mem_e8152a6079",
        "mem_e201ef0a26",
        "mem_8c90d32376",
        "mem_30ab697eb7",
        "mem_aa2af4b2f9",
        "mem_dac2701cf4"
      ],
      "probes": 20,
      "evictedProbes": 19,
      "control": {
        "rows": [
          {
            "index": 0,
            "turn": 0,
            "topic": "gateway",
            "expected": "FACT-000-mkorf",
            "id": "mem_9c21542b1f",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 1,
            "turn": 15,
            "topic": "router",
            "expected": "FACT-015-03jmy",
            "id": "mem_0179226227",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 2,
            "turn": 30,
            "topic": "gateway",
            "expected": "FACT-030-izkkh",
            "id": "mem_9c6599a6d3",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 3,
            "turn": 45,
            "topic": "router",
            "expected": "FACT-045-zbfww",
            "id": "mem_862bcaddc9",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 4,
            "turn": 60,
            "topic": "gateway",
            "expected": "FACT-060-yv0n0",
            "id": "mem_62863ad365",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 5,
            "turn": 75,
            "topic": "router",
            "expected": "FACT-075-a4b5x",
            "id": "mem_d5372f5622",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 6,
            "turn": 90,
            "topic": "gateway",
            "expected": "FACT-090-m28ly",
            "id": "mem_3b6c4793c5",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 7,
            "turn": 105,
            "topic": "router",
            "expected": "FACT-105-py2zp",
            "id": "mem_ac8aff11e7",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 8,
            "turn": 120,
            "topic": "gateway",
            "expected": "FACT-120-fbtzd",
            "id": "mem_a5fedea27b",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 9,
            "turn": 135,
            "topic": "router",
            "expected": "FACT-135-ztagj",
            "id": "mem_0ca902f4ee",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 10,
            "turn": 150,
            "topic": "gateway",
            "expected": "FACT-150-ge14o",
            "id": "mem_09894a057b",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 11,
            "turn": 165,
            "topic": "router",
            "expected": "FACT-165-hlgwo",
            "id": "mem_04ee0498a4",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 12,
            "turn": 180,
            "topic": "gateway",
            "expected": "FACT-180-50232",
            "id": "mem_d1fa71f0f1",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 13,
            "turn": 195,
            "topic": "router",
            "expected": "FACT-195-ej946",
            "id": "mem_0190eec4d4",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 14,
            "turn": 210,
            "topic": "gateway",
            "expected": "FACT-210-8wsh3",
            "id": "mem_4bb8461b65",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 15,
            "turn": 225,
            "topic": "router",
            "expected": "FACT-225-5k815",
            "id": "mem_d5932cc029",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 16,
            "turn": 240,
            "topic": "gateway",
            "expected": "FACT-240-hlxhw",
            "id": "mem_86e5069b33",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 17,
            "turn": 255,
            "topic": "router",
            "expected": "FACT-255-2fb7j",
            "id": "mem_c70a9b4407",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 18,
            "turn": 270,
            "topic": "gateway",
            "expected": "FACT-270-e7fcx",
            "id": "mem_17ec0fa036",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 19,
            "turn": 285,
            "topic": "router",
            "expected": "FACT-285-rm66w",
            "id": "mem_4ce1671669",
            "inTail": true,
            "evicted": false,
            "answer": "FACT-285-rm66w",
            "correct": true
          }
        ],
        "correct": 1,
        "total": 20
      },
      "treatment": {
        "rows": [
          {
            "index": 0,
            "turn": 0,
            "topic": "gateway",
            "expected": "FACT-000-mkorf",
            "id": "mem_9c21542b1f",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-000-mkorf",
            "correct": true
          },
          {
            "index": 1,
            "turn": 15,
            "topic": "router",
            "expected": "FACT-015-03jmy",
            "id": "mem_0179226227",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-015-03jmy",
            "correct": true
          },
          {
            "index": 2,
            "turn": 30,
            "topic": "gateway",
            "expected": "FACT-030-izkkh",
            "id": "mem_9c6599a6d3",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-030-izkkh",
            "correct": true
          },
          {
            "index": 3,
            "turn": 45,
            "topic": "router",
            "expected": "FACT-045-zbfww",
            "id": "mem_862bcaddc9",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-045-zbfww",
            "correct": true
          },
          {
            "index": 4,
            "turn": 60,
            "topic": "gateway",
            "expected": "FACT-060-yv0n0",
            "id": "mem_62863ad365",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-060-yv0n0",
            "correct": true
          },
          {
            "index": 5,
            "turn": 75,
            "topic": "router",
            "expected": "FACT-075-a4b5x",
            "id": "mem_d5372f5622",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-075-a4b5x",
            "correct": true
          },
          {
            "index": 6,
            "turn": 90,
            "topic": "gateway",
            "expected": "FACT-090-m28ly",
            "id": "mem_3b6c4793c5",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-090-m28ly",
            "correct": true
          },
          {
            "index": 7,
            "turn": 105,
            "topic": "router",
            "expected": "FACT-105-py2zp",
            "id": "mem_ac8aff11e7",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-105-py2zp",
            "correct": true
          },
          {
            "index": 8,
            "turn": 120,
            "topic": "gateway",
            "expected": "FACT-120-fbtzd",
            "id": "mem_a5fedea27b",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-120-fbtzd",
            "correct": true
          },
          {
            "index": 9,
            "turn": 135,
            "topic": "router",
            "expected": "FACT-135-ztagj",
            "id": "mem_0ca902f4ee",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-135-ztagj",
            "correct": true
          },
          {
            "index": 10,
            "turn": 150,
            "topic": "gateway",
            "expected": "FACT-150-ge14o",
            "id": "mem_09894a057b",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-150-ge14o",
            "correct": true
          },
          {
            "index": 11,
            "turn": 165,
            "topic": "router",
            "expected": "FACT-165-hlgwo",
            "id": "mem_04ee0498a4",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-165-hlgwo",
            "correct": true
          },
          {
            "index": 12,
            "turn": 180,
            "topic": "gateway",
            "expected": "FACT-180-50232",
            "id": "mem_d1fa71f0f1",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-180-50232",
            "correct": true
          },
          {
            "index": 13,
            "turn": 195,
            "topic": "router",
            "expected": "FACT-195-ej946",
            "id": "mem_0190eec4d4",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-195-ej946",
            "correct": true
          },
          {
            "index": 14,
            "turn": 210,
            "topic": "gateway",
            "expected": "FACT-210-8wsh3",
            "id": "mem_4bb8461b65",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-210-8wsh3",
            "correct": true
          },
          {
            "index": 15,
            "turn": 225,
            "topic": "router",
            "expected": "FACT-225-5k815",
            "id": "mem_d5932cc029",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-225-5k815",
            "correct": true
          },
          {
            "index": 16,
            "turn": 240,
            "topic": "gateway",
            "expected": "FACT-240-hlxhw",
            "id": "mem_86e5069b33",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-240-hlxhw",
            "correct": true
          },
          {
            "index": 17,
            "turn": 255,
            "topic": "router",
            "expected": "FACT-255-2fb7j",
            "id": "mem_c70a9b4407",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-255-2fb7j",
            "correct": true
          },
          {
            "index": 18,
            "turn": 270,
            "topic": "gateway",
            "expected": "FACT-270-e7fcx",
            "id": "mem_17ec0fa036",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-270-e7fcx",
            "correct": true
          },
          {
            "index": 19,
            "turn": 285,
            "topic": "router",
            "expected": "FACT-285-rm66w",
            "id": "mem_4ce1671669",
            "inTail": true,
            "evicted": false,
            "answer": "FACT-285-rm66w",
            "correct": true
          }
        ],
        "correct": 20,
        "total": 20
      }
    },
    {
      "seed": 2,
      "turns": 300,
      "tailCount": 20,
      "tailIds": [
        "mem_538d1a02d5",
        "mem_58a15151ec",
        "mem_0176793df6",
        "mem_3b6cbdbbb7",
        "mem_d335ebbbe4",
        "mem_311b16c744",
        "mem_9ad4f5ff32",
        "mem_c29319428d",
        "mem_32fec15c04",
        "mem_cf352bc283",
        "mem_d166822cdd",
        "mem_70cee94181",
        "mem_b82daeceda",
        "mem_f775ff0bfc",
        "mem_af47919027",
        "mem_3dd7a79589",
        "mem_f7a167f167",
        "mem_5f42481df2",
        "mem_16162d6bbd",
        "mem_02b6b298f9"
      ],
      "probes": 20,
      "evictedProbes": 19,
      "control": {
        "rows": [
          {
            "index": 0,
            "turn": 0,
            "topic": "gateway",
            "expected": "FACT-000-qfl7m",
            "id": "mem_9252da08cd",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 1,
            "turn": 15,
            "topic": "router",
            "expected": "FACT-015-bp74k",
            "id": "mem_0197f5984b",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 2,
            "turn": 30,
            "topic": "gateway",
            "expected": "FACT-030-a9qrt",
            "id": "mem_542025e485",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 3,
            "turn": 45,
            "topic": "router",
            "expected": "FACT-045-jd6u3",
            "id": "mem_af63bbd55b",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 4,
            "turn": 60,
            "topic": "gateway",
            "expected": "FACT-060-vidfn",
            "id": "mem_867d950150",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 5,
            "turn": 75,
            "topic": "router",
            "expected": "FACT-075-mpk5r",
            "id": "mem_ade1541ca8",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 6,
            "turn": 90,
            "topic": "gateway",
            "expected": "FACT-090-hyyqy",
            "id": "mem_fe87a4317e",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 7,
            "turn": 105,
            "topic": "router",
            "expected": "FACT-105-cuzlu",
            "id": "mem_a7a8edea68",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 8,
            "turn": 120,
            "topic": "gateway",
            "expected": "FACT-120-s4xt8",
            "id": "mem_e5fe5f9573",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 9,
            "turn": 135,
            "topic": "router",
            "expected": "FACT-135-43s9i",
            "id": "mem_cbbb11cd94",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 10,
            "turn": 150,
            "topic": "gateway",
            "expected": "FACT-150-saher",
            "id": "mem_2e86b7d941",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 11,
            "turn": 165,
            "topic": "router",
            "expected": "FACT-165-797as",
            "id": "mem_0f6969739c",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 12,
            "turn": 180,
            "topic": "gateway",
            "expected": "FACT-180-f9so7",
            "id": "mem_9bf3feb244",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 13,
            "turn": 195,
            "topic": "router",
            "expected": "FACT-195-u25n2",
            "id": "mem_d5034a77c7",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 14,
            "turn": 210,
            "topic": "gateway",
            "expected": "FACT-210-kxe32",
            "id": "mem_0813ec5521",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 15,
            "turn": 225,
            "topic": "router",
            "expected": "FACT-225-52qt3",
            "id": "mem_bf7abfb882",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 16,
            "turn": 240,
            "topic": "gateway",
            "expected": "FACT-240-o77fl",
            "id": "mem_ac38372ccf",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 17,
            "turn": 255,
            "topic": "router",
            "expected": "FACT-255-d78oa",
            "id": "mem_73fdfe786c",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 18,
            "turn": 270,
            "topic": "gateway",
            "expected": "FACT-270-yz7lh",
            "id": "mem_4e098daad1",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 19,
            "turn": 285,
            "topic": "router",
            "expected": "FACT-285-1esyu",
            "id": "mem_311b16c744",
            "inTail": true,
            "evicted": false,
            "answer": "FACT-285-1esyu",
            "correct": true
          }
        ],
        "correct": 1,
        "total": 20
      },
      "treatment": {
        "rows": [
          {
            "index": 0,
            "turn": 0,
            "topic": "gateway",
            "expected": "FACT-000-qfl7m",
            "id": "mem_9252da08cd",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-000-qfl7m",
            "correct": true
          },
          {
            "index": 1,
            "turn": 15,
            "topic": "router",
            "expected": "FACT-015-bp74k",
            "id": "mem_0197f5984b",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-015-bp74k",
            "correct": true
          },
          {
            "index": 2,
            "turn": 30,
            "topic": "gateway",
            "expected": "FACT-030-a9qrt",
            "id": "mem_542025e485",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-030-a9qrt",
            "correct": true
          },
          {
            "index": 3,
            "turn": 45,
            "topic": "router",
            "expected": "FACT-045-jd6u3",
            "id": "mem_af63bbd55b",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-045-jd6u3",
            "correct": true
          },
          {
            "index": 4,
            "turn": 60,
            "topic": "gateway",
            "expected": "FACT-060-vidfn",
            "id": "mem_867d950150",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-060-vidfn",
            "correct": true
          },
          {
            "index": 5,
            "turn": 75,
            "topic": "router",
            "expected": "FACT-075-mpk5r",
            "id": "mem_ade1541ca8",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-075-mpk5r",
            "correct": true
          },
          {
            "index": 6,
            "turn": 90,
            "topic": "gateway",
            "expected": "FACT-090-hyyqy",
            "id": "mem_fe87a4317e",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-090-hyyqy",
            "correct": true
          },
          {
            "index": 7,
            "turn": 105,
            "topic": "router",
            "expected": "FACT-105-cuzlu",
            "id": "mem_a7a8edea68",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-105-cuzlu",
            "correct": true
          },
          {
            "index": 8,
            "turn": 120,
            "topic": "gateway",
            "expected": "FACT-120-s4xt8",
            "id": "mem_e5fe5f9573",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-120-s4xt8",
            "correct": true
          },
          {
            "index": 9,
            "turn": 135,
            "topic": "router",
            "expected": "FACT-135-43s9i",
            "id": "mem_cbbb11cd94",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-135-43s9i",
            "correct": true
          },
          {
            "index": 10,
            "turn": 150,
            "topic": "gateway",
            "expected": "FACT-150-saher",
            "id": "mem_2e86b7d941",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-150-saher",
            "correct": true
          },
          {
            "index": 11,
            "turn": 165,
            "topic": "router",
            "expected": "FACT-165-797as",
            "id": "mem_0f6969739c",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-165-797as",
            "correct": true
          },
          {
            "index": 12,
            "turn": 180,
            "topic": "gateway",
            "expected": "FACT-180-f9so7",
            "id": "mem_9bf3feb244",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-180-f9so7",
            "correct": true
          },
          {
            "index": 13,
            "turn": 195,
            "topic": "router",
            "expected": "FACT-195-u25n2",
            "id": "mem_d5034a77c7",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-195-u25n2",
            "correct": true
          },
          {
            "index": 14,
            "turn": 210,
            "topic": "gateway",
            "expected": "FACT-210-kxe32",
            "id": "mem_0813ec5521",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-210-kxe32",
            "correct": true
          },
          {
            "index": 15,
            "turn": 225,
            "topic": "router",
            "expected": "FACT-225-52qt3",
            "id": "mem_bf7abfb882",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-225-52qt3",
            "correct": true
          },
          {
            "index": 16,
            "turn": 240,
            "topic": "gateway",
            "expected": "FACT-240-o77fl",
            "id": "mem_ac38372ccf",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-240-o77fl",
            "correct": true
          },
          {
            "index": 17,
            "turn": 255,
            "topic": "router",
            "expected": "FACT-255-d78oa",
            "id": "mem_73fdfe786c",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-255-d78oa",
            "correct": true
          },
          {
            "index": 18,
            "turn": 270,
            "topic": "gateway",
            "expected": "FACT-270-yz7lh",
            "id": "mem_4e098daad1",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-270-yz7lh",
            "correct": true
          },
          {
            "index": 19,
            "turn": 285,
            "topic": "router",
            "expected": "FACT-285-1esyu",
            "id": "mem_311b16c744",
            "inTail": true,
            "evicted": false,
            "answer": "FACT-285-1esyu",
            "correct": true
          }
        ],
        "correct": 20,
        "total": 20
      }
    },
    {
      "seed": 3,
      "turns": 300,
      "tailCount": 20,
      "tailIds": [
        "mem_b21e238924",
        "mem_5f3225343e",
        "mem_2dc29eabe0",
        "mem_86a499927c",
        "mem_902cd9db06",
        "mem_8df971f7b1",
        "mem_54301f1ce9",
        "mem_09758b28a2",
        "mem_5d54458b4b",
        "mem_d735ba65b2",
        "mem_8cacde5735",
        "mem_066c7e85df",
        "mem_ab7cbaa7a2",
        "mem_144bd85db7",
        "mem_32eeb66a5a",
        "mem_ad304b7d52",
        "mem_770554f2b6",
        "mem_7c7cff2e7b",
        "mem_1d41633d09",
        "mem_ed7f306e71"
      ],
      "probes": 20,
      "evictedProbes": 19,
      "control": {
        "rows": [
          {
            "index": 0,
            "turn": 0,
            "topic": "gateway",
            "expected": "FACT-000-pxewf",
            "id": "mem_dd89292728",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 1,
            "turn": 15,
            "topic": "router",
            "expected": "FACT-015-1e3tl",
            "id": "mem_1a8e773a8e",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 2,
            "turn": 30,
            "topic": "gateway",
            "expected": "FACT-030-gf83p",
            "id": "mem_2f2f378a0e",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 3,
            "turn": 45,
            "topic": "router",
            "expected": "FACT-045-2p3ua",
            "id": "mem_9a6d250ba9",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 4,
            "turn": 60,
            "topic": "gateway",
            "expected": "FACT-060-rgus2",
            "id": "mem_f1c06ae66a",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 5,
            "turn": 75,
            "topic": "router",
            "expected": "FACT-075-hblyb",
            "id": "mem_0b7408ea88",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 6,
            "turn": 90,
            "topic": "gateway",
            "expected": "FACT-090-hisus",
            "id": "mem_b5b46c82c2",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 7,
            "turn": 105,
            "topic": "router",
            "expected": "FACT-105-5toyt",
            "id": "mem_94c9a6d99d",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 8,
            "turn": 120,
            "topic": "gateway",
            "expected": "FACT-120-9h6hf",
            "id": "mem_b8c26384c1",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 9,
            "turn": 135,
            "topic": "router",
            "expected": "FACT-135-jw6fx",
            "id": "mem_6b5ba2073b",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 10,
            "turn": 150,
            "topic": "gateway",
            "expected": "FACT-150-70m0g",
            "id": "mem_af494b11ef",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 11,
            "turn": 165,
            "topic": "router",
            "expected": "FACT-165-30c3x",
            "id": "mem_a7c3c1e36c",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 12,
            "turn": 180,
            "topic": "gateway",
            "expected": "FACT-180-pp2gq",
            "id": "mem_d12b66ac3b",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 13,
            "turn": 195,
            "topic": "router",
            "expected": "FACT-195-h5u81",
            "id": "mem_9b8ea08f99",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 14,
            "turn": 210,
            "topic": "gateway",
            "expected": "FACT-210-skod2",
            "id": "mem_f26fdd0a41",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 15,
            "turn": 225,
            "topic": "router",
            "expected": "FACT-225-vo6pc",
            "id": "mem_93b3e05b52",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 16,
            "turn": 240,
            "topic": "gateway",
            "expected": "FACT-240-bnezh",
            "id": "mem_b4f14990a0",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 17,
            "turn": 255,
            "topic": "router",
            "expected": "FACT-255-ouzrk",
            "id": "mem_42f7c843ba",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 18,
            "turn": 270,
            "topic": "gateway",
            "expected": "FACT-270-abmgy",
            "id": "mem_6d350c3b45",
            "inTail": false,
            "evicted": true,
            "answer": "UNKNOWN",
            "correct": false
          },
          {
            "index": 19,
            "turn": 285,
            "topic": "router",
            "expected": "FACT-285-o73dt",
            "id": "mem_8df971f7b1",
            "inTail": true,
            "evicted": false,
            "answer": "FACT-285-o73dt",
            "correct": true
          }
        ],
        "correct": 1,
        "total": 20
      },
      "treatment": {
        "rows": [
          {
            "index": 0,
            "turn": 0,
            "topic": "gateway",
            "expected": "FACT-000-pxewf",
            "id": "mem_dd89292728",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-000-pxewf",
            "correct": true
          },
          {
            "index": 1,
            "turn": 15,
            "topic": "router",
            "expected": "FACT-015-1e3tl",
            "id": "mem_1a8e773a8e",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-015-1e3tl",
            "correct": true
          },
          {
            "index": 2,
            "turn": 30,
            "topic": "gateway",
            "expected": "FACT-030-gf83p",
            "id": "mem_2f2f378a0e",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-030-gf83p",
            "correct": true
          },
          {
            "index": 3,
            "turn": 45,
            "topic": "router",
            "expected": "FACT-045-2p3ua",
            "id": "mem_9a6d250ba9",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-045-2p3ua",
            "correct": true
          },
          {
            "index": 4,
            "turn": 60,
            "topic": "gateway",
            "expected": "FACT-060-rgus2",
            "id": "mem_f1c06ae66a",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-060-rgus2",
            "correct": true
          },
          {
            "index": 5,
            "turn": 75,
            "topic": "router",
            "expected": "FACT-075-hblyb",
            "id": "mem_0b7408ea88",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-075-hblyb",
            "correct": true
          },
          {
            "index": 6,
            "turn": 90,
            "topic": "gateway",
            "expected": "FACT-090-hisus",
            "id": "mem_b5b46c82c2",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-090-hisus",
            "correct": true
          },
          {
            "index": 7,
            "turn": 105,
            "topic": "router",
            "expected": "FACT-105-5toyt",
            "id": "mem_94c9a6d99d",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-105-5toyt",
            "correct": true
          },
          {
            "index": 8,
            "turn": 120,
            "topic": "gateway",
            "expected": "FACT-120-9h6hf",
            "id": "mem_b8c26384c1",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-120-9h6hf",
            "correct": true
          },
          {
            "index": 9,
            "turn": 135,
            "topic": "router",
            "expected": "FACT-135-jw6fx",
            "id": "mem_6b5ba2073b",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-135-jw6fx",
            "correct": true
          },
          {
            "index": 10,
            "turn": 150,
            "topic": "gateway",
            "expected": "FACT-150-70m0g",
            "id": "mem_af494b11ef",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-150-70m0g",
            "correct": true
          },
          {
            "index": 11,
            "turn": 165,
            "topic": "router",
            "expected": "FACT-165-30c3x",
            "id": "mem_a7c3c1e36c",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-165-30c3x",
            "correct": true
          },
          {
            "index": 12,
            "turn": 180,
            "topic": "gateway",
            "expected": "FACT-180-pp2gq",
            "id": "mem_d12b66ac3b",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-180-pp2gq",
            "correct": true
          },
          {
            "index": 13,
            "turn": 195,
            "topic": "router",
            "expected": "FACT-195-h5u81",
            "id": "mem_9b8ea08f99",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-195-h5u81",
            "correct": true
          },
          {
            "index": 14,
            "turn": 210,
            "topic": "gateway",
            "expected": "FACT-210-skod2",
            "id": "mem_f26fdd0a41",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-210-skod2",
            "correct": true
          },
          {
            "index": 15,
            "turn": 225,
            "topic": "router",
            "expected": "FACT-225-vo6pc",
            "id": "mem_93b3e05b52",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-225-vo6pc",
            "correct": true
          },
          {
            "index": 16,
            "turn": 240,
            "topic": "gateway",
            "expected": "FACT-240-bnezh",
            "id": "mem_b4f14990a0",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-240-bnezh",
            "correct": true
          },
          {
            "index": 17,
            "turn": 255,
            "topic": "router",
            "expected": "FACT-255-ouzrk",
            "id": "mem_42f7c843ba",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-255-ouzrk",
            "correct": true
          },
          {
            "index": 18,
            "turn": 270,
            "topic": "gateway",
            "expected": "FACT-270-abmgy",
            "id": "mem_6d350c3b45",
            "inTail": false,
            "evicted": true,
            "answer": "FACT-270-abmgy",
            "correct": true
          },
          {
            "index": 19,
            "turn": 285,
            "topic": "router",
            "expected": "FACT-285-o73dt",
            "id": "mem_8df971f7b1",
            "inTail": true,
            "evicted": false,
            "answer": "FACT-285-o73dt",
            "correct": true
          }
        ],
        "correct": 20,
        "total": 20
      }
    }
  ],
  "aggregate": {
    "control": {
      "correct": 3,
      "n": 60,
      "accuracy": 0.05
    },
    "treatment": {
      "correct": 60,
      "n": 60,
      "accuracy": 1
    },
    "evictedControl": {
      "correct": 0,
      "n": 57,
      "accuracy": 0
    },
    "evictedTreatment": {
      "correct": 57,
      "n": 57,
      "accuracy": 1
    }
  },
  "raw": [
    {
      "seed": 1,
      "label": "control",
      "status": 0,
      "stderrTail": "[redacted from public artifact: diagnostic output may contain sensitive local or authentication data]",
      "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-rm66w",
      "stderrTailSha256": "fbc8f9edf25e8684182af08a3dececa3804320398eb70fc3265e79bd00b07695",
      "cliReportedTokens": 10096
    },
    {
      "seed": 1,
      "label": "treatment",
      "status": 0,
      "stderrTail": "[redacted from public artifact: diagnostic output may contain sensitive local or authentication data]",
      "textTail": "Q0: FACT-000-mkorf\nQ1: FACT-015-03jmy\nQ2: FACT-030-izkkh\nQ3: FACT-045-zbfww\nQ4: FACT-060-yv0n0\nQ5: FACT-075-a4b5x\nQ6: FACT-090-m28ly\nQ7: FACT-105-py2zp\nQ8: FACT-120-fbtzd\nQ9: FACT-135-ztagj\nQ10: FACT-150-ge14o\nQ11: FACT-165-hlgwo\nQ12: FACT-180-50232\nQ13: FACT-195-ej946\nQ14: FACT-210-8wsh3\nQ15: FACT-225-5k815\nQ16: FACT-240-hlxhw\nQ17: FACT-255-2fb7j\nQ18: FACT-270-e7fcx\nQ19: FACT-285-rm66w",
      "stderrTailSha256": "4f6a29356d4ae7d130aaf641fe03e4a1af1b4ad5154cf1670a081bfba3b2d338",
      "cliReportedTokens": 16682
    },
    {
      "seed": 2,
      "label": "control",
      "status": 0,
      "stderrTail": "[redacted from public artifact: diagnostic output may contain sensitive local or authentication data]",
      "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-1esyu",
      "stderrTailSha256": "ac0f04c95fc288ea47f0337a9d539824414670358219fb1d9a579d2a54af929d",
      "cliReportedTokens": 10112
    },
    {
      "seed": 2,
      "label": "treatment",
      "status": 0,
      "stderrTail": "[redacted from public artifact: diagnostic output may contain sensitive local or authentication data]",
      "textTail": "Q0: FACT-000-qfl7m\nQ1: FACT-015-bp74k\nQ2: FACT-030-a9qrt\nQ3: FACT-045-jd6u3\nQ4: FACT-060-vidfn\nQ5: FACT-075-mpk5r\nQ6: FACT-090-hyyqy\nQ7: FACT-105-cuzlu\nQ8: FACT-120-s4xt8\nQ9: FACT-135-43s9i\nQ10: FACT-150-saher\nQ11: FACT-165-797as\nQ12: FACT-180-f9so7\nQ13: FACT-195-u25n2\nQ14: FACT-210-kxe32\nQ15: FACT-225-52qt3\nQ16: FACT-240-o77fl\nQ17: FACT-255-d78oa\nQ18: FACT-270-yz7lh\nQ19: FACT-285-1esyu",
      "stderrTailSha256": "8e6d9a82ce1ee7f7500a78064081955051a4c99eef99de27b4dcdd9b34a0602b",
      "cliReportedTokens": 16723
    },
    {
      "seed": 3,
      "label": "control",
      "status": 0,
      "stderrTail": "[redacted from public artifact: diagnostic output may contain sensitive local or authentication data]",
      "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-o73dt",
      "stderrTailSha256": "0fc6463dd7baec1b9ffb12bd38b96e68e479c2028ac6db37f0097568c70d0869",
      "cliReportedTokens": 10157
    },
    {
      "seed": 3,
      "label": "treatment",
      "status": 0,
      "stderrTail": "[redacted from public artifact: diagnostic output may contain sensitive local or authentication data]",
      "textTail": "Q0: FACT-000-pxewf\nQ1: FACT-015-1e3tl\nQ2: FACT-030-gf83p\nQ3: FACT-045-2p3ua\nQ4: FACT-060-rgus2\nQ5: FACT-075-hblyb\nQ6: FACT-090-hisus\nQ7: FACT-105-5toyt\nQ8: FACT-120-9h6hf\nQ9: FACT-135-jw6fx\nQ10: FACT-150-70m0g\nQ11: FACT-165-30c3x\nQ12: FACT-180-pp2gq\nQ13: FACT-195-h5u81\nQ14: FACT-210-skod2\nQ15: FACT-225-vo6pc\nQ16: FACT-240-bnezh\nQ17: FACT-255-ouzrk\nQ18: FACT-270-abmgy\nQ19: FACT-285-o73dt",
      "stderrTailSha256": "3af3f2d8ec19012ca630bee4e6840bf45a1d4ae14fee3a1f0d4a7fc764fe5d72",
      "cliReportedTokens": 16682
    }
  ],
  "caveats": [
    "Control/treatment prompts are synthetic fixtures; do not include private data.",
    "The Codex agent has its own tools; the prompt asks it not to use them. Review the raw replies before publishing.",
    "The bridge does not capture structured provider usage or fees. CLI-reported total token counts were extracted from diagnostic tails; their billing meaning is unknown."
  ],
  "testedCommit": "c1cda8ba1ba931fa4db9acdf6787cb0a80872c00",
  "originalReportSha256": "574d5eaf9d3564580fe3cdba769b40ce0c2627e137774a6b19ef92b81f2d7fbb",
  "publicationNote": "The original report is retained locally. Diagnostic stderr tails are redacted in this public copy; exact answers, grades, statuses, and tail IDs are unchanged. CLI-reported token totals are parsed from the retained stderr tails and are not provider billing receipts."
}