{ "format": 1, "kind": "live-ab-claude-result", "generatedAt": "2026-10-29T11:14:32.088Z", "provider": "apimaster", "baseUrl": "https://apimaster.ai/v1", "model": "claude-fable-5-1", "method": "live-ab.mjs --emit-prompts batch prompts; OpenAI-compatible /chat/completions; exact-FACT grading; tail membership saved", "seeds": [ { "seed": 1, "turns": 300, "tailCount": 20, "tailIds": [ "mem_32ac64dca1", "mem_51e1e327c9", "mem_8ee50f268d", "mem_ee0487a666", "mem_91e13dd57b", "mem_b8258cb2e4", "mem_d97f5d6691", "mem_84ed994629", "mem_96326b2518", "mem_c4ae175b49", "mem_f695404c1d", "mem_0a7774d802", "mem_2b3400de82", "mem_8e21141470", "mem_083df30ce2", "mem_47be586302", "mem_a15c68dab5", "mem_d046c6cee0", "mem_3e49078cf1", "mem_bae3967de1" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_536cc79f80", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_8c3a6f4277", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_600234ed71", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_53865039ed", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_e1b5e51cd5", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_2c7f50f1fa", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_571cad5aa1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_d5164386bf", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_38c99ed59f", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_85a11bc10d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_68818dff8b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_cca5b37697", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_836d36ab87", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_79823c335e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_0ab47785de", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_cd540a3cb4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_d8c060b02d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_1ecb7d0de6", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_9a49c74c0e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_d97f5d6691", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-mkorf", "id": "mem_536cc79f80", "inTail": false, "evicted": true, "answer": "FACT-000-mkorf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-03jmy", "id": "mem_8c3a6f4277", "inTail": false, "evicted": true, "answer": "FACT-015-03jmy", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-izkkh", "id": "mem_600234ed71", "inTail": false, "evicted": true, "answer": "FACT-030-izkkh", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-zbfww", "id": "mem_53865039ed", "inTail": false, "evicted": true, "answer": "FACT-045-zbfww", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-yv0n0", "id": "mem_e1b5e51cd5", "inTail": false, "evicted": true, "answer": "FACT-060-yv0n0", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-a4b5x", "id": "mem_2c7f50f1fa", "inTail": false, "evicted": true, "answer": "FACT-075-a4b5x", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-m28ly", "id": "mem_571cad5aa1", "inTail": false, "evicted": true, "answer": "FACT-090-m28ly", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-py2zp", "id": "mem_d5164386bf", "inTail": false, "evicted": true, "answer": "FACT-105-py2zp", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-fbtzd", "id": "mem_38c99ed59f", "inTail": false, "evicted": true, "answer": "FACT-120-fbtzd", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-ztagj", "id": "mem_85a11bc10d", "inTail": false, "evicted": true, "answer": "FACT-135-ztagj", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-ge14o", "id": "mem_68818dff8b", "inTail": false, "evicted": true, "answer": "FACT-150-ge14o", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-hlgwo", "id": "mem_cca5b37697", "inTail": false, "evicted": true, "answer": "FACT-165-hlgwo", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-50232", "id": "mem_836d36ab87", "inTail": false, "evicted": true, "answer": "FACT-180-50232", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-ej946", "id": "mem_79823c335e", "inTail": false, "evicted": true, "answer": "FACT-195-ej946", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-8wsh3", "id": "mem_0ab47785de", "inTail": false, "evicted": true, "answer": "FACT-210-8wsh3", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-5k815", "id": "mem_cd540a3cb4", "inTail": false, "evicted": true, "answer": "FACT-225-5k815", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-hlxhw", "id": "mem_d8c060b02d", "inTail": false, "evicted": true, "answer": "FACT-240-hlxhw", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-2fb7j", "id": "mem_1ecb7d0de6", "inTail": false, "evicted": true, "answer": "FACT-255-2fb7j", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-e7fcx", "id": "mem_9a49c74c0e", "inTail": false, "evicted": true, "answer": "FACT-270-e7fcx", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-rm66w", "id": "mem_d97f5d6691", "inTail": true, "evicted": false, "answer": "FACT-285-rm66w", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 2, "turns": 300, "tailCount": 20, "tailIds": [ "mem_e146e4b337", "mem_f7e0636803", "mem_02cf56c488", "mem_10dff5e3d6", "mem_625117fa7a", "mem_a08f3905a8", "mem_2a6f3ce21c", "mem_7ddd0eb971", "mem_b06c566fac", "mem_cfd4b2317d", "mem_88c3fe20c0", "mem_8dc20b8fcd", "mem_d626fd9963", "mem_ef988270f3", "mem_495d5cf0b2", "mem_5d139fd850", "mem_7faa4fddb6", "mem_aad8e79405", "mem_b306c5aa7e", "mem_c8833749f4" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_a1d510b5e7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_4fb75f3f83", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_8fc03fea1b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_0d926a6d3e", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_455f92c737", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_bc33213131", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_156d461130", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_e8f3e81b5d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_65ddbc90a8", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_c3bac40cce", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_e64b6376e1", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_80ad877e72", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_43c1cd03ee", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_cca80e53c9", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_5e0a884190", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_47c66e92db", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_c34265a388", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_54ef1e20dd", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_43171ae47d", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_625117fa7a", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-qfl7m", "id": "mem_a1d510b5e7", "inTail": false, "evicted": true, "answer": "FACT-000-qfl7m", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-bp74k", "id": "mem_4fb75f3f83", "inTail": false, "evicted": true, "answer": "FACT-015-bp74k", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-a9qrt", "id": "mem_8fc03fea1b", "inTail": false, "evicted": true, "answer": "FACT-030-a9qrt", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-jd6u3", "id": "mem_0d926a6d3e", "inTail": false, "evicted": true, "answer": "FACT-045-jd6u3", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-vidfn", "id": "mem_455f92c737", "inTail": false, "evicted": true, "answer": "FACT-060-vidfn", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-mpk5r", "id": "mem_bc33213131", "inTail": false, "evicted": true, "answer": "FACT-075-mpk5r", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hyyqy", "id": "mem_156d461130", "inTail": false, "evicted": true, "answer": "FACT-090-hyyqy", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-cuzlu", "id": "mem_e8f3e81b5d", "inTail": false, "evicted": true, "answer": "FACT-105-cuzlu", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-s4xt8", "id": "mem_65ddbc90a8", "inTail": false, "evicted": true, "answer": "FACT-120-s4xt8", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-43s9i", "id": "mem_c3bac40cce", "inTail": false, "evicted": true, "answer": "FACT-135-43s9i", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-saher", "id": "mem_e64b6376e1", "inTail": false, "evicted": true, "answer": "FACT-150-saher", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-797as", "id": "mem_80ad877e72", "inTail": false, "evicted": true, "answer": "FACT-165-797as", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-f9so7", "id": "mem_43c1cd03ee", "inTail": false, "evicted": true, "answer": "FACT-180-f9so7", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-u25n2", "id": "mem_cca80e53c9", "inTail": false, "evicted": true, "answer": "FACT-195-u25n2", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-kxe32", "id": "mem_5e0a884190", "inTail": false, "evicted": true, "answer": "FACT-210-kxe32", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-52qt3", "id": "mem_47c66e92db", "inTail": false, "evicted": true, "answer": "FACT-225-52qt3", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-o77fl", "id": "mem_c34265a388", "inTail": false, "evicted": true, "answer": "FACT-240-o77fl", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-d78oa", "id": "mem_54ef1e20dd", "inTail": false, "evicted": true, "answer": "FACT-255-d78oa", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-yz7lh", "id": "mem_43171ae47d", "inTail": false, "evicted": true, "answer": "FACT-270-yz7lh", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-1esyu", "id": "mem_625117fa7a", "inTail": true, "evicted": false, "answer": "FACT-285-1esyu", "correct": true } ], "correct": 20, "total": 20 } }, { "seed": 3, "turns": 300, "tailCount": 20, "tailIds": [ "mem_6224ac65bf", "mem_6978c1007e", "mem_d40651cb87", "mem_4839a143a6", "mem_a1f6a9569e", "mem_b10114df14", "mem_b1ceb19480", "mem_c43e567a99", "mem_43b2a212cb", "mem_851b8f998c", "mem_e871e903ef", "mem_ec53df5659", "mem_26f73cdff9", "mem_4e8d2c2a81", "mem_5cfc1165f8", "mem_aecc1fc7f1", "mem_ef0ccfa87e", "mem_b551a84985", "mem_cab7f4166a", "mem_e236801864" ], "probes": 20, "evictedProbes": 19, "control": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_35d006a5d4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_5292d66f83", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_cba70b104b", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_d0bb469721", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_9b12de3924", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_761b07af02", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_ac22f5b571", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_d005a27b26", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_9564d2d603", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_14160364fe", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_3e69bb45c2", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_dd1589e886", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_a698bcbc46", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_2103dd127a", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_e759eaf3ce", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_be99cff6a4", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_25e6b6ea17", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_2a85e8facb", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_87dc617eb7", "inTail": false, "evicted": true, "answer": "UNKNOWN", "correct": false }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_a1f6a9569e", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 1, "total": 20 }, "treatment": { "rows": [ { "index": 0, "turn": 0, "topic": "gateway", "expected": "FACT-000-pxewf", "id": "mem_35d006a5d4", "inTail": false, "evicted": true, "answer": "FACT-000-pxewf", "correct": true }, { "index": 1, "turn": 15, "topic": "router", "expected": "FACT-015-1e3tl", "id": "mem_5292d66f83", "inTail": false, "evicted": true, "answer": "FACT-015-1e3tl", "correct": true }, { "index": 2, "turn": 30, "topic": "gateway", "expected": "FACT-030-gf83p", "id": "mem_cba70b104b", "inTail": false, "evicted": true, "answer": "FACT-030-gf83p", "correct": true }, { "index": 3, "turn": 45, "topic": "router", "expected": "FACT-045-2p3ua", "id": "mem_d0bb469721", "inTail": false, "evicted": true, "answer": "FACT-045-2p3ua", "correct": true }, { "index": 4, "turn": 60, "topic": "gateway", "expected": "FACT-060-rgus2", "id": "mem_9b12de3924", "inTail": false, "evicted": true, "answer": "FACT-060-rgus2", "correct": true }, { "index": 5, "turn": 75, "topic": "router", "expected": "FACT-075-hblyb", "id": "mem_761b07af02", "inTail": false, "evicted": true, "answer": "FACT-075-hblyb", "correct": true }, { "index": 6, "turn": 90, "topic": "gateway", "expected": "FACT-090-hisus", "id": "mem_ac22f5b571", "inTail": false, "evicted": true, "answer": "FACT-090-hisus", "correct": true }, { "index": 7, "turn": 105, "topic": "router", "expected": "FACT-105-5toyt", "id": "mem_d005a27b26", "inTail": false, "evicted": true, "answer": "FACT-105-5toyt", "correct": true }, { "index": 8, "turn": 120, "topic": "gateway", "expected": "FACT-120-9h6hf", "id": "mem_9564d2d603", "inTail": false, "evicted": true, "answer": "FACT-120-9h6hf", "correct": true }, { "index": 9, "turn": 135, "topic": "router", "expected": "FACT-135-jw6fx", "id": "mem_14160364fe", "inTail": false, "evicted": true, "answer": "FACT-135-jw6fx", "correct": true }, { "index": 10, "turn": 150, "topic": "gateway", "expected": "FACT-150-70m0g", "id": "mem_3e69bb45c2", "inTail": false, "evicted": true, "answer": "FACT-150-70m0g", "correct": true }, { "index": 11, "turn": 165, "topic": "router", "expected": "FACT-165-30c3x", "id": "mem_dd1589e886", "inTail": false, "evicted": true, "answer": "FACT-165-30c3x", "correct": true }, { "index": 12, "turn": 180, "topic": "gateway", "expected": "FACT-180-pp2gq", "id": "mem_a698bcbc46", "inTail": false, "evicted": true, "answer": "FACT-180-pp2gq", "correct": true }, { "index": 13, "turn": 195, "topic": "router", "expected": "FACT-195-h5u81", "id": "mem_2103dd127a", "inTail": false, "evicted": true, "answer": "FACT-195-h5u81", "correct": true }, { "index": 14, "turn": 210, "topic": "gateway", "expected": "FACT-210-skod2", "id": "mem_e759eaf3ce", "inTail": false, "evicted": true, "answer": "FACT-210-skod2", "correct": true }, { "index": 15, "turn": 225, "topic": "router", "expected": "FACT-225-vo6pc", "id": "mem_be99cff6a4", "inTail": false, "evicted": true, "answer": "FACT-225-vo6pc", "correct": true }, { "index": 16, "turn": 240, "topic": "gateway", "expected": "FACT-240-bnezh", "id": "mem_25e6b6ea17", "inTail": false, "evicted": true, "answer": "FACT-240-bnezh", "correct": true }, { "index": 17, "turn": 255, "topic": "router", "expected": "FACT-255-ouzrk", "id": "mem_2a85e8facb", "inTail": false, "evicted": true, "answer": "FACT-255-ouzrk", "correct": true }, { "index": 18, "turn": 270, "topic": "gateway", "expected": "FACT-270-abmgy", "id": "mem_87dc617eb7", "inTail": false, "evicted": true, "answer": "FACT-270-abmgy", "correct": true }, { "index": 19, "turn": 285, "topic": "router", "expected": "FACT-285-o73dt", "id": "mem_a1f6a9569e", "inTail": true, "evicted": false, "answer": "FACT-285-o73dt", "correct": true } ], "correct": 20, "total": 20 } } ], "aggregate": { "control": { "correct": 3, "n": 60, "accuracy": 0.05 }, "treatment": { "correct": 60, "n": 60, "accuracy": 1 }, "evictedControl": { "correct": 0, "n": 57, "accuracy": 0 }, "evictedTreatment": { "correct": 57, "n": 57, "accuracy": 1 } }, "usage": { "promptTokens": 39015, "completionTokens": 1648, "totalTokens": 40663, "calls": 6 }, "raw": [ { "seed": 1, "label": "control", "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-rm66w", "usage": { "prompt_tokens": 1199, "completion_tokens": 244, "total_tokens": 1443, "usage_semantic": "openai", "usage_source": "anthropic", "prompt_tokens_details": { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0 }, "completion_tokens_details": { "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0, "reasoning_tokens": 0 }, "input_tokens": 1199, "output_tokens": 0, "input_tokens_details": null, "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0 } }, { "seed": 1, "label": "treatment", "textTail": "Q0: FACT-000-mkorf\nQ1: FACT-015-03jmy\nQ2: FACT-030-izkkh\nQ3: FACT-045-zbfww\nQ4: FACT-060-yv0n0\nQ5: FACT-075-a4b5x\nQ6: FACT-090-m28ly\nQ7: FACT-105-py2zp\nQ8: FACT-120-fbtzd\nQ9: FACT-135-ztagj\nQ10: FACT-150-ge14o\nQ11: FACT-165-hlgwo\nQ12: FACT-180-50232\nQ13: FACT-195-ej946\nQ14: FACT-210-8wsh3\nQ15: FACT-225-5k815\nQ16: FACT-240-hlxhw\nQ17: FACT-255-2fb7j\nQ18: FACT-270-e7fcx\nQ19: FACT-285-rm66w", "usage": { "prompt_tokens": 11812, "completion_tokens": 305, "total_tokens": 12117, "usage_semantic": "openai", "usage_source": "anthropic", "prompt_tokens_details": { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0 }, "completion_tokens_details": { "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0, "reasoning_tokens": 0 }, "input_tokens": 11812, "output_tokens": 0, "input_tokens_details": null, "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0 } }, { "seed": 2, "label": "control", "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-1esyu", "usage": { "prompt_tokens": 1189, "completion_tokens": 244, "total_tokens": 1433, "usage_semantic": "openai", "usage_source": "anthropic", "prompt_tokens_details": { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0 }, "completion_tokens_details": { "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0, "reasoning_tokens": 0 }, "input_tokens": 1189, "output_tokens": 0, "input_tokens_details": null, "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0 } }, { "seed": 2, "label": "treatment", "textTail": "Q0: FACT-000-qfl7m\nQ1: FACT-015-bp74k\nQ2: FACT-030-a9qrt\nQ3: FACT-045-jd6u3\nQ4: FACT-060-vidfn\nQ5: FACT-075-mpk5r\nQ6: FACT-090-hyyqy\nQ7: FACT-105-cuzlu\nQ8: FACT-120-s4xt8\nQ9: FACT-135-43s9i\nQ10: FACT-150-saher\nQ11: FACT-165-797as\nQ12: FACT-180-f9so7\nQ13: FACT-195-u25n2\nQ14: FACT-210-kxe32\nQ15: FACT-225-52qt3\nQ16: FACT-240-o77fl\nQ17: FACT-255-d78oa\nQ18: FACT-270-yz7lh\nQ19: FACT-285-1esyu", "usage": { "prompt_tokens": 11792, "completion_tokens": 305, "total_tokens": 12097, "usage_semantic": "openai", "usage_source": "anthropic", "prompt_tokens_details": { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0 }, "completion_tokens_details": { "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0, "reasoning_tokens": 0 }, "input_tokens": 11792, "output_tokens": 0, "input_tokens_details": null, "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0 } }, { "seed": 3, "label": "control", "textTail": "Q0: UNKNOWN\nQ1: UNKNOWN\nQ2: UNKNOWN\nQ3: UNKNOWN\nQ4: UNKNOWN\nQ5: UNKNOWN\nQ6: UNKNOWN\nQ7: UNKNOWN\nQ8: UNKNOWN\nQ9: UNKNOWN\nQ10: UNKNOWN\nQ11: UNKNOWN\nQ12: UNKNOWN\nQ13: UNKNOWN\nQ14: UNKNOWN\nQ15: UNKNOWN\nQ16: UNKNOWN\nQ17: UNKNOWN\nQ18: UNKNOWN\nQ19: FACT-285-o73dt", "usage": { "prompt_tokens": 1194, "completion_tokens": 244, "total_tokens": 1438, "usage_semantic": "openai", "usage_source": "anthropic", "prompt_tokens_details": { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0 }, "completion_tokens_details": { "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0, "reasoning_tokens": 0 }, "input_tokens": 1194, "output_tokens": 0, "input_tokens_details": null, "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0 } }, { "seed": 3, "label": "treatment", "textTail": "Q0: FACT-000-pxewf\nQ1: FACT-015-1e3tl\nQ2: FACT-030-gf83p\nQ3: FACT-045-2p3ua\nQ4: FACT-060-rgus2\nQ5: FACT-075-hblyb\nQ6: FACT-090-hisus\nQ7: FACT-105-5toyt\nQ8: FACT-120-9h6hf\nQ9: FACT-135-jw6fx\nQ10: FACT-150-70m0g\nQ11: FACT-165-30c3x\nQ12: FACT-180-pp2gq\nQ13: FACT-195-h5u81\nQ14: FACT-210-skod2\nQ15: FACT-225-vo6pc\nQ16: FACT-240-bnezh\nQ17: FACT-255-ouzrk\nQ18: FACT-270-abmgy\nQ19: FACT-285-o73dt", "usage": { "prompt_tokens": 11829, "completion_tokens": 306, "total_tokens": 12135, "usage_semantic": "openai", "usage_source": "anthropic", "prompt_tokens_details": { "cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0 }, "completion_tokens_details": { "text_tokens": 0, "audio_tokens": 0, "image_tokens": 0, "reasoning_tokens": 0 }, "input_tokens": 11829, "output_tokens": 0, "input_tokens_details": null, "claude_cache_creation_5_m_tokens": 0, "claude_cache_creation_1_h_tokens": 0 } } ], "caveats": [ "Control/treatment prompts use synthetic fixtures; no private data is sent.", "Batch prompts ask the model to answer only from the supplied context; this is evidence-in-prompt, not a tool-calling loop.", "Usage comes from the gateway response; verify with your own APIMaster dashboard if needed.", "Tail membership is saved explicitly; evicted subset is verified.", "Synthetic fixture and evidence-in-prompt; no private prompts or keys are stored." ], "contributedBy": "DSH APIMaster Claude run", "runBy": "local APIMaster OpenAI-compatible gateway", "verifiedTail": true, "seedsRun": "3 seeds x 20 probes" }