{
 "run": "ollama-cloud-gauntlet-20261017",
 "date": "2026-10-17",
 "hardware": "Ollama cloud (remote inference); harness on Mac Studio M1 Max",
 "backend": "Ollama 0.12.6, OpenAI-compatible /v1/chat/completions, cloud models",
 "profile": "extraction (temperature 0, max_tokens 2048, thinking=False requested)",
 "caveats": [
  "thinking=False was NOT applied: Ollama /v1 drops the llama.cpp thinking-off fields the harness sent. gpt-oss:120b-cloud reasoned on every request (~470 output tokens/case), so its numbers are thinking-ON. gemma4:31b-cloud and mistral-large-3:675b-cloud do not think by default on Ollama, so theirs are thinking-off as intended.",
  "grounding and nullschema not run: Ollama ignores response_format json_schema (honours_schema() = False), and the runner refuses them by design.",
  "shapegen (DatasetDrift, SUITE_VERSION not bumped) and sitebuild (KeyError 'nesting') fail in the harness for every model; not scored.",
  "gemma4:31b-cloud enrichment crashed the grader on a JSON-array reply at case 24; not scored.",
  "No /props on Ollama: server_runtime is empty, so every Ollama run shares a configuration_id shape; not one-to-one comparable with llama.cpp seat runs.",
  "synthesis/2 string matching misses Unicode punctuation gpt-oss emits (U+2011, U+202F, fullwidth brackets); an offline re-grade with those normalised scores gpt-oss 43/60 vs 33/60 as graded. The graded number is what is recorded here.",
  "chainwalk has 2 cases; treat as uninformative.",
  "Speeds include Ollama cloud's own hardware and load, which were not controlled."
 ],
 "not_scored": {
  "deepseek-v4.1-flash:cloud": "abandoned after 39 cases: thinks 1.4-1.9k tokens/case and ran to the full 8192 cap on 7/39 even under the synthesis profile",
  "glm-5.3-flash:cloud": "abandoned after 101 cases: thinking-on run judged the wrong configuration for agentic suites",
  "beam:cloud": "6-case trial only: always thinks; no off switch on Ollama"
 },
 "followup": "Harness PR #5 (native Ollama connector + thinking-off precondition on every backend).",
 "summary": {
  "gpt-oss:120b-cloud": {
   "tagger": {
    "rows": 240,
    "measured": 240,
    "passed": 210,
    "accuracy": 0.875,
    "ci95": [
     82.7,
     91.1
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 2,
    "tokens_in": 118788,
    "tokens_out": 131135,
    "median_s_per_case": 1.92,
    "reasons": {
     "ok": 238,
     "hit-task-budget": 2
    },
    "suite_version": "tagger/2",
    "dataset_id": "7ef587bf33e5788ac8eb405e52495c6898271491e4611286f3c41f66150e119d",
    "grader_id": "63e04d139a0e5ed3",
    "configuration_id": "330bfddc0a79f92d",
    "complete": true
   },
   "tagger_nullable": {
    "rows": 240,
    "measured": 240,
    "passed": 232,
    "accuracy": 0.9667,
    "ci95": [
     93.6,
     98.3
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 127908,
    "tokens_out": 123077,
    "median_s_per_case": 1.88,
    "reasons": {
     "ok": 240
    },
    "suite_version": "tagger-nullable/1",
    "dataset_id": "db3addf23a2bc4ce4c5329bff23b79b7ecea8ba924312a752b771711fec3ac6e",
    "grader_id": "04e875431e26c4b5",
    "configuration_id": "11aa3e9adaa97f67",
    "complete": true
   },
   "needle": {
    "rows": 110,
    "measured": 110,
    "passed": 71,
    "accuracy": 0.6455,
    "ci95": [
     55.3,
     72.9
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 3,
    "tokens_in": 2261960,
    "tokens_out": 26731,
    "median_s_per_case": 1.18,
    "reasons": {
     "ok": 107,
     "hit-task-budget": 3
    },
    "suite_version": "needle/1",
    "dataset_id": "5c5199b9eee8a595c6cb042a6499235567e577b397f9cc9019c648ac8d6d8b7b",
    "grader_id": "c38dec33aa1c86c2",
    "configuration_id": "5c1878c923409293",
    "complete": true
   },
   "enrichment": {
    "rows": 54,
    "measured": 54,
    "passed": 44,
    "accuracy": 0.8148,
    "ci95": [
     69.2,
     89.6
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 13876,
    "tokens_out": 16289,
    "median_s_per_case": 0.95,
    "reasons": {
     "ok": 54
    },
    "suite_version": "enrichment/3",
    "dataset_id": "75361864baa727abe654f0e0f5496d69771dfbe5f6480a3e9e80d2459189f75f",
    "grader_id": "b51222c2ca38cfdf",
    "configuration_id": "e9735bd2b8138f6c",
    "complete": true
   },
   "epistemics": {
    "rows": 48,
    "measured": 48,
    "passed": 47,
    "accuracy": 0.9792,
    "ci95": [
     89.1,
     99.6
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 22262,
    "tokens_out": 5146,
    "median_s_per_case": 0.66,
    "reasons": {
     "ok": 48
    },
    "suite_version": "epistemics/4",
    "dataset_id": "4bce453fd376553c768cca67db1d27f08893309a1270c3f1e66738747ac18f32",
    "grader_id": "acf84972a0ad5f38",
    "configuration_id": "57631b3b10398597",
    "complete": true
   },
   "synthesis": {
    "rows": 60,
    "measured": 60,
    "passed": 33,
    "accuracy": 0.55,
    "ci95": [
     42.5,
     66.9
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 19670,
    "tokens_out": 6751,
    "median_s_per_case": 0.64,
    "reasons": {
     "ok": 60
    },
    "suite_version": "synthesis/2",
    "dataset_id": "267d1954e225c5b63c99367e900464f5d6fb9cb19e6654dba710fed8bcb8779c",
    "grader_id": "4156dd3c13361e8b",
    "configuration_id": "8467fbaac1b54daa",
    "complete": true
   },
   "worldview": {
    "rows": 40,
    "measured": 40,
    "passed": 21,
    "accuracy": 0.525,
    "ci95": [
     37.5,
     67.1
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 11893,
    "tokens_out": 7959,
    "median_s_per_case": 0.76,
    "reasons": {
     "ok": 40
    },
    "suite_version": "worldview/2",
    "dataset_id": "e97835de9e6fc227447915fb524b8e8059b094d9a411885ecfc2455ea7cc001a",
    "grader_id": "9acfba1b704feb28",
    "configuration_id": "c29cd431efdc9567",
    "complete": true
   },
   "toolcall": {
    "rows": 60,
    "measured": 60,
    "passed": 38,
    "accuracy": 0.6333,
    "ci95": [
     50.7,
     74.4
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 13546,
    "tokens_out": 8759,
    "median_s_per_case": 0.72,
    "reasons": {
     "ok": 60
    },
    "suite_version": "toolcall/1",
    "dataset_id": "10af88c170513155a775a8941ef522dfad1fbfc19693f7757a20392d3d6587e2",
    "grader_id": "804e8fc9457649d8",
    "configuration_id": "6a9811c5a2f9c488",
    "complete": true
   },
   "toolscale": {
    "rows": 40,
    "measured": 40,
    "passed": 35,
    "accuracy": 0.875,
    "ci95": [
     73.9,
     94.5
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 33826,
    "tokens_out": 2069,
    "median_s_per_case": 0.5,
    "reasons": {
     "ok": 40
    },
    "suite_version": "toolscale/1",
    "dataset_id": "b24abb108c40917887fbcc7c3d158dc721d4976945a591952aa883f42ec6a2dd",
    "grader_id": "a2bc3751abfa28e0",
    "configuration_id": "7d06d72021a5d93b",
    "complete": true
   },
   "tools": {
    "rows": 15,
    "measured": 15,
    "passed": 13,
    "accuracy": 0.8667,
    "ci95": [
     62.1,
     96.3
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 3290,
    "tokens_out": 980,
    "median_s_per_case": 0.52,
    "reasons": {
     "ok": 15
    },
    "suite_version": "tools/1",
    "dataset_id": "33a44812f0ad4c394857ceac8e6666c156d4f975c0216ff8006ebc68cc7cc6b3",
    "grader_id": "2763675a5c3da564",
    "configuration_id": "dacf6dd451b1dc06",
    "complete": true
   },
   "screening": {
    "rows": 60,
    "measured": 60,
    "passed": 55,
    "accuracy": 0.9167,
    "ci95": [
     81.9,
     96.4
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 16936,
    "tokens_out": 8647,
    "median_s_per_case": 0.6,
    "reasons": {
     "ok": 60
    },
    "suite_version": "screening/1",
    "dataset_id": "64c97ca2269c29afe26cb1fd083655b2b53a2d596c300ae60fd19d4f0fa2bdca",
    "grader_id": "0661b49a10be2d2a",
    "configuration_id": "d9a9610d122c3ca6",
    "complete": true
   },
   "contract": {
    "rows": 48,
    "measured": 48,
    "passed": 24,
    "accuracy": 0.5,
    "ci95": [
     36.4,
     63.6
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 10650,
    "tokens_out": 5516,
    "median_s_per_case": 0.67,
    "reasons": {
     "ok": 48
    },
    "suite_version": "contract/2",
    "dataset_id": "1fe53d30cb5480f6844365a851a35bf098f7b53e5e537f8876639a68651a8eb0",
    "grader_id": "cc2d23c5bae45474",
    "configuration_id": "848ae6dfc6e00979",
    "complete": true
   },
   "chainwalk": {
    "rows": 2,
    "measured": 2,
    "passed": 0,
    "accuracy": 0.0,
    "ci95": [
     0.0,
     65.8
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 446,
    "tokens_out": 38,
    "median_s_per_case": 1.4,
    "reasons": {
     "ok": 2
    },
    "suite_version": "chainwalk/3",
    "dataset_id": "129a9e74408c89194f3cbffeb9c21a39c3ed006a1cccbc77cb5f177bc5bc55b6",
    "grader_id": "f4bfcf6f040b0978",
    "configuration_id": "1dfdaf5e3ebafc87",
    "complete": true
   }
  },
  "gemma4:31b-cloud": {
   "tagger": {
    "rows": 240,
    "measured": 240,
    "passed": 233,
    "accuracy": 0.9708,
    "ci95": [
     94.1,
     98.6
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 113588,
    "tokens_out": 30143,
    "median_s_per_case": 0.66,
    "reasons": {
     "ok": 240
    },
    "suite_version": "tagger/2",
    "dataset_id": "7ef587bf33e5788ac8eb405e52495c6898271491e4611286f3c41f66150e119d",
    "grader_id": "63e04d139a0e5ed3",
    "configuration_id": "60fa55b37e5cfb25",
    "complete": true
   },
   "tagger_nullable": {
    "rows": 240,
    "measured": 240,
    "passed": 167,
    "accuracy": 0.6958,
    "ci95": [
     63.5,
     75.1
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 122228,
    "tokens_out": 26193,
    "median_s_per_case": 0.61,
    "reasons": {
     "ok": 240
    },
    "suite_version": "tagger-nullable/1",
    "dataset_id": "db3addf23a2bc4ce4c5329bff23b79b7ecea8ba924312a752b771711fec3ac6e",
    "grader_id": "04e875431e26c4b5",
    "configuration_id": "7f4c0a07d4d0a94a",
    "complete": true
   },
   "needle": {
    "rows": 110,
    "measured": 110,
    "passed": 69,
    "accuracy": 0.6273,
    "ci95": [
     53.4,
     71.2
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 2513273,
    "tokens_out": 2900,
    "median_s_per_case": 0.92,
    "reasons": {
     "ok": 110
    },
    "suite_version": "needle/1",
    "dataset_id": "5c5199b9eee8a595c6cb042a6499235567e577b397f9cc9019c648ac8d6d8b7b",
    "grader_id": "c38dec33aa1c86c2",
    "configuration_id": "17dd266c3e6b4c74",
    "complete": true
   },
   "enrichment": {
    "rows": 24,
    "measured": 24,
    "passed": 23,
    "accuracy": null,
    "ci95": null,
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 2673,
    "tokens_out": 266,
    "median_s_per_case": 0.42,
    "reasons": {
     "ok": 24
    },
    "suite_version": "enrichment/3",
    "dataset_id": "75361864baa727abe654f0e0f5496d69771dfbe5f6480a3e9e80d2459189f75f",
    "grader_id": "b51222c2ca38cfdf",
    "configuration_id": "7357a406813d47b9",
    "complete": false,
    "note": "incomplete: 24 of 54 cases before the run stopped; not scored"
   },
   "epistemics": {
    "rows": 48,
    "measured": 48,
    "passed": 48,
    "accuracy": 1.0,
    "ci95": [
     92.6,
     100.0
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 20931,
    "tokens_out": 726,
    "median_s_per_case": 0.41,
    "reasons": {
     "ok": 48
    },
    "suite_version": "epistemics/4",
    "dataset_id": "4bce453fd376553c768cca67db1d27f08893309a1270c3f1e66738747ac18f32",
    "grader_id": "acf84972a0ad5f38",
    "configuration_id": "5f5bb8f30fc77223",
    "complete": true
   },
   "synthesis": {
    "rows": 60,
    "measured": 60,
    "passed": 55,
    "accuracy": 0.9167,
    "ci95": [
     81.9,
     96.4
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 18843,
    "tokens_out": 4085,
    "median_s_per_case": 0.56,
    "reasons": {
     "ok": 60
    },
    "suite_version": "synthesis/2",
    "dataset_id": "267d1954e225c5b63c99367e900464f5d6fb9cb19e6654dba710fed8bcb8779c",
    "grader_id": "4156dd3c13361e8b",
    "configuration_id": "196f5a66899b3d83",
    "complete": true
   },
   "worldview": {
    "rows": 40,
    "measured": 40,
    "passed": 40,
    "accuracy": 1.0,
    "ci95": [
     91.2,
     100.0
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 10538,
    "tokens_out": 1742,
    "median_s_per_case": 0.51,
    "reasons": {
     "ok": 40
    },
    "suite_version": "worldview/2",
    "dataset_id": "e97835de9e6fc227447915fb524b8e8059b094d9a411885ecfc2455ea7cc001a",
    "grader_id": "9acfba1b704feb28",
    "configuration_id": "4195fe91ad38863e",
    "complete": true
   },
   "toolcall": {
    "rows": 60,
    "measured": 60,
    "passed": 20,
    "accuracy": 0.3333,
    "ci95": [
     22.7,
     45.9
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 9634,
    "tokens_out": 673,
    "median_s_per_case": 0.41,
    "reasons": {
     "ok": 60
    },
    "suite_version": "toolcall/1",
    "dataset_id": "10af88c170513155a775a8941ef522dfad1fbfc19693f7757a20392d3d6587e2",
    "grader_id": "804e8fc9457649d8",
    "configuration_id": "1f745b9357efd551",
    "complete": true
   },
   "toolscale": {
    "rows": 40,
    "measured": 40,
    "passed": 40,
    "accuracy": 1.0,
    "ci95": [
     91.2,
     100.0
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 53806,
    "tokens_out": 770,
    "median_s_per_case": 0.47,
    "reasons": {
     "ok": 40
    },
    "suite_version": "toolscale/1",
    "dataset_id": "b24abb108c40917887fbcc7c3d158dc721d4976945a591952aa883f42ec6a2dd",
    "grader_id": "a2bc3751abfa28e0",
    "configuration_id": "6af04302d2f3b7a3",
    "complete": true
   },
   "tools": {
    "rows": 15,
    "measured": 15,
    "passed": 14,
    "accuracy": 0.9333,
    "ci95": [
     70.2,
     98.8
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 3288,
    "tokens_out": 235,
    "median_s_per_case": 0.43,
    "reasons": {
     "ok": 15
    },
    "suite_version": "tools/1",
    "dataset_id": "33a44812f0ad4c394857ceac8e6666c156d4f975c0216ff8006ebc68cc7cc6b3",
    "grader_id": "2763675a5c3da564",
    "configuration_id": "5b34f4b1f6ecab13",
    "complete": true
   },
   "screening": {
    "rows": 60,
    "measured": 60,
    "passed": 55,
    "accuracy": 0.9167,
    "ci95": [
     81.9,
     96.4
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 14227,
    "tokens_out": 2539,
    "median_s_per_case": 0.52,
    "reasons": {
     "ok": 60
    },
    "suite_version": "screening/1",
    "dataset_id": "64c97ca2269c29afe26cb1fd083655b2b53a2d596c300ae60fd19d4f0fa2bdca",
    "grader_id": "0661b49a10be2d2a",
    "configuration_id": "fef2f87e06b30b73",
    "complete": true
   },
   "contract": {
    "rows": 48,
    "measured": 48,
    "passed": 12,
    "accuracy": 0.25,
    "ci95": [
     14.9,
     38.8
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 6930,
    "tokens_out": 569,
    "median_s_per_case": 0.41,
    "reasons": {
     "ok": 48
    },
    "suite_version": "contract/2",
    "dataset_id": "1fe53d30cb5480f6844365a851a35bf098f7b53e5e537f8876639a68651a8eb0",
    "grader_id": "cc2d23c5bae45474",
    "configuration_id": "04bae2cdc92cc886",
    "complete": true
   },
   "chainwalk": {
    "rows": 2,
    "measured": 2,
    "passed": 1,
    "accuracy": 0.5,
    "ci95": [
     9.5,
     90.5
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 27859,
    "tokens_out": 26,
    "median_s_per_case": 112.0,
    "reasons": {
     "ok": 2
    },
    "suite_version": "chainwalk/3",
    "dataset_id": "129a9e74408c89194f3cbffeb9c21a39c3ed006a1cccbc77cb5f177bc5bc55b6",
    "grader_id": "f4bfcf6f040b0978",
    "configuration_id": "9b15a49aeeee7970",
    "complete": true
   }
  },
  "mistral-large-3:675b-cloud": {
   "tagger": {
    "rows": 240,
    "measured": 240,
    "passed": 211,
    "accuracy": 0.8792,
    "ci95": [
     83.2,
     91.5
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 108925,
    "tokens_out": 33724,
    "median_s_per_case": 2.23,
    "reasons": {
     "ok": 240
    },
    "suite_version": "tagger/2",
    "dataset_id": "7ef587bf33e5788ac8eb405e52495c6898271491e4611286f3c41f66150e119d",
    "grader_id": "63e04d139a0e5ed3",
    "configuration_id": "b3cf19c8a635d6ec",
    "complete": true
   },
   "tagger_nullable": {
    "rows": 240,
    "measured": 240,
    "passed": 235,
    "accuracy": 0.9792,
    "ci95": [
     95.2,
     99.1
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 117805,
    "tokens_out": 27547,
    "median_s_per_case": 1.94,
    "reasons": {
     "ok": 240
    },
    "suite_version": "tagger-nullable/1",
    "dataset_id": "db3addf23a2bc4ce4c5329bff23b79b7ecea8ba924312a752b771711fec3ac6e",
    "grader_id": "04e875431e26c4b5",
    "configuration_id": "a5036690788ab840",
    "complete": true
   },
   "needle": {
    "rows": 110,
    "measured": 110,
    "passed": 68,
    "accuracy": 0.6182,
    "ci95": [
     52.5,
     70.4
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 2436388,
    "tokens_out": 2639,
    "median_s_per_case": 1.79,
    "reasons": {
     "ok": 110
    },
    "suite_version": "needle/1",
    "dataset_id": "5c5199b9eee8a595c6cb042a6499235567e577b397f9cc9019c648ac8d6d8b7b",
    "grader_id": "c38dec33aa1c86c2",
    "configuration_id": "37f4ecc564d8d0d0",
    "complete": true
   },
   "enrichment": {
    "rows": 54,
    "measured": 54,
    "passed": 41,
    "accuracy": 0.7593,
    "ci95": [
     63.1,
     85.4
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 11604,
    "tokens_out": 4621,
    "median_s_per_case": 1.4,
    "reasons": {
     "ok": 54
    },
    "suite_version": "enrichment/3",
    "dataset_id": "75361864baa727abe654f0e0f5496d69771dfbe5f6480a3e9e80d2459189f75f",
    "grader_id": "b51222c2ca38cfdf",
    "configuration_id": "a0562d30788534eb",
    "complete": true
   },
   "epistemics": {
    "rows": 48,
    "measured": 48,
    "passed": 48,
    "accuracy": 1.0,
    "ci95": [
     92.6,
     100.0
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 20835,
    "tokens_out": 848,
    "median_s_per_case": 0.97,
    "reasons": {
     "ok": 48
    },
    "suite_version": "epistemics/4",
    "dataset_id": "4bce453fd376553c768cca67db1d27f08893309a1270c3f1e66738747ac18f32",
    "grader_id": "acf84972a0ad5f38",
    "configuration_id": "7a08159454b64bd1",
    "complete": true
   },
   "synthesis": {
    "rows": 60,
    "measured": 60,
    "passed": 35,
    "accuracy": 0.5833,
    "ci95": [
     45.7,
     69.9
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 17761,
    "tokens_out": 3363,
    "median_s_per_case": 1.69,
    "reasons": {
     "ok": 60
    },
    "suite_version": "synthesis/2",
    "dataset_id": "267d1954e225c5b63c99367e900464f5d6fb9cb19e6654dba710fed8bcb8779c",
    "grader_id": "4156dd3c13361e8b",
    "configuration_id": "5208d929b728a91a",
    "complete": true
   },
   "worldview": {
    "rows": 40,
    "measured": 40,
    "passed": 31,
    "accuracy": 0.775,
    "ci95": [
     62.5,
     87.7
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 9943,
    "tokens_out": 2153,
    "median_s_per_case": 1.53,
    "reasons": {
     "ok": 40
    },
    "suite_version": "worldview/2",
    "dataset_id": "e97835de9e6fc227447915fb524b8e8059b094d9a411885ecfc2455ea7cc001a",
    "grader_id": "9acfba1b704feb28",
    "configuration_id": "457d1f016ff2b55f",
    "complete": true
   },
   "toolcall": {
    "rows": 60,
    "measured": 60,
    "passed": 38,
    "accuracy": 0.6333,
    "ci95": [
     50.7,
     74.4
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 9869,
    "tokens_out": 400,
    "median_s_per_case": 0.9,
    "reasons": {
     "ok": 60
    },
    "suite_version": "toolcall/1",
    "dataset_id": "10af88c170513155a775a8941ef522dfad1fbfc19693f7757a20392d3d6587e2",
    "grader_id": "804e8fc9457649d8",
    "configuration_id": "291c7e98e855813e",
    "complete": true
   },
   "toolscale": {
    "rows": 40,
    "measured": 40,
    "passed": 39,
    "accuracy": 0.975,
    "ci95": [
     87.1,
     99.6
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 66085,
    "tokens_out": 588,
    "median_s_per_case": 1.17,
    "reasons": {
     "ok": 40
    },
    "suite_version": "toolscale/1",
    "dataset_id": "b24abb108c40917887fbcc7c3d158dc721d4976945a591952aa883f42ec6a2dd",
    "grader_id": "a2bc3751abfa28e0",
    "configuration_id": "63aaa96f398fe806",
    "complete": true
   },
   "tools": {
    "rows": 15,
    "measured": 15,
    "passed": 15,
    "accuracy": 1.0,
    "ci95": [
     79.6,
     100.0
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 3794,
    "tokens_out": 212,
    "median_s_per_case": 1.33,
    "reasons": {
     "ok": 15
    },
    "suite_version": "tools/1",
    "dataset_id": "33a44812f0ad4c394857ceac8e6666c156d4f975c0216ff8006ebc68cc7cc6b3",
    "grader_id": "2763675a5c3da564",
    "configuration_id": "940aa566764272fb",
    "complete": true
   },
   "screening": {
    "rows": 60,
    "measured": 60,
    "passed": 59,
    "accuracy": 0.9833,
    "ci95": [
     91.1,
     99.7
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 13303,
    "tokens_out": 3407,
    "median_s_per_case": 1.65,
    "reasons": {
     "ok": 60
    },
    "suite_version": "screening/1",
    "dataset_id": "64c97ca2269c29afe26cb1fd083655b2b53a2d596c300ae60fd19d4f0fa2bdca",
    "grader_id": "0661b49a10be2d2a",
    "configuration_id": "a7ff261260c9c879",
    "complete": true
   },
   "contract": {
    "rows": 48,
    "measured": 48,
    "passed": 28,
    "accuracy": 0.5833,
    "ci95": [
     44.3,
     71.2
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 7622,
    "tokens_out": 611,
    "median_s_per_case": 1.04,
    "reasons": {
     "ok": 48
    },
    "suite_version": "contract/2",
    "dataset_id": "1fe53d30cb5480f6844365a851a35bf098f7b53e5e537f8876639a68651a8eb0",
    "grader_id": "cc2d23c5bae45474",
    "configuration_id": "41b14cb8922ca62f",
    "complete": true
   },
   "chainwalk": {
    "rows": 2,
    "measured": 2,
    "passed": 0,
    "accuracy": 0.0,
    "ci95": [
     0.0,
     65.8
    ],
    "unmeasured": 0,
    "errors": 0,
    "hit_cap": 0,
    "tokens_in": 0,
    "tokens_out": 0,
    "median_s_per_case": 6.05,
    "reasons": {
     "ok": 2
    },
    "suite_version": "chainwalk/3",
    "dataset_id": "129a9e74408c89194f3cbffeb9c21a39c3ed006a1cccbc77cb5f177bc5bc55b6",
    "grader_id": "f4bfcf6f040b0978",
    "configuration_id": "a5405ce61602993a",
    "complete": true
   }
  }
 }
}