{
 "generated": "2026-09-09",
 "suites": [
  {
   "id": "synthesis",
   "title": "Turning tool results into an answer",
   "version": "synthesis/2",
   "question": "Can a model synthesise a set of tool results into a usable, attributed answer?",
   "method": "A question plus results, each carrying a REAL address: a turn id, an OKF doc slug, or an fcbench: case reference. Six arms vary only the result set - sufficient, distractor, conflicting, insufficient, stale, and single_source (one external document, no corroboration).",
   "grading": "Mechanical, no judge model. The answer must carry the right value AND cite a valid address in the real grammar; an invented address fails on the grammar rather than on opinion. Conflicts must be flagged with both addresses, absent answers declared, and single-source claims qualified - neither asserted bare nor refused.",
   "caveat": "Every model tested scores 0/10 on single_source, all overclaiming, despite an explicit instruction. That arm currently measures a capability nothing has rather than separating models.",
   "cases": 60,
   "runs": 4
  },
  {
   "id": "screening",
   "title": "Second-pass screening",
   "version": "screening/1",
   "question": "Can a second model catch the errors a grounding filter lets through?",
   "method": "A record and a candidate extraction are shown to a judge model. Some candidates are correct, some are values lifted from the wrong part of the document, some are invented. The judge marks each one accept or reject.",
   "grading": "Binary. A case passes when the judge's verdict matches the label.",
   "caveat": "This suite is close to saturated. Several models now score above 95 and their confidence intervals overlap, so it no longer separates the top of the field.",
   "cases": 60,
   "runs": 33
  },
  {
   "id": "grounding",
   "title": "Grounding and output mode",
   "version": "grounding/1",
   "question": "Does the mitigation work, and what does each output mode cost?",
   "method": "The same question is asked of the same document under different output modes, including free text and constrained decoding. Documents carry line addresses so any claim can be traced back to a source line.",
   "grading": "A case passes when the answer is present in the document and correctly attributed. Answers taken from the wrong line count as misattribution, not as fabrication.",
   "caveat": "Requires a server that honours response_format. Results from mlx_lm are not valid because it ignores the field.",
   "cases": 125,
   "runs": 38
  },
  {
   "id": "nullschema",
   "title": "Schema shape and abstention",
   "version": "nullschema/1",
   "question": "Is it the schema that causes fabrication, or is it forbidding absence?",
   "method": "Three arms run over the same documents. In forbid, the field is required and non-nullable. In permit, the type union includes null. In optional, the key may be omitted. Only cases whose true answer is absent are used, because a case with a present answer cannot separate the arms.",
   "grading": "A case passes when the model declines, either by returning null or by omitting the key. Returning any value counts as a failure and is further classified as misattribution or fabrication.",
   "caveat": "The three arms must differ. If they come out identical, the server ignored the schema and the run is void.",
   "cases": 72,
   "runs": 29
  },
  {
   "id": "tagger",
   "title": "Tagger contract compliance",
   "version": "tagger/2",
   "question": "Does the model produce tags that satisfy the contract?",
   "method": "240 conversation turn pairs are tagged for category, topics and entities. The category must come from a fixed list, topics are bounded in number, and every entity must appear in the turn.",
   "grading": "Rule based. Category outside the list, topic count out of range, or an entity absent from the source all fail the case.",
   "caveat": "Contract compliance is not the same as tag quality. A human check of the gold pairs is still outstanding.",
   "cases": 480,
   "runs": 12
  },
  {
   "id": "tagger_nullable",
   "title": "Tagger with absence permitted",
   "version": "tagger-nullable/1",
   "question": "Does allowing empty fields change tagging behaviour?",
   "method": "The same 240 turn pairs and the same grader as the tagger suite, with the schema permitting empty topic and entity lists.",
   "grading": "Identical to the tagger suite.",
   "caveat": "Paired with tagger by design. The two should be read together, never separately.",
   "cases": 240,
   "runs": 7
  },
  {
   "id": "epistemics",
   "title": "Given versus prior knowledge",
   "version": "epistemics/4",
   "question": "Does the model answer from the document it was given, or from what it already knows?",
   "method": "Documents contain facts that contradict widely known information. The model is asked questions whose answers differ depending on which source it trusts.",
   "grading": "A case passes when the answer matches the supplied document rather than general knowledge.",
   "caveat": "Several models now score 100 on this suite, so it has stopped discriminating.",
   "cases": 48,
   "runs": 6
  },
  {
   "id": "enrichment",
   "title": "Memory layer enrichment",
   "version": "enrichment/3",
   "question": "Can a local model do the enrichment work the memory layer needs?",
   "method": "Turns are enriched with structured context. The suite measures whether the enrichment is usable downstream rather than whether it reads well.",
   "grading": "Field level checks against the expected structure.",
   "caveat": "Small case count. Treat single runs as indicative only.",
   "cases": 54,
   "runs": 4
  },
  {
   "id": "tools",
   "title": "Tool and skill selection",
   "version": "tools/1",
   "question": "Given a request and a set of tools, does the model pick the right one and chain correctly?",
   "method": "The model is offered tool definitions and asked to complete a task. Chaining matters because entities have to survive from one call to the next.",
   "grading": "A case passes when the correct tool is selected and the arguments carry the right entities.",
   "caveat": "Superseded for discrimination by toolscale, which varies difficulty instead of holding it fixed.",
   "cases": 15,
   "runs": 9
  },
  {
   "id": "toolscale",
   "title": "Tool selection as a staircase",
   "version": "toolscale/1",
   "question": "How many tools can the model choose between before it picks the wrong one?",
   "method": "The number of available tools increases step by step. The task stays the same. The suite records the rung at which selection breaks down.",
   "grading": "Scored by the highest rung cleared, not by a fixed pass rate.",
   "caveat": "Built to avoid saturation. A model that clears every rung tells you the staircase needs more steps, not that the model is finished.",
   "cases": 40,
   "runs": 7
  },
  {
   "id": "chainwalk",
   "title": "Long horizon chaining",
   "version": "chainwalk/3",
   "question": "How deep can the model chain tool calls before it loses its place?",
   "method": "Each step appends a call and its result to the history. By step 200 the model is reading a long transcript of its own work. The suite records how far it gets and whether it deviates.",
   "grading": "Depth reached against depth targeted, plus a check on arithmetic carried across steps.",
   "caveat": "Chaining and arithmetic separate cleanly. Models walk 200 steps without deviating and still get the running total wrong.",
   "cases": 2,
   "runs": 16
  },
  {
   "id": "contract",
   "title": "Following a convention it has to go and get",
   "version": "contract/2",
   "question": "Will a model follow an output convention it can only obtain by asking for it?",
   "method": "Four arms over the same extraction task, where the answer is always absent from the document. In silent, nothing is said. In told, the convention is stated in the system prompt. In fetch, a tool is offered that returns the convention and the prompt points at it. In unmentioned, the same tool is present but nothing refers to it. The convention is an arbitrary sentinel string, so it cannot be recalled from training.",
   "grading": "Binary. A case passes only when the field carries the exact sentinel. Abstaining in the model's own words fails, and is classified separately from inventing a value.",
   "caveat": "A grader bug conflated a missing field with a present null field until 2026-09-09, mislabelling every null abstention as unparseable output. Pass rates were unaffected but failure-mode breakdowns before that date are wrong.",
   "cases": 48,
   "runs": 9
  },
  {
   "id": "toolcall",
   "title": "What makes a model reach for a tool",
   "version": "toolcall/1",
   "question": "Is a model unable to call tools, or does it not perceive a need to?",
   "method": "Five arms, identical tool machinery throughout, varying only the framing. Availability states a tool exists. Conditional puts an if-then rule in the system prompt. Descriptive puts the same rule in the tool description. Need makes the tool hold a value the model cannot otherwise produce. Unmentioned offers the tool with no reference to it anywhere.",
   "grading": "Scored on invocation, not on the answer. A model that calls the tool and then misuses the result fails contract, not this suite.",
   "caveat": "Results move more with the serving runtime than with the model. The same weights scored 78.3 on one server and 35.0 on another, so any comparison must hold the runtime fixed.",
   "cases": 60,
   "runs": 8
  },
  {
   "id": "shapegen",
   "title": "Build to an unseen contract",
   "version": "shapegen/2",
   "question": "Can a model write a component to a composition contract it has never seen?",
   "method": "The model is given an interface, a set of primitives, theme tokens and one site profile. Profiles differ in their composition rules, so the same brief has different correct answers depending on the profile supplied.",
   "grading": "Mechanical. The output is parsed and checked against the profile's rules. No judge model is involved.",
   "caveat": "The methodology is not in any training data, so a model can only pass by following rules given in context. Currently confounded by whether the model knows @scope syntax at all.",
   "cases": 12,
   "runs": 1
  },
  {
   "id": "sitebuild",
   "title": "Agentic coding harness",
   "version": "sitebuild/1",
   "question": "Can a model build a working site in a real project with tools and a build it can run?",
   "method": "The model gets an Astro project, benchmark data, and tools to list, read, write and build. It works until the build passes and the shapeset is accurate, or until its step budget runs out.",
   "grading": "The final state of the filesystem. The build must pass, the shapeset must describe what actually exists, and the pages must render real data.",
   "caveat": "A model that does nothing scored 0.91 before deliverable checks were added. Conformance rules pass vacuously when nothing was built.",
   "cases": 1,
   "runs": 1
  }
 ]
}