{
  "version": 1,
  "runDate": "2026-10-24",
  "scope": "Offline results only. No paid model calls. Not the release benchmark protocol, which has not run.",
  "environment": {
    "hardware": "Apple M4",
    "node": "v25.3.0",
    "tokenizer": "o200k_base via js-tiktoken 1.0.21",
    "latencyPublished": false,
    "latencyNote": "Other jobs shared the machine during the run, so latency is not published."
  },
  "engine": {
    "name": "SuperCompress Fast",
    "kind": "local, query-anchored extraction",
    "released": false,
    "releaseNote": "Pre-release. Not in supercompress-proxy 0.5.40 on npm."
  },
  "compression": {
    "singleFactSuites": [
      { "suite": "Caveman-style needle suite", "tasks": 180, "fast": { "kept": "180/180", "cutPct": 99.7 }, "cavemanEngine": { "version": "bin-v4.0.1", "keptPct": 94.4, "cutPct": 50.9 } },
      { "suite": "Holdout v2", "tasks": 192, "fast": { "keptPct": 100, "cutPct": 99.2 }, "note": "Used for engine tuning; development data now." },
      { "suite": "Holdout v3", "tasks": 168, "fast": { "keptPct": 100, "cutPct": 99.4, "decoyLeakPct": 0 }, "note": "Used for engine tuning; development data now." }
    ],
    "fourWaySynthetic": {
      "tasks": 108,
      "metric": "single planted answer kept",
      "rows": [
        { "contender": "SuperCompress Fast", "kept": "108/108", "cutPct": 99.7 },
        { "contender": "Caveman engine bin-v4.0.1", "kept": "102/108", "cutPct": 50.6 },
        { "contender": "Headroom 0.37.0 (local profile)", "kept": "90/108", "cutPct": 46.4 },
        { "contender": "LiteLLM 1.103.2 (lexical)", "kept": "69/108", "cutPct": 33.0 }
      ]
    },
    "everyFactCorpus": {
      "cases": 84,
      "metric": "every required fact kept and structure still valid",
      "rows": [
        { "contender": "SuperCompress Fast", "passed": "78/84", "facts": "198/222", "cutPct": 72.9, "invalidStructure": 0 },
        { "contender": "SuperCompress Fast (2026-10-21 engine)", "passed": "84/84", "facts": "222/222", "cutPct": 59.65, "invalidStructure": 0 },
        { "contender": "Headroom (local profile)", "passed": "84/84", "facts": "222/222", "cutPct": 10.5, "invalidStructure": 0 },
        { "contender": "Caveman engine", "passed": "78/84", "facts": "222/222", "cutPct": 18.6, "invalidStructure": 6 },
        { "contender": "LiteLLM (lexical)", "passed": "66/84", "facts": "191/222", "cutPct": 43.3, "invalidStructure": 3 },
        { "contender": "Caveman Cloud (2026-10-21)", "passed": "77/84", "facts": "219/222", "cutPct": 18.58, "invalidStructure": 6, "timeouts": 1 }
      ],
      "knownIssue": "All six Fast failures are the markdown-procedure family: Fast keeps the matching heading and drops the numbered steps under it."
    },
    "realWorldDumps": { "dumps": 12, "fast": { "allGoldsKeptPct": 100, "cutPctTokenWeighted": 88.6 } },
    "semanticHoldout": {
      "cases": 24,
      "rows": [
        { "contender": "SuperCompress Fast", "passed": "17/24", "cutPct": 37.8, "note": "Fails all 3 multilingual cases." },
        { "contender": "Headroom", "passed": "22/24", "cutPct": 3.8 },
        { "contender": "LiteLLM", "passed": "24/24", "cutPct": 0, "note": "Returns input unchanged." },
        { "contender": "Caveman Cloud (2026-10-21)", "passed": "24/24", "cutPct": 0, "note": "Returns input unchanged." }
      ]
    },
    "agentAnswers": {
      "runDate": "2026-10-22",
      "sessions": 72,
      "note": "Isolated agent sessions answered from each arm's context. Single-fact tasks. Engine has changed since; not rerun.",
      "rows": [
        { "arm": "Original", "correct": "24/24", "cutPct": 0 },
        { "arm": "SuperCompress Fast", "correct": "24/24", "cutPct": 99.46 },
        { "arm": "Caveman", "correct": "23/24", "cutPct": 42.32 }
      ]
    },
    "hosted": {
      "status": "not_rerun",
      "note": "A 2026-10-17 run of the hosted API and Neural Keep had timeouts and over-cap rejections. No current hosted quality or latency claim."
    }
  },
  "routing": {
    "launched": false,
    "scope": "Offline tier classification and synthetic selector economics. No requests were sent to models; task success is not measured.",
    "autoRouterExamples": { "turns": 18, "tierMatches": "18/18", "routedTooLow": 0, "listPriceUsd": { "supercompress": 0.81, "autoRouterPicks": 2.17, "allTopModel": 4.59 } },
    "heldOutPrompts": { "test": "56/59", "development": "60/60", "routedTooLow": 0, "note": "Self-written prompts, labeled before the classifier ran." },
    "litellmSelectors": { "litellm": "1.106.0", "requests": 2000, "synthetic": true, "sameGatesUsd": { "supercompress": 24.36, "litellmLowestCost": 24.92, "litellmLowestLatency": 28.43 }, "violations": 0 }
  }
}
