{ "manifest": { "tokenizer": "tiktoken cl100k_base", "seed_longbench": 20260822, "supercompress_v2": { "checkpoint": "/Users/arjunkshah21/Downloads/supercompress/checkpoints/sc-keep-crossencoder-v4-large", "inference": "query-conditioned cross-encoder v4-large" }, "truncation": "keep last 35% chars (matched-budget mode overrides)", "full_context": "identity (no compression)", "headroom": "0.36.4", "llmlingua": "0.2.2", "llmlingua_model": "microsoft/llmlingua-2-xlm-roberta-large-meetingbank", "supercompress_v1": { "engine": "JS compiler + 12-dim MLP (model.json)", "inference": "web/assets/js/compress-engine.js compressAdaptive" } }, "methodology": { "tokenizer": "tiktoken cl100k_base", "quality": "answer/needle/evidence containment in compressed text", "seed_longbench": 20260822, "downstream_llm": "NOT RUN \u2014 set OPENAI_API_KEY for headroom.evals suite --tier 1" }, "summary_all": { "Full": { "n": 442, "passed": 442, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 85.29, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 1602197, "total_tokens_out": 1602197, "downstream_cost_usd_est": 4.005492, "cost_per_successful_est": 4.0054925, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 447, "passed": 447, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 3.61, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 85.46, "max_reduction_at_99pct_quality": 96.6, "max_reduction_at_95pct_quality": 96.6, "mean_latency_ms": 6613.19, "p50_latency_ms": 2014.91, "p95_latency_ms": 20950.77, "total_tokens_in": 1727945, "total_tokens_out": 1715846, "downstream_cost_usd_est": 4.289615, "cost_per_successful_est": 4.289615, "cost_reduction_vs_full_pct": 0.7, "cost_reduction_vs_headroom_pct": -10.93 }, "Headroom": { "n": 447, "passed": 434, "pass_rate_pct": 97.09, "mean_token_reduction_pct": 12.8, "median_token_reduction_pct": 14.17, "p25_reduction": 5.32, "p75_reduction": 22.32, "mean_quality_retention_pct": 83.13, "max_reduction_at_99pct_quality": 38.15, "max_reduction_at_95pct_quality": 38.15, "mean_latency_ms": 3258.83, "p50_latency_ms": 1043.33, "p95_latency_ms": 20254.38, "total_tokens_in": 1727945, "total_tokens_out": 1546811, "downstream_cost_usd_est": 3.867027, "cost_per_successful_est": 3.8670275, "cost_reduction_vs_full_pct": 10.48, "cost_reduction_vs_headroom_pct": 0.0 }, "Truncation": { "n": 442, "passed": 136, "pass_rate_pct": 30.77, "mean_token_reduction_pct": 64.85, "median_token_reduction_pct": 64.94, "p25_reduction": 64.64, "p75_reduction": 65.09, "mean_quality_retention_pct": 33.42, "max_reduction_at_99pct_quality": 70.3, "max_reduction_at_95pct_quality": 70.3, "mean_latency_ms": 0.02, "p50_latency_ms": 0.01, "p95_latency_ms": 0.02, "total_tokens_in": 1602197, "total_tokens_out": 561807, "downstream_cost_usd_est": 1.404517, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.94 }, "LLMLingua-2": { "n": 174, "passed": 162, "pass_rate_pct": 93.1, "mean_token_reduction_pct": 60.2, "median_token_reduction_pct": 61.85, "p25_reduction": 60.0, "p75_reduction": 62.17, "mean_quality_retention_pct": 95.93, "max_reduction_at_99pct_quality": 68.09, "max_reduction_at_95pct_quality": 68.09, "mean_latency_ms": 1894.48, "p50_latency_ms": 1806.7, "p95_latency_ms": 3403.6, "total_tokens_in": 148147, "total_tokens_out": 59255, "downstream_cost_usd_est": 0.148138, "cost_per_successful_est": 0.1481375, "cost_reduction_vs_full_pct": 60.0, "cost_reduction_vs_headroom_pct": 96.17 } }, "summary_coding_agents_b5": { "Full": { "n": 24, "passed": 24, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 16647, "total_tokens_out": 16647, "downstream_cost_usd_est": 0.041617, "cost_per_successful_est": 0.0416175, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 24, "passed": 24, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 64.14, "median_token_reduction_pct": 84.95, "p25_reduction": 40.66, "p75_reduction": 93.61, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 96.6, "max_reduction_at_95pct_quality": 96.6, "mean_latency_ms": 5886.63, "p50_latency_ms": 5780.72, "p95_latency_ms": 10863.51, "total_tokens_in": 16647, "total_tokens_out": 5148, "downstream_cost_usd_est": 0.01287, "cost_per_successful_est": 0.01287, "cost_reduction_vs_full_pct": 69.08 }, "Headroom": { "n": 24, "passed": 18, "pass_rate_pct": 75.0, "mean_token_reduction_pct": 14.76, "median_token_reduction_pct": 15.83, "p25_reduction": 8.21, "p75_reduction": 19.01, "mean_quality_retention_pct": 85.83, "max_reduction_at_99pct_quality": 31.49, "max_reduction_at_95pct_quality": 31.49, "mean_latency_ms": 380.32, "p50_latency_ms": 318.37, "p95_latency_ms": 1059.81, "total_tokens_in": 16647, "total_tokens_out": 14009, "downstream_cost_usd_est": 0.035022, "cost_per_successful_est": 0.0350225, "cost_reduction_vs_full_pct": 15.85 }, "Truncation": { "n": 24, "passed": 11, "pass_rate_pct": 45.83, "mean_token_reduction_pct": 64.31, "median_token_reduction_pct": 64.97, "p25_reduction": 61.95, "p75_reduction": 65.99, "mean_quality_retention_pct": 56.46, "max_reduction_at_99pct_quality": 68.04, "max_reduction_at_95pct_quality": 68.04, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 16647, "total_tokens_out": 5851, "downstream_cost_usd_est": 0.014628, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.85 }, "LLMLingua-2": { "n": 24, "passed": 12, "pass_rate_pct": 50.0, "mean_token_reduction_pct": 52.68, "median_token_reduction_pct": 51.59, "p25_reduction": 48.78, "p75_reduction": 56.55, "mean_quality_retention_pct": 70.49, "max_reduction_at_99pct_quality": 68.09, "max_reduction_at_95pct_quality": 68.09, "mean_latency_ms": 834.69, "p50_latency_ms": 699.88, "p95_latency_ms": 1688.33, "total_tokens_in": 16647, "total_tokens_out": 8030, "downstream_cost_usd_est": 0.020075, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 51.76 }, "SuperCompress-v1": { "n": 24, "passed": 21, "pass_rate_pct": 87.5, "mean_token_reduction_pct": 60.73, "median_token_reduction_pct": 67.97, "p25_reduction": 60.2, "p75_reduction": 73.41, "mean_quality_retention_pct": 90.0, "max_reduction_at_99pct_quality": 88.23, "max_reduction_at_95pct_quality": 88.23, "mean_latency_ms": 17.12, "p50_latency_ms": 9.5, "p95_latency_ms": 52.35, "total_tokens_in": 16647, "total_tokens_out": 5918, "downstream_cost_usd_est": 0.014795, "cost_per_successful_est": 0.014795, "cost_reduction_vs_full_pct": 64.45 } }, "per_benchmark": { "b1_ccr_needle": { "Full": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 37800, "total_tokens_out": 37800, "downstream_cost_usd_est": 0.0945, "cost_per_successful_est": 0.0945, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 22762.22, "p50_latency_ms": 18552.62, "p95_latency_ms": 48903.07, "total_tokens_in": 37800, "total_tokens_out": 37800, "downstream_cost_usd_est": 0.0945, "cost_per_successful_est": 0.0945, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 24.87, "median_token_reduction_pct": 25.0, "p25_reduction": 24.6, "p75_reduction": 25.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 25.26, "max_reduction_at_95pct_quality": 25.26, "mean_latency_ms": 643.63, "p50_latency_ms": 519.18, "p95_latency_ms": 1194.98, "total_tokens_in": 37800, "total_tokens_out": 28399, "downstream_cost_usd_est": 0.070998, "cost_per_successful_est": 0.0709975, "cost_reduction_vs_full_pct": 24.87 }, "Truncation": { "n": 50, "passed": 0, "pass_rate_pct": 0.0, "mean_token_reduction_pct": 65.87, "median_token_reduction_pct": 65.87, "p25_reduction": 65.87, "p75_reduction": 65.87, "mean_quality_retention_pct": 25.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.03, "total_tokens_in": 37800, "total_tokens_out": 12900, "downstream_cost_usd_est": 0.03225, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 65.87 } }, "b1_tool_outputs": { "Full": { "n": 8, "passed": 8, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 1657, "total_tokens_out": 1657, "downstream_cost_usd_est": 0.004143, "cost_per_successful_est": 0.0041425, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 8, "passed": 8, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 3.1, "median_token_reduction_pct": 0.47, "p25_reduction": 0.0, "p75_reduction": 5.45, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 12.9, "max_reduction_at_95pct_quality": 12.9, "mean_latency_ms": 3636.85, "p50_latency_ms": 3615.61, "p95_latency_ms": 6720.33, "total_tokens_in": 1657, "total_tokens_out": 1591, "downstream_cost_usd_est": 0.003978, "cost_per_successful_est": 0.0039775, "cost_reduction_vs_full_pct": 3.98 }, "Headroom": { "n": 8, "passed": 8, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 13.84, "median_token_reduction_pct": 14.73, "p25_reduction": 11.64, "p75_reduction": 17.29, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 17.87, "max_reduction_at_95pct_quality": 17.87, "mean_latency_ms": 184.22, "p50_latency_ms": 180.85, "p95_latency_ms": 221.96, "total_tokens_in": 1657, "total_tokens_out": 1421, "downstream_cost_usd_est": 0.003553, "cost_per_successful_est": 0.0035525, "cost_reduction_vs_full_pct": 14.24 }, "Truncation": { "n": 8, "passed": 4, "pass_rate_pct": 50.0, "mean_token_reduction_pct": 63.72, "median_token_reduction_pct": 64.03, "p25_reduction": 61.73, "p75_reduction": 65.14, "mean_quality_retention_pct": 50.0, "max_reduction_at_99pct_quality": 66.27, "max_reduction_at_95pct_quality": 66.27, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 1657, "total_tokens_out": 592, "downstream_cost_usd_est": 0.00148, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.27 } }, "b1_squad": { "Full": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 4228, "total_tokens_out": 4228, "downstream_cost_usd_est": 0.01057, "cost_per_successful_est": 0.01057, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 381.11, "p50_latency_ms": 375.02, "p95_latency_ms": 534.51, "total_tokens_in": 4228, "total_tokens_out": 4228, "downstream_cost_usd_est": 0.01057, "cost_per_successful_est": 0.01057, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 25, "passed": 23, "pass_rate_pct": 92.0, "mean_token_reduction_pct": -0.93, "median_token_reduction_pct": -1.21, "p25_reduction": -6.11, "p75_reduction": 5.56, "mean_quality_retention_pct": 92.0, "max_reduction_at_99pct_quality": 7.88, "max_reduction_at_95pct_quality": 7.88, "mean_latency_ms": 260.68, "p50_latency_ms": 253.68, "p95_latency_ms": 385.05, "total_tokens_in": 4228, "total_tokens_out": 4219, "downstream_cost_usd_est": 0.010547, "cost_per_successful_est": 0.0105475, "cost_reduction_vs_full_pct": 0.21 }, "Truncation": { "n": 25, "passed": 9, "pass_rate_pct": 36.0, "mean_token_reduction_pct": 65.48, "median_token_reduction_pct": 65.09, "p25_reduction": 62.88, "p75_reduction": 68.7, "mean_quality_retention_pct": 36.0, "max_reduction_at_99pct_quality": 70.3, "max_reduction_at_95pct_quality": 70.3, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 4228, "total_tokens_out": 1466, "downstream_cost_usd_est": 0.003665, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 65.33 } }, "b1_bfcl": { "Full": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 0.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 3515, "total_tokens_out": 3515, "downstream_cost_usd_est": 0.008788, "cost_per_successful_est": 0.0087875, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.59, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 0.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 5786.03, "p50_latency_ms": 5452.12, "p95_latency_ms": 10416.03, "total_tokens_in": 3515, "total_tokens_out": 3497, "downstream_cost_usd_est": 0.008743, "cost_per_successful_est": 0.0087425, "cost_reduction_vs_full_pct": 0.51 }, "Headroom": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 20.53, "median_token_reduction_pct": 21.08, "p25_reduction": 20.18, "p75_reduction": 23.44, "mean_quality_retention_pct": 0.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 310.89, "p50_latency_ms": 317.76, "p95_latency_ms": 501.36, "total_tokens_in": 3515, "total_tokens_out": 2803, "downstream_cost_usd_est": 0.007007, "cost_per_successful_est": 0.0070075, "cost_reduction_vs_full_pct": 20.26 }, "Truncation": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 63.87, "median_token_reduction_pct": 63.71, "p25_reduction": 63.19, "p75_reduction": 64.71, "mean_quality_retention_pct": 0.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 3515, "total_tokens_out": 1275, "downstream_cost_usd_est": 0.003188, "cost_per_successful_est": 0.0031875, "cost_reduction_vs_full_pct": 63.73 } }, "b1_hotpotqa": { "Full": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 52.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 37247, "total_tokens_out": 37247, "downstream_cost_usd_est": 0.093118, "cost_per_successful_est": 0.0931175, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.32, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 52.0, "max_reduction_at_99pct_quality": 3.63, "max_reduction_at_95pct_quality": 3.63, "mean_latency_ms": 10444.04, "p50_latency_ms": 10009.32, "p95_latency_ms": 15438.84, "total_tokens_in": 37247, "total_tokens_out": 37132, "downstream_cost_usd_est": 0.09283, "cost_per_successful_est": 0.09283, "cost_reduction_vs_full_pct": 0.31 }, "Headroom": { "n": 25, "passed": 24, "pass_rate_pct": 96.0, "mean_token_reduction_pct": 18.35, "median_token_reduction_pct": 18.33, "p25_reduction": 16.54, "p75_reduction": 20.4, "mean_quality_retention_pct": 48.0, "max_reduction_at_99pct_quality": 21.41, "max_reduction_at_95pct_quality": 21.41, "mean_latency_ms": 1858.73, "p50_latency_ms": 1699.99, "p95_latency_ms": 3259.13, "total_tokens_in": 37247, "total_tokens_out": 30279, "downstream_cost_usd_est": 0.075698, "cost_per_successful_est": 0.0756975, "cost_reduction_vs_full_pct": 18.71 }, "Truncation": { "n": 25, "passed": 17, "pass_rate_pct": 68.0, "mean_token_reduction_pct": 64.68, "median_token_reduction_pct": 64.86, "p25_reduction": 64.4, "p75_reduction": 65.69, "mean_quality_retention_pct": 20.0, "max_reduction_at_99pct_quality": 65.09, "max_reduction_at_95pct_quality": 65.09, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.02, "total_tokens_in": 37247, "total_tokens_out": 13230, "downstream_cost_usd_est": 0.033075, "cost_per_successful_est": 0.033075, "cost_reduction_vs_full_pct": 64.48 } }, "b2_longbench_multifieldqa_en": { "Full": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 23347, "total_tokens_out": 23347, "downstream_cost_usd_est": 0.058368, "cost_per_successful_est": 0.0583675, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.45, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 3.63, "max_reduction_at_95pct_quality": 3.63, "mean_latency_ms": 9663.51, "p50_latency_ms": 9660.77, "p95_latency_ms": 13109.22, "total_tokens_in": 23347, "total_tokens_out": 23248, "downstream_cost_usd_est": 0.05812, "cost_per_successful_est": 0.05812, "cost_reduction_vs_full_pct": 0.42 }, "Headroom": { "n": 15, "passed": 14, "pass_rate_pct": 93.33, "mean_token_reduction_pct": 18.61, "median_token_reduction_pct": 18.35, "p25_reduction": 16.66, "p75_reduction": 20.54, "mean_quality_retention_pct": 46.67, "max_reduction_at_99pct_quality": 21.41, "max_reduction_at_95pct_quality": 21.41, "mean_latency_ms": 2101.81, "p50_latency_ms": 1942.44, "p95_latency_ms": 3377.98, "total_tokens_in": 23347, "total_tokens_out": 18892, "downstream_cost_usd_est": 0.04723, "cost_per_successful_est": 0.04723, "cost_reduction_vs_full_pct": 19.08 }, "Truncation": { "n": 15, "passed": 10, "pass_rate_pct": 66.67, "mean_token_reduction_pct": 64.45, "median_token_reduction_pct": 64.65, "p25_reduction": 64.39, "p75_reduction": 65.44, "mean_quality_retention_pct": 20.0, "max_reduction_at_99pct_quality": 65.09, "max_reduction_at_95pct_quality": 65.09, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 23347, "total_tokens_out": 8372, "downstream_cost_usd_est": 0.02093, "cost_per_successful_est": 0.02093, "cost_reduction_vs_full_pct": 64.14 } }, "b2_longbench_hotpotqa": { "Full": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 23347, "total_tokens_out": 23347, "downstream_cost_usd_est": 0.058368, "cost_per_successful_est": 0.0583675, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.45, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 3.63, "max_reduction_at_95pct_quality": 3.63, "mean_latency_ms": 9872.15, "p50_latency_ms": 10126.23, "p95_latency_ms": 13844.01, "total_tokens_in": 23347, "total_tokens_out": 23248, "downstream_cost_usd_est": 0.05812, "cost_per_successful_est": 0.05812, "cost_reduction_vs_full_pct": 0.42 }, "Headroom": { "n": 15, "passed": 14, "pass_rate_pct": 93.33, "mean_token_reduction_pct": 18.61, "median_token_reduction_pct": 18.35, "p25_reduction": 16.66, "p75_reduction": 20.54, "mean_quality_retention_pct": 46.67, "max_reduction_at_99pct_quality": 21.41, "max_reduction_at_95pct_quality": 21.41, "mean_latency_ms": 1847.25, "p50_latency_ms": 1567.1, "p95_latency_ms": 3707.98, "total_tokens_in": 23347, "total_tokens_out": 18892, "downstream_cost_usd_est": 0.04723, "cost_per_successful_est": 0.04723, "cost_reduction_vs_full_pct": 19.08 }, "Truncation": { "n": 15, "passed": 10, "pass_rate_pct": 66.67, "mean_token_reduction_pct": 64.45, "median_token_reduction_pct": 64.65, "p25_reduction": 64.39, "p75_reduction": 65.44, "mean_quality_retention_pct": 20.0, "max_reduction_at_99pct_quality": 65.09, "max_reduction_at_95pct_quality": 65.09, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.02, "total_tokens_in": 23347, "total_tokens_out": 8372, "downstream_cost_usd_est": 0.02093, "cost_per_successful_est": 0.02093, "cost_reduction_vs_full_pct": 64.14 } }, "b2_longbench_2wikimqa": { "Full": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 23347, "total_tokens_out": 23347, "downstream_cost_usd_est": 0.058368, "cost_per_successful_est": 0.0583675, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.45, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 3.63, "max_reduction_at_95pct_quality": 3.63, "mean_latency_ms": 6699.1, "p50_latency_ms": 6742.29, "p95_latency_ms": 9202.29, "total_tokens_in": 23347, "total_tokens_out": 23248, "downstream_cost_usd_est": 0.05812, "cost_per_successful_est": 0.05812, "cost_reduction_vs_full_pct": 0.42 }, "Headroom": { "n": 15, "passed": 14, "pass_rate_pct": 93.33, "mean_token_reduction_pct": 18.61, "median_token_reduction_pct": 18.35, "p25_reduction": 16.66, "p75_reduction": 20.54, "mean_quality_retention_pct": 46.67, "max_reduction_at_99pct_quality": 21.41, "max_reduction_at_95pct_quality": 21.41, "mean_latency_ms": 1212.32, "p50_latency_ms": 1036.82, "p95_latency_ms": 2513.63, "total_tokens_in": 23347, "total_tokens_out": 18892, "downstream_cost_usd_est": 0.04723, "cost_per_successful_est": 0.04723, "cost_reduction_vs_full_pct": 19.08 }, "Truncation": { "n": 15, "passed": 10, "pass_rate_pct": 66.67, "mean_token_reduction_pct": 64.45, "median_token_reduction_pct": 64.65, "p25_reduction": 64.39, "p75_reduction": 65.44, "mean_quality_retention_pct": 20.0, "max_reduction_at_99pct_quality": 65.09, "max_reduction_at_95pct_quality": 65.09, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 23347, "total_tokens_out": 8372, "downstream_cost_usd_est": 0.02093, "cost_per_successful_est": 0.02093, "cost_reduction_vs_full_pct": 64.14 } }, "b2_longbench_musique": { "Full": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 23347, "total_tokens_out": 23347, "downstream_cost_usd_est": 0.058368, "cost_per_successful_est": 0.0583675, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 15, "passed": 15, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.45, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 53.33, "max_reduction_at_99pct_quality": 3.63, "max_reduction_at_95pct_quality": 3.63, "mean_latency_ms": 6485.56, "p50_latency_ms": 6571.25, "p95_latency_ms": 9064.22, "total_tokens_in": 23347, "total_tokens_out": 23248, "downstream_cost_usd_est": 0.05812, "cost_per_successful_est": 0.05812, "cost_reduction_vs_full_pct": 0.42 }, "Headroom": { "n": 15, "passed": 14, "pass_rate_pct": 93.33, "mean_token_reduction_pct": 18.61, "median_token_reduction_pct": 18.35, "p25_reduction": 16.66, "p75_reduction": 20.54, "mean_quality_retention_pct": 46.67, "max_reduction_at_99pct_quality": 21.41, "max_reduction_at_95pct_quality": 21.41, "mean_latency_ms": 1103.2, "p50_latency_ms": 959.77, "p95_latency_ms": 2257.88, "total_tokens_in": 23347, "total_tokens_out": 18892, "downstream_cost_usd_est": 0.04723, "cost_per_successful_est": 0.04723, "cost_reduction_vs_full_pct": 19.08 }, "Truncation": { "n": 15, "passed": 10, "pass_rate_pct": 66.67, "mean_token_reduction_pct": 64.45, "median_token_reduction_pct": 64.65, "p25_reduction": 64.39, "p75_reduction": 65.44, "mean_quality_retention_pct": 20.0, "max_reduction_at_99pct_quality": 65.09, "max_reduction_at_95pct_quality": 65.09, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 23347, "total_tokens_out": 8372, "downstream_cost_usd_est": 0.02093, "cost_per_successful_est": 0.02093, "cost_reduction_vs_full_pct": 64.14 } }, "b3_needle_10000": { "Full": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 150405, "total_tokens_out": 150405, "downstream_cost_usd_est": 0.376013, "cost_per_successful_est": 0.3760125, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 354.63, "p50_latency_ms": 339.42, "p95_latency_ms": 445.59, "total_tokens_in": 150405, "total_tokens_out": 150405, "downstream_cost_usd_est": 0.376013, "cost_per_successful_est": 0.3760125, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 26.49, "median_token_reduction_pct": 26.45, "p25_reduction": 26.33, "p75_reduction": 26.55, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 26.94, "max_reduction_at_95pct_quality": 26.94, "mean_latency_ms": 6564.37, "p50_latency_ms": 5389.23, "p95_latency_ms": 8894.96, "total_tokens_in": 150405, "total_tokens_out": 110567, "downstream_cost_usd_est": 0.276417, "cost_per_successful_est": 0.2764175, "cost_reduction_vs_full_pct": 26.49 }, "Truncation": { "n": 25, "passed": 10, "pass_rate_pct": 40.0, "mean_token_reduction_pct": 64.97, "median_token_reduction_pct": 65.04, "p25_reduction": 64.86, "p75_reduction": 65.04, "mean_quality_retention_pct": 40.0, "max_reduction_at_99pct_quality": 64.86, "max_reduction_at_95pct_quality": 64.86, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 150405, "total_tokens_out": 52685, "downstream_cost_usd_est": 0.131713, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.97 } }, "b3_needle_25000": { "Full": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 375405, "total_tokens_out": 375405, "downstream_cost_usd_est": 0.938512, "cost_per_successful_est": 0.9385125, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 416.56, "p50_latency_ms": 297.04, "p95_latency_ms": 587.7, "total_tokens_in": 375405, "total_tokens_out": 375405, "downstream_cost_usd_est": 0.938512, "cost_per_successful_est": 0.9385125, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 20.08, "median_token_reduction_pct": 26.34, "p25_reduction": 26.32, "p75_reduction": 26.38, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 26.64, "max_reduction_at_95pct_quality": 26.64, "mean_latency_ms": 15906.67, "p50_latency_ms": 14111.16, "p95_latency_ms": 20548.69, "total_tokens_in": 375405, "total_tokens_out": 300035, "downstream_cost_usd_est": 0.750088, "cost_per_successful_est": 0.7500875, "cost_reduction_vs_full_pct": 20.08 }, "Truncation": { "n": 25, "passed": 10, "pass_rate_pct": 40.0, "mean_token_reduction_pct": 64.99, "median_token_reduction_pct": 65.02, "p25_reduction": 64.94, "p75_reduction": 65.02, "mean_quality_retention_pct": 40.0, "max_reduction_at_99pct_quality": 64.94, "max_reduction_at_95pct_quality": 64.94, "mean_latency_ms": 0.07, "p50_latency_ms": 0.01, "p95_latency_ms": 0.43, "total_tokens_in": 375405, "total_tokens_out": 131435, "downstream_cost_usd_est": 0.328587, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.99 } }, "b3_needle_50000": { "Full": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 750405, "total_tokens_out": 750405, "downstream_cost_usd_est": 1.876012, "cost_per_successful_est": 1.8760125, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 539.23, "p50_latency_ms": 498.44, "p95_latency_ms": 725.06, "total_tokens_in": 750405, "total_tokens_out": 750405, "downstream_cost_usd_est": 1.876012, "cost_per_successful_est": 1.8760125, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 25, "passed": 25, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 20361.75, "p50_latency_ms": 20320.83, "p95_latency_ms": 20664.75, "total_tokens_in": 750405, "total_tokens_out": 750405, "downstream_cost_usd_est": 1.876012, "cost_per_successful_est": 1.8760125, "cost_reduction_vs_full_pct": 0.0 }, "Truncation": { "n": 25, "passed": 10, "pass_rate_pct": 40.0, "mean_token_reduction_pct": 64.99, "median_token_reduction_pct": 65.01, "p25_reduction": 64.97, "p75_reduction": 65.01, "mean_quality_retention_pct": 40.0, "max_reduction_at_99pct_quality": 64.97, "max_reduction_at_95pct_quality": 64.97, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.02, "total_tokens_in": 750405, "total_tokens_out": 262685, "downstream_cost_usd_est": 0.656713, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.99 } }, "b4_multihop_2": { "Full": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 26300, "total_tokens_out": 26300, "downstream_cost_usd_est": 0.06575, "cost_per_successful_est": 0.06575, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 583.66, "p50_latency_ms": 518.68, "p95_latency_ms": 896.23, "total_tokens_in": 26300, "total_tokens_out": 26300, "downstream_cost_usd_est": 0.06575, "cost_per_successful_est": 0.06575, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.38, "median_token_reduction_pct": 0.38, "p25_reduction": 0.38, "p75_reduction": 0.38, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.38, "max_reduction_at_95pct_quality": 0.38, "mean_latency_ms": 422.39, "p50_latency_ms": 358.23, "p95_latency_ms": 909.94, "total_tokens_in": 26300, "total_tokens_out": 26200, "downstream_cost_usd_est": 0.0655, "cost_per_successful_est": 0.0655, "cost_reduction_vs_full_pct": 0.38 }, "Truncation": { "n": 50, "passed": 0, "pass_rate_pct": 0.0, "mean_token_reduction_pct": 64.64, "median_token_reduction_pct": 64.64, "p25_reduction": 64.64, "p75_reduction": 64.64, "mean_quality_retention_pct": 50.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 26300, "total_tokens_out": 9300, "downstream_cost_usd_est": 0.02325, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.64 }, "LLMLingua-2": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 62.09, "median_token_reduction_pct": 62.17, "p25_reduction": 61.98, "p75_reduction": 62.17, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 62.55, "max_reduction_at_95pct_quality": 62.55, "mean_latency_ms": 1222.69, "p50_latency_ms": 1199.88, "p95_latency_ms": 1750.94, "total_tokens_in": 26300, "total_tokens_out": 9970, "downstream_cost_usd_est": 0.024925, "cost_per_successful_est": 0.024925, "cost_reduction_vs_full_pct": 62.09 } }, "b4_multihop_3": { "Full": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 39450, "total_tokens_out": 39450, "downstream_cost_usd_est": 0.098625, "cost_per_successful_est": 0.098625, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 1927.88, "p50_latency_ms": 1571.96, "p95_latency_ms": 2643.21, "total_tokens_in": 39450, "total_tokens_out": 39450, "downstream_cost_usd_est": 0.098625, "cost_per_successful_est": 0.098625, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 5.32, "median_token_reduction_pct": 5.32, "p25_reduction": 5.32, "p75_reduction": 5.32, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 5.32, "max_reduction_at_95pct_quality": 5.32, "mean_latency_ms": 1073.29, "p50_latency_ms": 1062.98, "p95_latency_ms": 1554.57, "total_tokens_in": 39450, "total_tokens_out": 37350, "downstream_cost_usd_est": 0.093375, "cost_per_successful_est": 0.093375, "cost_reduction_vs_full_pct": 5.32 }, "Truncation": { "n": 50, "passed": 0, "pass_rate_pct": 0.0, "mean_token_reduction_pct": 65.02, "median_token_reduction_pct": 65.02, "p25_reduction": 65.02, "p75_reduction": 65.02, "mean_quality_retention_pct": 33.33, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.02, "total_tokens_in": 39450, "total_tokens_out": 13800, "downstream_cost_usd_est": 0.0345, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 65.02 }, "LLMLingua-2": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 62.12, "median_token_reduction_pct": 62.1, "p25_reduction": 61.98, "p75_reduction": 62.23, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 62.74, "max_reduction_at_95pct_quality": 62.74, "mean_latency_ms": 2460.61, "p50_latency_ms": 2006.3, "p95_latency_ms": 3711.09, "total_tokens_in": 39450, "total_tokens_out": 14942, "downstream_cost_usd_est": 0.037355, "cost_per_successful_est": 0.037355, "cost_reduction_vs_full_pct": 62.12 } }, "b4_multihop_5": { "Full": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 65750, "total_tokens_out": 65750, "downstream_cost_usd_est": 0.164375, "cost_per_successful_est": 0.164375, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 2232.36, "p50_latency_ms": 1994.28, "p95_latency_ms": 2926.49, "total_tokens_in": 65750, "total_tokens_out": 65750, "downstream_cost_usd_est": 0.164375, "cost_per_successful_est": 0.164375, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 7.07, "median_token_reduction_pct": 7.07, "p25_reduction": 7.07, "p75_reduction": 7.07, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 7.07, "max_reduction_at_95pct_quality": 7.07, "mean_latency_ms": 1484.91, "p50_latency_ms": 1402.38, "p95_latency_ms": 2191.96, "total_tokens_in": 65750, "total_tokens_out": 61100, "downstream_cost_usd_est": 0.15275, "cost_per_successful_est": 0.15275, "cost_reduction_vs_full_pct": 7.07 }, "Truncation": { "n": 50, "passed": 0, "pass_rate_pct": 0.0, "mean_token_reduction_pct": 64.87, "median_token_reduction_pct": 64.87, "p25_reduction": 64.87, "p75_reduction": 64.87, "mean_quality_retention_pct": 40.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.02, "total_tokens_in": 65750, "total_tokens_out": 23100, "downstream_cost_usd_est": 0.05775, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.87 }, "LLMLingua-2": { "n": 50, "passed": 50, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 59.98, "median_token_reduction_pct": 60.0, "p25_reduction": 59.85, "p75_reduction": 60.08, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 60.53, "max_reduction_at_95pct_quality": 60.53, "mean_latency_ms": 2508.82, "p50_latency_ms": 2299.37, "p95_latency_ms": 3520.69, "total_tokens_in": 65750, "total_tokens_out": 26313, "downstream_cost_usd_est": 0.065782, "cost_per_successful_est": 0.0657825, "cost_reduction_vs_full_pct": 59.98 } }, "b5_coding_agent": { "Full": { "n": 24, "passed": 24, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 0, "p50_latency_ms": 0.0, "p95_latency_ms": 0.0, "total_tokens_in": 16647, "total_tokens_out": 16647, "downstream_cost_usd_est": 0.041617, "cost_per_successful_est": 0.0416175, "cost_reduction_vs_full_pct": 0.0 }, "SuperCompress-v2": { "n": 24, "passed": 24, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 64.14, "median_token_reduction_pct": 84.95, "p25_reduction": 40.66, "p75_reduction": 93.61, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 96.6, "max_reduction_at_95pct_quality": 96.6, "mean_latency_ms": 5886.63, "p50_latency_ms": 5780.72, "p95_latency_ms": 10863.51, "total_tokens_in": 16647, "total_tokens_out": 5148, "downstream_cost_usd_est": 0.01287, "cost_per_successful_est": 0.01287, "cost_reduction_vs_full_pct": 69.08 }, "Headroom": { "n": 24, "passed": 18, "pass_rate_pct": 75.0, "mean_token_reduction_pct": 14.76, "median_token_reduction_pct": 15.83, "p25_reduction": 8.21, "p75_reduction": 19.01, "mean_quality_retention_pct": 85.83, "max_reduction_at_99pct_quality": 31.49, "max_reduction_at_95pct_quality": 31.49, "mean_latency_ms": 380.32, "p50_latency_ms": 318.37, "p95_latency_ms": 1059.81, "total_tokens_in": 16647, "total_tokens_out": 14009, "downstream_cost_usd_est": 0.035022, "cost_per_successful_est": 0.0350225, "cost_reduction_vs_full_pct": 15.85 }, "Truncation": { "n": 24, "passed": 11, "pass_rate_pct": 45.83, "mean_token_reduction_pct": 64.31, "median_token_reduction_pct": 64.97, "p25_reduction": 61.95, "p75_reduction": 65.99, "mean_quality_retention_pct": 56.46, "max_reduction_at_99pct_quality": 68.04, "max_reduction_at_95pct_quality": 68.04, "mean_latency_ms": 0.01, "p50_latency_ms": 0.01, "p95_latency_ms": 0.01, "total_tokens_in": 16647, "total_tokens_out": 5851, "downstream_cost_usd_est": 0.014628, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 64.85 }, "LLMLingua-2": { "n": 24, "passed": 12, "pass_rate_pct": 50.0, "mean_token_reduction_pct": 52.68, "median_token_reduction_pct": 51.59, "p25_reduction": 48.78, "p75_reduction": 56.55, "mean_quality_retention_pct": 70.49, "max_reduction_at_99pct_quality": 68.09, "max_reduction_at_95pct_quality": 68.09, "mean_latency_ms": 834.69, "p50_latency_ms": 699.88, "p95_latency_ms": 1688.33, "total_tokens_in": 16647, "total_tokens_out": 8030, "downstream_cost_usd_est": 0.020075, "cost_per_successful_est": null, "cost_reduction_vs_full_pct": 51.76 }, "SuperCompress-v1": { "n": 24, "passed": 21, "pass_rate_pct": 87.5, "mean_token_reduction_pct": 60.73, "median_token_reduction_pct": 67.97, "p25_reduction": 60.2, "p75_reduction": 73.41, "mean_quality_retention_pct": 90.0, "max_reduction_at_99pct_quality": 88.23, "max_reduction_at_95pct_quality": 88.23, "mean_latency_ms": 17.12, "p50_latency_ms": 9.5, "p95_latency_ms": 52.35, "total_tokens_in": 16647, "total_tokens_out": 5918, "downstream_cost_usd_est": 0.014795, "cost_per_successful_est": 0.014795, "cost_reduction_vs_full_pct": 64.45 } }, "b6_scaling_1000": { "SuperCompress-v2": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.07, "median_token_reduction_pct": 0.07, "p25_reduction": 0.07, "p75_reduction": 0.07, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.07, "max_reduction_at_95pct_quality": 0.07, "mean_latency_ms": 4246.02, "p50_latency_ms": 4246.02, "p95_latency_ms": 4246.02, "total_tokens_in": 1372, "total_tokens_out": 1371, "downstream_cost_usd_est": 0.003427, "cost_per_successful_est": 0.0034275, "cost_reduction_vs_full_pct": 0.07 }, "Headroom": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 31.92, "median_token_reduction_pct": 31.92, "p25_reduction": 31.92, "p75_reduction": 31.92, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 31.92, "max_reduction_at_95pct_quality": 31.92, "mean_latency_ms": 1290.84, "p50_latency_ms": 1290.84, "p95_latency_ms": 1290.84, "total_tokens_in": 1372, "total_tokens_out": 934, "downstream_cost_usd_est": 0.002335, "cost_per_successful_est": 0.002335, "cost_reduction_vs_full_pct": 31.92 } }, "b6_scaling_5000": { "SuperCompress-v2": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.01, "median_token_reduction_pct": 0.01, "p25_reduction": 0.01, "p75_reduction": 0.01, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.01, "max_reduction_at_95pct_quality": 0.01, "mean_latency_ms": 24925.33, "p50_latency_ms": 24925.33, "p95_latency_ms": 24925.33, "total_tokens_in": 6888, "total_tokens_out": 6887, "downstream_cost_usd_est": 0.017218, "cost_per_successful_est": 0.0172175, "cost_reduction_vs_full_pct": 0.01 }, "Headroom": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 36.45, "median_token_reduction_pct": 36.45, "p25_reduction": 36.45, "p75_reduction": 36.45, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 36.45, "max_reduction_at_95pct_quality": 36.45, "mean_latency_ms": 2216.66, "p50_latency_ms": 2216.66, "p95_latency_ms": 2216.66, "total_tokens_in": 6888, "total_tokens_out": 4377, "downstream_cost_usd_est": 0.010942, "cost_per_successful_est": 0.0109425, "cost_reduction_vs_full_pct": 36.45 } }, "b6_scaling_10000": { "SuperCompress-v2": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.01, "median_token_reduction_pct": 0.01, "p25_reduction": 0.01, "p75_reduction": 0.01, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.01, "max_reduction_at_95pct_quality": 0.01, "mean_latency_ms": 52448.47, "p50_latency_ms": 52448.47, "p95_latency_ms": 52448.47, "total_tokens_in": 13804, "total_tokens_out": 13803, "downstream_cost_usd_est": 0.034507, "cost_per_successful_est": 0.0345075, "cost_reduction_vs_full_pct": 0.01 }, "Headroom": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 37.4, "median_token_reduction_pct": 37.4, "p25_reduction": 37.4, "p75_reduction": 37.4, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 37.4, "max_reduction_at_95pct_quality": 37.4, "mean_latency_ms": 4536.38, "p50_latency_ms": 4536.38, "p95_latency_ms": 4536.38, "total_tokens_in": 13804, "total_tokens_out": 8641, "downstream_cost_usd_est": 0.021602, "cost_per_successful_est": 0.0216025, "cost_reduction_vs_full_pct": 37.4 } }, "b6_scaling_25000": { "SuperCompress-v2": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 127884.25, "p50_latency_ms": 127884.25, "p95_latency_ms": 127884.25, "total_tokens_in": 34552, "total_tokens_out": 34551, "downstream_cost_usd_est": 0.086377, "cost_per_successful_est": 0.0863775, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 38.15, "median_token_reduction_pct": 38.15, "p25_reduction": 38.15, "p75_reduction": 38.15, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 38.15, "max_reduction_at_95pct_quality": 38.15, "mean_latency_ms": 11028.66, "p50_latency_ms": 11028.66, "p95_latency_ms": 11028.66, "total_tokens_in": 34552, "total_tokens_out": 21372, "downstream_cost_usd_est": 0.05343, "cost_per_successful_est": 0.05343, "cost_reduction_vs_full_pct": 38.15 } }, "b6_scaling_50000": { "SuperCompress-v2": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 262068.16, "p50_latency_ms": 262068.16, "p95_latency_ms": 262068.16, "total_tokens_in": 69132, "total_tokens_out": 69131, "downstream_cost_usd_est": 0.172827, "cost_per_successful_est": 0.1728275, "cost_reduction_vs_full_pct": 0.0 }, "Headroom": { "n": 1, "passed": 1, "pass_rate_pct": 100.0, "mean_token_reduction_pct": 0.0, "median_token_reduction_pct": 0.0, "p25_reduction": 0.0, "p75_reduction": 0.0, "mean_quality_retention_pct": 100.0, "max_reduction_at_99pct_quality": 0.0, "max_reduction_at_95pct_quality": 0.0, "mean_latency_ms": 20267.82, "p50_latency_ms": 20267.82, "p95_latency_ms": 20267.82, "total_tokens_in": 69132, "total_tokens_out": 69132, "downstream_cost_usd_est": 0.17283, "cost_per_successful_est": 0.17283, "cost_reduction_vs_full_pct": 0.0 } } }, "matched_token_budget": { "50": { "SuperCompress-v2": { "mean_score_pct": 95.0, "n": 20 }, "Headroom": { "mean_score_pct": 73.75, "n": 20 }, "Truncation": { "mean_score_pct": 73.75, "n": 20 } }, "35": { "SuperCompress-v2": { "mean_score_pct": 84.0, "n": 20 }, "Headroom": { "mean_score_pct": 56.5, "n": 20 }, "Truncation": { "mean_score_pct": 56.5, "n": 20 } }, "20": { "SuperCompress-v2": { "mean_score_pct": 81.17, "n": 20 }, "Headroom": { "mean_score_pct": 81.17, "n": 20 }, "Truncation": { "mean_score_pct": 33.67, "n": 20 } } }, "launch_chart": { "Tokens removed": { "Headroom": 12.8, "LLMLingua-2": 60.2, "SuperCompress-v2": 3.61 }, "Quality retained": { "Headroom": 83.13, "LLMLingua-2": 95.93, "SuperCompress-v2": 85.46 }, "Max reduction @ \u226599% quality": { "Headroom": 38.15, "LLMLingua-2": 68.09, "SuperCompress-v2": 96.6 }, "Cost / successful task": { "Headroom": 3.8670275, "LLMLingua-2": 0.1481375, "SuperCompress-v2": 4.289615 }, "Needle recall": { "Headroom": 83.13, "LLMLingua-2": 95.93, "SuperCompress-v2": 85.46 } }, "n_rows": 1952, "n_cases": 390 }