{"licence": "Creative Commons Attribution 4.0 International (CC BY 4.0)", "licence_url": "https://creativecommons.org/licenses/by/4.0/", "attribution": "Spring Prompt (springprompt.com)", "source_url": "https://springprompt.com/data", "release": "2026-11-10-cad1a1ab581e", "test_your_prompt": "These are our tasks, not yours. Test your own prompt on these models: https://springprompt.com/prompt-test", "results": [{"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 892.0, "ci_low": 641.0, "ci_high": 1081.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 780.0, "ci_low": 589.0, "ci_high": 940.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 727.0, "ci_low": 567.0, "ci_high": 890.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 614.0, "ci_low": 470.0, "ci_high": 766.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 584.0, "ci_low": 327.0, "ci_high": 806.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 572.0, "ci_low": 428.0, "ci_high": 718.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 571.0, "ci_low": 420.0, "ci_high": 710.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 571.0, "ci_low": 420.0, "ci_high": 699.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 571.0, "ci_low": 412.0, "ci_high": 721.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 571.0, "ci_low": 428.0, "ci_high": 698.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 571.0, "ci_low": 429.0, "ci_high": 719.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 555.0, "ci_low": 263.0, "ci_high": 727.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 529.0, "ci_low": 388.0, "ci_high": 670.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 529.0, "ci_low": 373.0, "ci_high": 653.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 529.0, "ci_low": 361.0, "ci_high": 669.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 529.0, "ci_low": 308.0, "ci_high": 680.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 529.0, "ci_low": 407.0, "ci_high": 643.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 491.0, "ci_low": 0.0, "ci_high": 727.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 420.0, "ci_low": 367.0, "ci_high": 488.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 310.0, "ci_low": 0.0, "ci_high": 641.0, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 249.0, "ci_low": 0.0, "ci_high": 486.0, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 197.0, "ci_low": 0.0, "ci_high": 439.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 197.0, "ci_low": 0.0, "ci_high": 425.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 197.0, "ci_low": 0.0, "ci_high": 404.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 33.0, "ci_low": 0.0, "ci_high": 248.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 248.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.ladder_elo", "metric_name": "BulletBench Lightning 10+1: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 272.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 368.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 421.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 512.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 515.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 517.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 523.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 543.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 569.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 586.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 594.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 615.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 713.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 767.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 826.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 882.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 886.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 943.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 1024.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 1048.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 1178.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 1241.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 1261.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1279.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 1370.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 1520.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 1844.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.median_latency_ms", "metric_name": "BulletBench Lightning 10+1: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 2083.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.time_loss_rate", "metric_name": "BulletBench Lightning 10+1: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.27, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.35, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 4.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.invalid_move_rate", "metric_name": "BulletBench Lightning 10+1: invalid moves", "unit": "% of moves", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 5.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.00045, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "jaredpalmer-kev-4b", "model_name": "Kev 4B", "developer": "Jared Palmer", "configuration": "Kev 4B", "value": 0.000635, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.000869, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.001035, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.001382, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.001925, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.002391, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 0.002494, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.002502, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.003521, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.004053, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.004091, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.004995, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.005835, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.006417, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.006979, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.007186, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.007678, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5 (low reasoning)", "value": 0.008041, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.009534, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.011659, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.013619, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.021301, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.029713, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.049138, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.158258, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "10p1.cost_per_game", "metric_name": "BulletBench Lightning 10+1: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.165374, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1147.0, "ci_low": 979.0, "ci_high": 1373.0, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 1126.0, "ci_low": 949.0, "ci_high": 1347.0, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1100.0, "ci_low": 789.0, "ci_high": 1450.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 927.0, "ci_low": 657.0, "ci_high": 1152.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 839.0, "ci_low": 595.0, "ci_high": 1072.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 707.0, "ci_low": 458.0, "ci_high": 933.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 683.0, "ci_low": 487.0, "ci_high": 1130.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 683.0, "ci_low": 487.0, "ci_high": 1130.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 645.0, "ci_low": 0.0, "ci_high": 890.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 557.0, "ci_low": 370.0, "ci_high": 727.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 555.0, "ci_low": 392.0, "ci_high": 727.0, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 555.0, "ci_low": 396.0, "ci_high": 765.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 491.0, "ci_low": 299.0, "ci_high": 659.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 358.0, "ci_low": 0.0, "ci_high": 619.0, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.ladder_elo", "metric_name": "BulletBench Blitz 3+2: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 358.0, "ci_low": 0.0, "ci_high": 600.0, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 519.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 789.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 837.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1297.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1937.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 2112.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 3722.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 4039.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 4783.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 4858.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 5365.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5612.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 6172.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 6219.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 6979.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 7106.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 7513.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.median_latency_ms", "metric_name": "BulletBench Blitz 3+2: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 8122.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 12.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.time_loss_rate", "metric_name": "BulletBench Blitz 3+2: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.53, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.invalid_move_rate", "metric_name": "BulletBench Blitz 3+2: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.007164, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.008238, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.008653, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.01643, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.026106, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.031778, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.070617, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.076745, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.079469, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.115847, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.12185, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.175778, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.207138, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.267174, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.284697, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.384659, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.523922, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "180p2.cost_per_game", "metric_name": "BulletBench Blitz 3+2: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 1.098534, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 1068.0, "ci_low": 888.0, "ci_high": 1241.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 928.0, "ci_low": 676.0, "ci_high": 1184.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 912.0, "ci_low": 697.0, "ci_high": 1087.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 810.0, "ci_low": 582.0, "ci_high": 1006.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 780.0, "ci_low": 586.0, "ci_high": 956.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 628.0, "ci_low": 429.0, "ci_high": 849.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 584.0, "ci_low": 435.0, "ci_high": 749.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 572.0, "ci_low": 428.0, "ci_high": 717.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 572.0, "ci_low": 428.0, "ci_high": 708.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 572.0, "ci_low": 429.0, "ci_high": 720.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 540.0, "ci_low": 296.0, "ci_high": 765.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 529.0, "ci_low": 388.0, "ci_high": 670.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 529.0, "ci_low": 373.0, "ci_high": 653.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 529.0, "ci_low": 329.0, "ci_high": 698.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 529.0, "ci_low": 263.0, "ci_high": 688.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 529.0, "ci_low": 393.0, "ci_high": 652.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 491.0, "ci_low": 230.0, "ci_high": 714.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 491.0, "ci_low": 187.0, "ci_high": 738.0, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 491.0, "ci_low": 299.0, "ci_high": 659.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 489.0, "ci_low": 376.0, "ci_high": 608.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 453.0, "ci_low": 355.0, "ci_high": 538.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 445.0, "ci_low": 135.0, "ci_high": 626.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 445.0, "ci_low": 215.0, "ci_high": 639.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 409.0, "ci_low": 100.0, "ci_high": 609.0, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 385.0, "ci_low": 276.0, "ci_high": 464.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 385.0, "ci_low": 279.0, "ci_high": 459.0, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 367.0, "ci_low": 126.0, "ci_high": 564.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 358.0, "ci_low": 0.0, "ci_high": 595.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 358.0, "ci_low": 0.0, "ci_high": 600.0, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 323.0, "ci_low": 51.0, "ci_high": 529.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 323.0, "ci_low": 0.0, "ci_high": 528.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 310.0, "ci_low": 0.0, "ci_high": 575.0, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 249.0, "ci_low": 0.0, "ci_high": 453.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 197.0, "ci_low": 0.0, "ci_high": 445.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 197.0, "ci_low": 0.0, "ci_high": 439.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 443.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 147.0, "ci_low": 0.0, "ci_high": 381.0, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 115.0, "ci_low": 0.0, "ci_high": 485.0, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 273.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.ladder_elo", "metric_name": "BulletBench Bullet 60s: ladder Elo", "unit": "ladder Elo", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 33.0, "ci_low": 0.0, "ci_high": 270.0, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 276.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 367.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 423.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 508.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 518.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 519.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 522.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 524.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 565.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 570.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 598.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 625.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 713.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 792.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 916.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 917.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 1061.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 1198.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1241.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 1394.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 1440.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 1467.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 1573.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 1580.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 1583.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 1695.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 1710.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 1813.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 1885.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 2197.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 2273.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 2579.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 2799.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 2847.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 2920.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 2958.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 2989.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 3009.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 3026.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 3090.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 3334.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 3433.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 3622.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 4184.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 4232.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.median_latency_ms", "metric_name": "BulletBench Bullet 60s: median move time", "unit": "milliseconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 5040.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 41.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 41.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 58.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 87.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.time_loss_rate", "metric_name": "BulletBench Bullet 60s: games lost on time", "unit": "% of games", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.31, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.42, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.46, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 3.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.invalid_move_rate", "metric_name": "BulletBench Bullet 60s: invalid moves", "unit": "% of moves", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 7.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-decide", "model_name": "Mercury Decide", "developer": "Inception", "configuration": "Mercury Decide", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "liquid-d1", "model_name": "d1", "developer": "Liquid AI", "configuration": "d1", "value": 0.000733, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "perplexity-pplx-decider-v1-27b", "model_name": "Decider V1 27B", "developer": "Perplexity", "configuration": "Decider V1 27B", "value": 0.000964, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "typesafe-jev-1-13", "model_name": "Jev 1.13", "developer": "TypeSafe", "configuration": "Jev 1.13", "value": 0.001314, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash (low reasoning)", "value": 0.001584, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna (no reasoning)", "value": 0.001624, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5 (no reasoning)", "value": 0.001915, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (no reasoning)", "value": 0.00195, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef-flash", "model_name": "Clef Flash", "developer": "Cloudflare", "configuration": "Clef Flash", "value": 0.002546, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna-decisions", "model_name": "GPT-6 Luna Decisions", "developer": "OpenAI", "configuration": "GPT-6 Luna Decisions", "value": 0.002591, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "bytedance-seed-2-0-mini", "model_name": "Seed-2.0-Mini", "developer": "ByteDance", "configuration": "Seed-2.0-Mini (minimal reasoning)", "value": 0.003021, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3 (low reasoning)", "value": 0.003208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.003586, "ci_low": null, "ci_high": null, "observed_on": "2026-10-14", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-small-2603", "model_name": "Mistral Small 4", "developer": "Mistral AI", "configuration": "Mistral Small 4 (no reasoning)", "value": 0.003857, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017 (low reasoning)", "value": 0.004732, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-nano", "model_name": "GPT-5.4 nano", "developer": "OpenAI", "configuration": "GPT-5.4 nano (no reasoning)", "value": 0.005024, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "cloudflare-clef", "model_name": "Clef", "developer": "Cloudflare", "configuration": "Clef", "value": 0.005627, "ci_low": null, "ci_high": null, "observed_on": "2026-10-07", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.005796, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal (low reasoning)", "value": 0.006058, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-flash-lite", "model_name": "Gemini 3.1 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.1 Flash-Lite (minimal reasoning)", "value": 0.006825, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "thinkingmachines-inkling-small", "model_name": "Inkling Small", "developer": "Thinking Machines", "configuration": "Inkling Small (no reasoning)", "value": 0.007487, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-6-flash", "model_name": "Gemini 3.6 Flash", "developer": "Google", "configuration": "Gemini 3.6 Flash (minimal reasoning)", "value": 0.011273, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3 (minimal reasoning)", "value": 0.014236, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.015378, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.015582, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.016258, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash (low reasoning)", "value": 0.016514, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra (low reasoning)", "value": 0.017217, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro (low reasoning)", "value": 0.017541, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-5-4-mini", "model_name": "GPT-5.4 mini", "developer": "OpenAI", "configuration": "GPT-5.4 mini (no reasoning)", "value": 0.020513, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.020984, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3 (low reasoning)", "value": 0.021951, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol (low reasoning)", "value": 0.024288, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash", "model_name": "Gemini 3.5 Flash", "developer": "Google", "configuration": "Gemini 3.5 Flash (minimal reasoning)", "value": 0.028933, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol (no reasoning)", "value": 0.029717, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5 (no reasoning)", "value": 0.035407, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "none", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.041972, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview (low reasoning)", "value": 0.044593, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "Gemini API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3 (low reasoning)", "value": 0.053958, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder (minimal reasoning)", "value": 0.066301, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "minimal", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2 (low reasoning)", "value": 0.079975, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.094311, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.094843, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra (low reasoning)", "value": 0.103691, "ci_low": null, "ci_high": null, "observed_on": "2026-10-01", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast (low reasoning)", "value": 0.336607, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "low", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "bulletbench", "metric": "60.cost_per_game", "metric_name": "BulletBench Bullet 60s: cost per game", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol Ultrafast", "value": 0.35722, "ci_low": null, "ci_high": null, "observed_on": "2026-10-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenAI API (direct)", "judge": ""}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 80.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 77.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 77.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 74.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 65.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 63.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63.09, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 61.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 58.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 54.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 47.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 46.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 46.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 40.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 39.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 29.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 12.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 71.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 64.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 60.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 57.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 48.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 41.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 41.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 35.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 28.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 98.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 98.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 97.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 97.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 97.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 97.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 97.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 97.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 97.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 96.91, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 96.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 96.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 95.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 95.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 95.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 94.39, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 94.18, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 94.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 90.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 63.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.024, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.03, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.077, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.226, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.393, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.411, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.417, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.435, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.446, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.494, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.536, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.661, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.708, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.786, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.905, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.976, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.083, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.179, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.185, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1.202, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.226, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.702, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 1.744, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.071, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 2.494, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.685, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 11.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 12.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 27.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 31.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 39.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 48.81, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 49.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 64.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 73.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 9.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 24.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 30.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 38.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 48.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 9.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 4.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 76.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 74.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 73.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 72.02, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 67.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 58.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 57.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 47.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 30.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 27.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 69.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 51.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 51.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 33.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 23.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.8, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 98.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.31, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 98.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 98.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 98.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 97.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 97.32, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 97.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 97.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 96.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 96.08, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 95.87, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 95.42, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 95.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 94.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 94.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 94.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 93.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 93.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 89.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 16.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.054, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.065, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.089, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.089, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.214, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.345, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.357, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.458, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.601, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.637, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.714, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 1.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 1.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.655, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.714, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.982, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 2.131, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 2.357, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 2.464, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.077, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.44, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.548, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 3.643, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 3.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 3.863, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 4.006, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 4.524, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 5.321, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 6.738, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 82.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 3.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.59, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 28.89, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 31.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 23.81, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 39.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 49.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 57.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 59.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 60.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 63.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 67.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 70.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 76.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 79.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 80.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 81.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 81.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 83.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 84.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 85.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 97.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 20.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 20.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 22.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 23.21, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 36.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 42.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 51.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 98.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.69, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 98.02, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 97.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 97.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 97.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 96.84, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 96.62, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 96.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 95.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 95.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 95.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 95.06, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 94.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 93.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 93.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 93.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 92.63, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 90.56, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 88.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 61.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.05, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 96.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 95.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 95.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 94.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 94.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 94.16, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 94.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 92.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 92.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 92.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 92.7, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 92.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 92.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 91.97, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 91.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 91.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 91.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 90.27, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 86.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 59.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.51, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.46, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 2.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 2.15, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 2.16, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 3.54, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 3.8, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 3.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 5.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.97, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 84.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 98.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 98.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.21, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 98.05, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.000477, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.001015, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.001172, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.001208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.001542, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.002047, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.003277, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.00356, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.005035, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.00607, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.006138, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.007011, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.008005, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.008384, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.008518, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.010292, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.010921, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.012462, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.016418, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.017191, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.023719, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.025572, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.035464, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.036073, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.03712, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.040006, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.045438, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.046895, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.087539, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.099128, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 4.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 6.7, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 7.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 8.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 9.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 10.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 10.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 10.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 10.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 12.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 16.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 17.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 17.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 18.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 21.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 23.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 25.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 32.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 41.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 47.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 62.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 66.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 78.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 97.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 114.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 457.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1226.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1442.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 2180.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 2890.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 3014.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3651.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3996.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1592.5, "ci_low": 1482.5, "ci_high": 1765.8, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 1475.0, "ci_low": 1412.6, "ci_high": 1583.8, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1424.9, "ci_low": 1318.5, "ci_high": 1571.6, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1420.1, "ci_low": 1348.2, "ci_high": 1522.8, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1376.8, "ci_low": 1300.1, "ci_high": 1469.2, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1359.0, "ci_low": 1275.5, "ci_high": 1455.3, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1331.7, "ci_low": 1229.5, "ci_high": 1443.7, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1291.2, "ci_low": 1235.9, "ci_high": 1368.5, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1288.7, "ci_low": 1188.2, "ci_high": 1390.9, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1284.9, "ci_low": 1227.0, "ci_high": 1368.8, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 1279.7, "ci_low": 1208.8, "ci_high": 1363.2, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1239.9, "ci_low": 1123.7, "ci_high": 1371.4, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1237.5, "ci_low": 1170.6, "ci_high": 1302.5, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1222.0, "ci_low": 1147.3, "ci_high": 1307.1, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1155.3, "ci_low": 1084.3, "ci_high": 1233.5, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1149.6, "ci_low": 1017.3, "ci_high": 1256.1, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1100.7, "ci_low": 1000.2, "ci_high": 1208.9, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1079.7, "ci_low": 973.2, "ci_high": 1163.3, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1060.2, "ci_low": 999.2, "ci_high": 1140.6, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 984.5, "ci_low": 912.6, "ci_high": 1058.3, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 842.6, "ci_low": 775.1, "ci_high": 888.2, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 822.6, "ci_low": 680.5, "ci_high": 936.6, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 813.4, "ci_low": 746.9, "ci_high": 871.1, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 760.0, "ci_low": 665.7, "ci_high": 851.8, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 742.1, "ci_low": 663.4, "ci_high": 802.1, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 681.0, "ci_low": 595.4, "ci_high": 756.7, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 640.0, "ci_low": 542.1, "ci_high": 724.1, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 632.1, "ci_low": 520.8, "ci_high": 731.7, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 619.5, "ci_low": 525.6, "ci_high": 700.9, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "rating", "metric_name": "DeckBench: deck rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 564.8, "ci_low": 427.8, "ci_high": 635.2, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "presentable_rate", "metric_name": "DeckBench: presentable decks", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "accurate_rate", "metric_name": "DeckBench: accurate decks", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "layout_clean_rate", "metric_name": "DeckBench: clean layout", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 88.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 87.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 83.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 81.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 80.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 78.47, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 78.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 77.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 76.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 71.96, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 71.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 68.47, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 67.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 66.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 66.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 61.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 61.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 60.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 58.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 56.05, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 55.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 54.37, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 51.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 47.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 46.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 43.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 40.87, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 30.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "design_quality", "metric_name": "DeckBench: design quality", "unit": "% of the maximum", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 89.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 88.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 86.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 84.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 82.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 77.97, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 77.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 75.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 71.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 70.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 70.35, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63.04, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 62.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 60.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 60.83, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 51.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 50.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 48.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 48.39, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 48.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 45.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 42.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 40.46, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 39.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 32.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 30.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 28.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 25.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "win_rate", "metric_name": "DeckBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 24.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.00857, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.00865, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.00905, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.01042, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.01162, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.01262, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.01477, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.02552, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0359, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0448, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.04607, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.04626, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0465, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.06134, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.06722, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.11468, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.1355, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.14527, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.15273, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.15295, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.18211, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.20966, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.25243, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.26102, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.4537, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.54068, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.80661, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.09215, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "cost_per_deck", "metric_name": "DeckBench: cost per deck", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.39571, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 15.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 28.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 51.7, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 62.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 67.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 73.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 78.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 83.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 83.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 89.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 93.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 98.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 114.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 134.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 154.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 157.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 169.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 174.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 178.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 217.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 285.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 311.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 330.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 332.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 374.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 437.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 543.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "seconds_per_deck", "metric_name": "DeckBench: time per deck", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1288.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 12086.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 16952.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 21019.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 27796.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 32448.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 33285.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33765.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 45141.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "tokens_per_deck", "metric_name": "DeckBench: output tokens per deck", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 51739.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.layout", "metric_name": "DeckBench: layout defects", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.draft", "metric_name": "DeckBench: draft figure quoted", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.numbers", "metric_name": "DeckBench: unsupported numbers", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.findings", "metric_name": "DeckBench: findings missing", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.recommendation", "metric_name": "DeckBench: recommendation late or wrong", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.caveats", "metric_name": "DeckBench: caveat dropped", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.misleading", "metric_name": "DeckBench: misleading metric used", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.claims", "metric_name": "DeckBench: unsupported claims", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 83.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "deckbench", "metric": "fail.design", "metric_name": "DeckBench: slides needing work", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "6", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 61.27, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 58.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 56.59, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 55.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 54.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 53.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 52.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 51.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 51.54, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 51.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 47.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 46.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 44.06, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 43.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 43.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 41.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 40.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 38.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 37.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 36.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 36.26, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 34.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 28.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 27.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 26.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 25.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 25.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 24.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 24.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 17.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 15.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.9615, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.7174, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.7138, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.7039, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.6872, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.6747, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.6574, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.6473, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6472, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.6435, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.547, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.5458, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.5233, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.5124, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.5024, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.4478, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.4426, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.4316, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.4192, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.413, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.2944, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.2574, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.2259, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.1435, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.1005, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0943, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0313, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0116, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0097, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": -0.1652, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": -0.2137, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 3.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1032894.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 965271.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 911566.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 858022.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 842431.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 738878.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 701058.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 695680.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 693379.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 684623.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 673560.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 654270.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 600356.58, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 599139.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 585571.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 571202.72, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 539202.03, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 533401.04, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 507735.97, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 489176.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 337250.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 301288.78, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 187636.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 156831.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 104867.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 99775.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 32146.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 11834.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 9717.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": -159881.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": -205000.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 72.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 59.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 58.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 57.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 57.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 54.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 52.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 52.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 52.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 50.96, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.92, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 43.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 40.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 39.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 38.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 38.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 35.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 34.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 31.85, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 26.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 25.82, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 23.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 19.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 17.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 15.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 13.47, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 12.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 12.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 7.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 55.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 55.22, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 55.08, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 55.04, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 54.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 54.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 54.96, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 54.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 54.87, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 54.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 54.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 54.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 54.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 54.82, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 54.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 54.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 54.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 54.66, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 54.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 54.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 54.63, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 54.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 54.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 54.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 54.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 54.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 54.21, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 54.15, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 53.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 53.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 51.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 69.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 69.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 68.56, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 68.49, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 68.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 67.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 67.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 67.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 67.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 67.13, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 65.53, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 65.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 65.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 65.13, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 63.24, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 61.78, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 60.81, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 60.17, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 58.22, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 55.81, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 54.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 51.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 51.27, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 50.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 48.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 45.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 44.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 43.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 39.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 30.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 28.54, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 48.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 44.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 42.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 36.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 36.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 35.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 34.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 31.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 30.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 28.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 28.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 27.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 26.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 25.92, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 21.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 20.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 19.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 17.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 15.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 14.92, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 14.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.58, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 9.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 9.08, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 8.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.014182, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.023373, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.029807, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.031284, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.037854, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.038977, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.053281, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.065733, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.079867, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.11103, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.131184, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.148765, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.158208, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.16178, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.183125, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.187194, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.197608, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.242268, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.244047, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.420282, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.42358, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.518802, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.567512, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.73084, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1.012818, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.064612, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.225685, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.60214, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.759152, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 2.48607, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1683.0, "ci_low": 1590.8, "ci_high": 1840.7, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1485.3, "ci_low": 1426.5, "ci_high": 1560.9, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1476.6, "ci_low": 1429.0, "ci_high": 1556.3, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1456.3, "ci_low": 1414.2, "ci_high": 1540.2, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1274.5, "ci_low": 1231.8, "ci_high": 1338.8, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1250.6, "ci_low": 1170.0, "ci_high": 1338.7, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1178.8, "ci_low": 1103.4, "ci_high": 1277.0, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1172.8, "ci_low": 1082.5, "ci_high": 1271.0, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 975.7, "ci_low": 856.1, "ci_high": 1093.8, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 970.6, "ci_low": 915.4, "ci_high": 1029.2, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 808.4, "ci_low": 754.0, "ci_high": 854.6, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 795.4, "ci_low": 736.6, "ci_high": 836.0, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 752.1, "ci_low": 660.2, "ci_high": 841.4, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 671.0, "ci_low": 609.7, "ci_high": 714.1, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 606.7, "ci_low": 536.0, "ci_high": 661.7, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 590.2, "ci_low": 515.5, "ci_high": 642.6, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 585.8, "ci_low": 441.0, "ci_high": 710.2, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 461.9, "ci_low": 299.8, "ci_high": 629.1, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 454.2, "ci_low": 260.8, "ci_high": 564.5, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 96.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 93.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 90.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 89.68, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 83.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 71.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 64.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 64.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 59.18, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 56.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 39.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 28.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.37, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 96.97, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 96.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 93.61, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 92.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 92.53, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 91.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 90.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 88.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 87.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 83.59, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 82.93, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 81.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 72.66, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 53.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 36.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 91.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 88.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 88.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 79.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 66.35, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 63.77, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 56.82, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 56.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 53.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 42.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 14.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 13.61, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 98.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 97.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 94.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 63.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 63.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 93.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 91.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 84.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 78.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.01, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 62.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 60.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 59.42, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 86.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 80.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 78.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 74.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 62.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 62.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 57.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 56.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 43.92, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 40.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 35.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 34.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 29.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 26.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 25.09, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 23.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.01398, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.01435, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.01848, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.01977, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.02156, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.03727, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.05686, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.06352, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.06934, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.07298, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0924, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.13101, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.1429, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.33324, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.3542, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.45707, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.74209, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.51051, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 47.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 74.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 83.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 92.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 133.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 147.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 169.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 180.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 193.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 221.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 252.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 337.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 351.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 434.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 439.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1956.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 8112.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 26601.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 30893.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 33902.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 51553.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 57343.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63237.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 70935.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}]}