{"licence": "Creative Commons Attribution 4.0 International (CC BY 4.0)", "licence_url": "https://creativecommons.org/licenses/by/4.0/", "attribution": "Spring Prompt (springprompt.com)", "source_url": "https://springprompt.com/benchmarks/surveybench", "release": "2026-11-10-cad1a1ab581e", "test_your_prompt": "These are our tasks, not yours. Test your own prompt on these models: https://springprompt.com/prompt-test", "results": [{"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1683.0, "ci_low": 1590.8, "ci_high": 1840.7, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1485.3, "ci_low": 1426.5, "ci_high": 1560.9, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1476.6, "ci_low": 1429.0, "ci_high": 1556.3, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1456.3, "ci_low": 1414.2, "ci_high": 1540.2, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1274.5, "ci_low": 1231.8, "ci_high": 1338.8, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1250.6, "ci_low": 1170.0, "ci_high": 1338.7, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1178.8, "ci_low": 1103.4, "ci_high": 1277.0, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1172.8, "ci_low": 1082.5, "ci_high": 1271.0, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 975.7, "ci_low": 856.1, "ci_high": 1093.8, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 970.6, "ci_low": 915.4, "ci_high": 1029.2, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 808.4, "ci_low": 754.0, "ci_high": 854.6, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 795.4, "ci_low": 736.6, "ci_high": 836.0, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 752.1, "ci_low": 660.2, "ci_high": 841.4, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 671.0, "ci_low": 609.7, "ci_high": 714.1, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 606.7, "ci_low": 536.0, "ci_high": 661.7, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 590.2, "ci_low": 515.5, "ci_high": 642.6, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 585.8, "ci_low": 441.0, "ci_high": 710.2, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 461.9, "ci_low": 299.8, "ci_high": 629.1, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "rating", "metric_name": "SurveyBench: analysis rating", "unit": "rating", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 454.2, "ci_low": 260.8, "ci_high": 564.5, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 96.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 93.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 90.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 89.68, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 83.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 71.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 64.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 64.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 59.18, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 56.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 39.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 28.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "research_score", "metric_name": "SurveyBench: research score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.37, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "sound_rate", "metric_name": "SurveyBench: sound research", "unit": "% of tasks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 97.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 96.97, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 96.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 93.61, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 92.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 92.53, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 91.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 90.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 88.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 87.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 83.59, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 82.93, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 81.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 72.66, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 62.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 53.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "plan_score", "metric_name": "SurveyBench: survey plan score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 36.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 97.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 96.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 91.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 90.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 88.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 88.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 79.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 66.35, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 63.77, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 56.82, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 56.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 53.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 42.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 14.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 13.61, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "analysis_score", "metric_name": "SurveyBench: analysis score", "unit": "points of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 98.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 97.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 94.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 75.49, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 63.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "numeric_accuracy", "metric_name": "SurveyBench: numbers right", "unit": "% of answers", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 63.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.55, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 95.65, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 94.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 93.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 91.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 84.06, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 83.02, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 78.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 71.01, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 62.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 60.66, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 59.42, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "traps_handled", "metric_name": "SurveyBench: traps handled", "unit": "% of traps", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 56.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 86.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 80.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 78.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 74.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 62.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 62.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 57.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 56.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 43.92, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 40.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 35.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 34.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 29.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 26.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 25.09, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 23.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "win_rate", "metric_name": "SurveyBench: head-to-head win rate", "unit": "% of comparisons", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.01398, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.01435, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.01848, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.01977, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.02156, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.03727, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.05686, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.06352, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.06934, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.07298, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0924, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.13101, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.1429, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.33324, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.3542, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.45707, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.74209, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "cost_per_task", "metric_name": "SurveyBench: cost per task", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.51051, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 16.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 47.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 74.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 83.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 92.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 92.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 133.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 147.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 169.8, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 180.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 193.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 221.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 252.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 337.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 351.8, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 434.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 439.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "seconds_per_task", "metric_name": "SurveyBench: time per task", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1956.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 8112.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 26601.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 30893.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 33902.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 51553.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 57343.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63237.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "tokens_per_task", "metric_name": "SurveyBench: output tokens per task", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 70935.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.questions", "metric_name": "SurveyBench: leading or double-barrelled questions", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.structure", "metric_name": "SurveyBench: survey structure", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.objectives", "metric_name": "SurveyBench: objectives unanswerable", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.hypothesis", "metric_name": "SurveyBench: stakeholder's view presumed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.plan.precision", "metric_name": "SurveyBench: precision ignored", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.numbers", "metric_name": "SurveyBench: wrong numbers", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.traps", "metric_name": "SurveyBench: traps missed", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.unsupported", "metric_name": "SurveyBench: unsupported findings", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 88.89, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.recommendation", "metric_name": "SurveyBench: wrong recommendation", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 11.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 22.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.44, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 55.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 66.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 77.78, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}, {"benchmark": "surveybench", "metric": "fail.analysis.wrong_step", "metric_name": "SurveyBench: mistaken next steps", "unit": "% of tasks", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "9", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": "GPT-6.1 Sol, Claude Opus 5.5 and Gemini 3.1 Pro, by majority"}]}