{"licence": "Creative Commons Attribution 4.0 International (CC BY 4.0)", "licence_url": "https://creativecommons.org/licenses/by/4.0/", "attribution": "Spring Prompt (springprompt.com)", "source_url": "https://springprompt.com/benchmarks/roasbench", "release": "2026-11-10-cad1a1ab581e", "test_your_prompt": "These are our tasks, not yours. Test your own prompt on these models: https://springprompt.com/prompt-test", "results": [{"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 61.27, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 58.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 56.59, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 55.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 54.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 53.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 52.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 51.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 51.54, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 51.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 47.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 46.22, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 44.06, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 43.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 43.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 41.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 40.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 38.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 37.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 36.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 36.26, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 34.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 28.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 27.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 26.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 25.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 25.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 24.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 24.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 17.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "score", "metric_name": "ROASBench: overall score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 15.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.9615, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.7174, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.7138, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.7039, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.6872, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.6747, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.6574, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.6473, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6472, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.6435, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.547, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.5458, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.5233, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.5124, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.5024, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.4478, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.4426, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.4316, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.4192, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.413, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.2944, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.2574, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.2259, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.1435, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.1005, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0943, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0313, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0116, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0097, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": -0.1652, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "roi", "metric_name": "ROASBench: return on ad spend", "unit": "profit per $1 spent", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": -0.2137, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 3.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "overspend_months", "metric_name": "ROASBench: months over budget", "unit": "months of 12", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1032894.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 965271.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 911566.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 858022.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 842431.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 738878.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 701058.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 695680.28, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 693379.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 684623.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 673560.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 654270.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 600356.58, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 599139.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 585571.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 571202.72, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 539202.03, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 533401.04, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 507735.97, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 489176.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 337250.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 301288.78, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 187636.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 156831.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 104867.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 99775.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 32146.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 11834.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 9717.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": -159881.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "profit", "metric_name": "ROASBench: contribution profit", "unit": "simulated US dollars", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": -205000.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 72.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 59.13, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 58.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 57.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 57.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 54.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 52.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 52.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 52.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 50.96, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 44.92, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 43.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 40.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 39.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 38.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 38.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 35.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 34.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 31.85, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 26.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 25.82, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 23.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 19.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 17.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 15.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 13.47, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 12.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 12.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 7.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "business_score", "metric_name": "ROASBench: business score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 55.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 55.22, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 55.08, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 55.04, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 54.99, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 54.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 54.96, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 54.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 54.87, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 54.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 54.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 54.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 54.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 54.82, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 54.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 54.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 54.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 54.66, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 54.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 54.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 54.63, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 54.41, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 54.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 54.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 54.32, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 54.3, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 54.21, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 54.15, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 53.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 53.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "planning_score", "metric_name": "ROASBench: planning score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 51.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 69.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 69.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 68.56, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 68.49, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 68.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 67.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 67.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 67.56, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 67.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 67.13, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 65.53, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 65.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 65.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 65.13, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 63.24, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 61.78, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 60.81, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 60.17, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 58.22, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 55.81, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 54.73, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 51.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 51.27, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 50.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 48.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 45.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 44.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 43.18, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 39.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 30.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "persona_score", "metric_name": "ROASBench: audience score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 28.54, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 48.75, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 44.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 42.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 36.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 36.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 35.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 34.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 33.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 33.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 31.58, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 30.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 28.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 28.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 27.83, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 26.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 25.92, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 21.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 20.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 19.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 17.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 15.08, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 14.92, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 14.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.58, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 9.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 9.08, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "behavioral_score", "metric_name": "ROASBench: consistency score", "unit": "score out of 100", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 8.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.014182, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.023373, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.029807, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.031284, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.037854, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.038977, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "inception-mercury-3", "model_name": "Mercury 3", "developer": "Inception", "configuration": "Mercury 3", "value": 0.053281, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.065733, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.079867, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.11103, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3", "model_name": "GLM-5.3", "developer": "Z.ai", "configuration": "GLM-5.3", "value": 0.131184, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.148765, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.158208, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.16178, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.183125, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.187194, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "high", "served_by": "Mistral API (direct)", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-pro", "model_name": "DeepSeek-V4-Pro (0423)", "developer": "DeepSeek", "configuration": "DeepSeek-V4-Pro (0423)", "value": 0.197608, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.242268, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.244047, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.420282, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.42358, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.518802, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.567512, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.73084, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1.012818, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.064612, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.225685, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.60214, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.759152, "ci_low": null, "ci_high": null, "observed_on": "2026-11-09", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}, {"benchmark": "roasbench", "metric": "run_cost", "metric_name": "ROASBench: cost of a run", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 2.48607, "ci_low": null, "ci_high": null, "observed_on": "2026-09-29", "tasks": "", "products": "", "repeats": "", "reasoning_effort": "provider default", "served_by": "OpenRouter", "judge": ""}]}