{"licence": "Creative Commons Attribution 4.0 International (CC BY 4.0)", "licence_url": "https://creativecommons.org/licenses/by/4.0/", "attribution": "Spring Prompt (springprompt.com)", "source_url": "https://springprompt.com/benchmarks/catalogbench", "release": "2026-11-10-cad1a1ab581e", "test_your_prompt": "These are our tasks, not yours. Test your own prompt on these models: https://springprompt.com/prompt-test", "results": [{"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 80.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 77.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 77.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 74.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 70.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 65.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 63.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 63.09, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 61.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 58.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 54.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 47.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 46.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 46.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 40.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 39.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 29.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 12.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "pass_rate", "metric_name": "CatalogBench: publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 71.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 64.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 60.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 57.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 50.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 48.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 41.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 41.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 35.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 28.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "reliable_rate", "metric_name": "CatalogBench: reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 98.89, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 98.22, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 97.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 97.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 97.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 97.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 97.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 97.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 97.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 96.91, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 96.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 96.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 96.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 95.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 95.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 95.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 94.39, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 94.18, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 94.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 90.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "quality_score", "metric_name": "CatalogBench: content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 63.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.024, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.03, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.077, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.226, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.393, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.411, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.417, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.435, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.446, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.494, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.536, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.661, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.708, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.786, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.905, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.976, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.083, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.179, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.185, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1.202, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.226, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.702, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 1.744, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.071, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 2.494, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "unsupported_claims_per_product", "metric_name": "CatalogBench: unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.685, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "failed_outputs", "metric_name": "CatalogBench: failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 11.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 12.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_flags", "metric_name": "CatalogBench: claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 27.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 31.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 32.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 33.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 39.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 48.81, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 49.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 55.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 64.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.claims", "metric_name": "CatalogBench: unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 73.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 9.52, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 22.62, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 24.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 30.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 38.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.attributes", "metric_name": "CatalogBench: wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 48.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.uk_info", "metric_name": "CatalogBench: missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.channel_rules", "metric_name": "CatalogBench: channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 9.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.category", "metric_name": "CatalogBench: wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 11.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 4.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "fail.findability", "metric_name": "CatalogBench: not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 76.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 75.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 74.41, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 73.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 72.02, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 67.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 66.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 58.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 57.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 47.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 34.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 30.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 27.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 16.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.pass_rate", "metric_name": "CatalogBench (sales brief): publish-ready listings", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 69.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 67.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 51.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 51.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 37.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 33.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 23.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.reliable_rate", "metric_name": "CatalogBench (sales brief): reliably publish-ready", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.8, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.56, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.51, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 98.39, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.31, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 98.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 98.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 98.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.77, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 97.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 97.32, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 97.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 97.23, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.37, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 96.25, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.25, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 96.08, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 95.87, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 95.42, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 95.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 94.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 94.84, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 94.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 93.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 93.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 89.23, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.quality_score", "metric_name": "CatalogBench (sales brief): content quality", "unit": "% of checks", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 16.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.054, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.065, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.089, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.089, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.214, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.345, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.357, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.458, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.601, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.637, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.714, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 1.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 1.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.655, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 1.714, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.982, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 2.131, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 2.357, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 2.464, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 3.077, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.44, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 3.548, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 3.643, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 3.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 3.863, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3.94, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 4.006, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 4.524, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 5.321, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.unsupported_claims_per_product", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "claims per product", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 6.738, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.failed_outputs", "metric_name": "CatalogBench (sales brief): failed outputs", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 82.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 3.28, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 7.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 25.59, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 28.89, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 31.55, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.compliance_flags", "metric_name": "CatalogBench (sales brief): claims to check", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 44.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.33, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 23.81, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 26.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 39.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 49.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 57.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 59.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 60.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 63.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 67.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 70.24, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 76.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 78.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 79.76, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 80.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 81.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 81.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 83.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 84.52, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 85.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 91.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.claims", "metric_name": "CatalogBench (sales brief): unsupported claims", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 97.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 11.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 14.29, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 14.88, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 15.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 16.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 17.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 17.86, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 18.45, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 19.05, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 19.64, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 20.24, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 20.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 20.83, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 22.02, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 23.21, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 35.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 36.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 42.26, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.attributes", "metric_name": "CatalogBench (sales brief): wrong attributes", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 51.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.uk_info", "metric_name": "CatalogBench (sales brief): missing UK information", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.channel_rules", "metric_name": "CatalogBench (sales brief): channel rules broken", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.19, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.category", "metric_name": "CatalogBench (sales brief): wrong category or variant", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 13.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 1.79, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 2.38, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 2.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3.57, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 4.17, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 5.36, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 5.95, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 7.14, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 8.93, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 10.12, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 10.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 13.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "marketing.fail.findability", "metric_name": "CatalogBench (sales brief): not findable", "unit": "% of products", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 21.43, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 98.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.69, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 98.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 98.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 98.34, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.07, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 98.02, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 97.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 97.69, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 97.55, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 97.31, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 97.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 96.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 96.84, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 96.62, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 96.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.11, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 96.07, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 95.88, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 95.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 95.64, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 95.63, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 95.06, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 94.85, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 93.42, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 93.26, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 93.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 92.63, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 90.56, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 88.71, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "decision_accuracy", "metric_name": "CatalogBench: decision accuracy", "unit": "% of decisions", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 61.03, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 98.05, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 96.11, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 95.62, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 95.38, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 94.65, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 94.65, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 94.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 94.16, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 94.16, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 93.67, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 92.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 92.94, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 92.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 92.7, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 92.46, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 92.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 91.97, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 91.73, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 91.48, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 91.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 90.27, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 86.86, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "field_accuracy", "metric_name": "CatalogBench: field accuracy", "unit": "% of missing fields", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 59.61, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.25, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.49, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.51, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.75, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.76, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.76, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.98, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 1.44, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 1.46, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 1.47, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 1.68, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 1.69, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1.91, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 1.92, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 2.14, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 2.15, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 2.16, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 2.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 3.07, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 3.54, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 3.8, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 3.84, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 4.36, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 4.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 5.74, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "invented_rate", "metric_name": "CatalogBench: invented values", "unit": "% of filled values", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 6.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 96.97, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 90.91, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "mismatch_recall", "metric_name": "CatalogBench: conflicts caught", "unit": "% of conflicts", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 84.85, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 100.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 99.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 98.29, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 98.21, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 98.21, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "compliance_rate", "metric_name": "CatalogBench: channel compliance", "unit": "% of products", "direction": "higher", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 98.05, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 0.000477, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 0.001015, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 0.001172, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 0.001208, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 0.001542, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 0.002047, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 0.003277, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 0.00356, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 0.005035, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 0.00607, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 0.006138, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 0.007011, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 0.008005, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 0.008384, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 0.008518, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 0.010292, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 0.010921, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 0.012462, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 0.016418, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 0.017191, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 0.023719, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 0.025572, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 0.035464, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 0.036073, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 0.03712, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 0.040006, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 0.045438, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 0.046895, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 0.087539, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "cost_per_product", "metric_name": "CatalogBench: cost per product", "unit": "US dollars", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 0.099128, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-5-flash-lite", "model_name": "Gemini 3.5 Flash-Lite", "developer": "Google", "configuration": "Gemini 3.5 Flash-Lite", "value": 3.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-medium-3-5", "model_name": "Mistral Medium 3.5", "developer": "Mistral AI", "configuration": "Mistral Medium 3.5", "value": 4.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 6.7, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-4-argon", "model_name": "Gemini 4 Argon", "developer": "Google", "configuration": "Gemini 4 Argon", "value": 7.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-13", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-4-5", "model_name": "Claude Haiku 4.5", "developer": "Anthropic", "configuration": "Claude Haiku 4.5", "value": 8.1, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4", "value": 8.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-luna", "model_name": "GPT-6 Luna", "developer": "OpenAI", "configuration": "GPT-6 Luna", "value": 8.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-sonnet-5-5", "model_name": "Claude Sonnet 5.5", "developer": "Anthropic", "configuration": "Claude Sonnet 5.5", "value": 9.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 10.1, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "meta-muse-wonder", "model_name": "Muse Wonder", "developer": "Meta", "configuration": "Muse Wonder", "value": 10.2, "ci_low": null, "ci_high": null, "observed_on": "2026-10-12", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-1-sol", "model_name": "GPT-6.1 Sol", "developer": "OpenAI", "configuration": "GPT-6.1 Sol", "value": 10.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-sol", "model_name": "GPT-6 Sol", "developer": "OpenAI", "configuration": "GPT-6 Sol", "value": 10.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-haiku-5-5", "model_name": "Claude Haiku 5.5", "developer": "Anthropic", "configuration": "Claude Haiku 5.5", "value": 11.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-08", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "openai-gpt-6-astra", "model_name": "GPT-6 Astra", "developer": "OpenAI", "configuration": "GPT-6 Astra", "value": 12.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-opus-5-5", "model_name": "Claude Opus 5.5", "developer": "Anthropic", "configuration": "Claude Opus 5.5", "value": 16.9, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 17.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash", "model_name": "DeepSeek-V4.1-Flash", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash", "value": 17.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-8-flash", "model_name": "Gemini 3.8 Flash", "developer": "Google", "configuration": "Gemini 3.8 Flash", "value": 18.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "anthropic-claude-fable-5-1", "model_name": "Claude Fable 5.1", "developer": "Anthropic", "configuration": "Claude Fable 5.1", "value": 21.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "google-gemini-3-1-pro-preview", "model_name": "Gemini 3.1 Pro Preview", "developer": "Google", "configuration": "Gemini 3.1 Pro Preview", "value": 22.2, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 23.5, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 25.2, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 29.4, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 32.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "xai-grok-4-7", "model_name": "Grok 4.7", "developer": "xAI", "configuration": "Grok 4.7", "value": 41.7, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "meta-muse-spark-1-3", "model_name": "Muse Spark 1.3", "developer": "Meta", "configuration": "Muse Spark 1.3", "value": 47.5, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "zai-glm-5v-turbo", "model_name": "GLM-5V-Turbo", "developer": "Z.ai", "configuration": "GLM-5V-Turbo", "value": 62.3, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3", "model_name": "Kimi K3", "developer": "Moonshot AI", "configuration": "Kimi K3", "value": 66.6, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "alibaba-qwen3-8-max-0902", "model_name": "Qwen3.8-Max (0902)", "developer": "Alibaba", "configuration": "Qwen3.8-Max (0902)", "value": 78.4, "ci_low": null, "ci_high": null, "observed_on": "2026-09-30", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "mistral-mistral-large-4-0", "model_name": "Mistral Large 4", "developer": "Mistral AI", "configuration": "Mistral Large 4 (high reasoning)", "value": 97.5, "ci_low": null, "ci_high": null, "observed_on": "2026-10-06", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "high", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "seconds_per_product", "metric_name": "CatalogBench: time per product", "unit": "seconds", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 114.3, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "openai-gpt-6-1-terra", "model_name": "GPT-6.1 Terra", "developer": "OpenAI", "configuration": "GPT-6.1 Terra", "value": 457.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "deepseek-deepseek-v4-1-flash-1017", "model_name": "DeepSeek-V4.1-Flash-1017", "developer": "DeepSeek", "configuration": "DeepSeek-V4.1-Flash-1017", "value": 1226.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "anthropic-claude-fable-5-2", "model_name": "Claude Fable 5.2", "developer": "Anthropic", "configuration": "Claude Fable 5.2", "value": 1442.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-10", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "particle-ai-particle-1", "model_name": "Particle-1", "developer": "Particle AI", "configuration": "Particle-1", "value": 2180.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 2890.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-19", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "minimax-minimax-m3-1-flash", "model_name": "MiniMax M3.1 Flash", "developer": "MiniMax", "configuration": "MiniMax M3.1 Flash", "value": 3014.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "zai-glm-5-3-flashx-legal", "model_name": "GLM-5.3 FlashX Legal", "developer": "Z.ai", "configuration": "GLM-5.3 FlashX Legal", "value": 3651.0, "ci_low": null, "ci_high": null, "observed_on": "2026-11-04", "tasks": "", "products": "56.0", "repeats": "3", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}, {"benchmark": "catalogbench", "metric": "tokens_per_product", "metric_name": "CatalogBench: output tokens per product", "unit": "tokens", "direction": "lower", "model": "moonshotai-kimi-k3-1-pro", "model_name": "Kimi K3.1 Pro", "developer": "Moonshot AI", "configuration": "Kimi K3.1 Pro", "value": 3996.0, "ci_low": null, "ci_high": null, "observed_on": "2026-10-27", "tasks": "", "products": "56", "repeats": "1", "reasoning_effort": "provider default", "served_by": "", "judge": "openai/gpt-6.1-sol"}]}