{"before":{"method":"cadence-personal-4","checked_at":"2026-11-09T14:10:14.197762+00:00","revision":"b03e35a899bb9f214f58635141ea3f64fe7cbebe4e45f5b4cc22f80e86b40b90","sources":[{"id":"epoch","ok":true,"checked_at":"2026-11-09T14:10:12.899910+00:00","updated_at":"2026-11-09T14:10:12.899910+00:00"},{"id":"writing-comparison","ok":true,"checked_at":"2026-11-09T14:10:12.901374+00:00","updated_at":"2026-11-09T14:10:12.901374+00:00"},{"id":"eq-writing","ok":true,"checked_at":"2026-11-09T14:10:12.901968+00:00","updated_at":"2026-11-09T14:10:12.901968+00:00"},{"id":"eq-longform","ok":true,"checked_at":"2026-11-09T14:10:12.902504+00:00","updated_at":"2026-11-09T14:10:12.902504+00:00"},{"id":"roboflow","ok":true,"checked_at":"2026-11-09T14:10:13.064934+00:00","updated_at":"2026-11-09T14:10:13.064934+00:00"},{"id":"blueprint","ok":true,"checked_at":"2026-11-09T14:10:13.066101+00:00","updated_at":"2026-11-09T14:10:13.066101+00:00"},{"id":"gdp-pdf","ok":true,"checked_at":"2026-11-09T14:10:13.175610+00:00","updated_at":"2026-11-09T14:10:13.175610+00:00"},{"id":"terminal-4","ok":true,"checked_at":"2026-11-09T14:10:13.205412+00:00","updated_at":"2026-11-09T14:10:13.205412+00:00"},{"id":"terminal-science","ok":true,"checked_at":"2026-11-09T14:10:13.281329+00:00","updated_at":"2026-11-09T14:10:13.281329+00:00"},{"id":"weirdml","ok":true,"checked_at":"2026-11-09T14:10:13.342359+00:00","updated_at":"2026-11-09T14:10:13.342359+00:00"},{"id":"livebench","ok":true,"checked_at":"2026-11-09T14:10:13.460455+00:00","updated_at":"2026-11-09T14:10:13.460455+00:00"},{"id":"extractbench","ok":true,"checked_at":"2026-11-09T14:10:13.720803+00:00","updated_at":"2026-11-09T14:10:13.720803+00:00"},{"id":"automationbench","ok":true,"checked_at":"2026-11-09T14:10:13.789546+00:00","updated_at":"2026-11-09T14:10:13.789546+00:00"}],"capabilities":[{"id":"code","name":"Coding","icon":"code","color":"#9ab8e8","scope":"Build software, fix bugs, and work across a codebase.","benchmarks":[{"id":"swe-bench-verified","name":"SWE-Bench verified","source":"Epoch AI","models":32,"latest_model_date":"2026-06-13","url":"https://epoch.ai/benchmarks"},{"id":"aider-polyglot","name":"Aider polyglot","source":"Epoch AI","models":55,"latest_model_date":"2025-12-01","url":"https://epoch.ai/benchmarks"},{"id":"deepswe","name":"DeepSWE","source":"Epoch AI","models":26,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"frontiercode","name":"FrontierCode","source":"Epoch AI","models":46,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"gso-bench","name":"GSO-Bench","source":"Epoch AI","models":35,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"mirrorcode","name":"MirrorCode","source":"Epoch AI","models":15,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"livebench-code-20260625","name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":87,"models":153},{"id":"writing","name":"Writing","icon":"pen","color":"#cfacd4","scope":"Shape ideas into clear, engaging prose.","benchmarks":[{"id":"eq-writing","name":"EQ-Bench Creative Writing v3","source":"EQ-Bench","models":146,"latest_model_date":"2026-10-23","url":"https://eqbench.com/creative_writing.html"},{"id":"eq-longform","name":"EQ-Bench Longform Writing","source":"EQ-Bench","models":143,"latest_model_date":"2026-10-23","url":"https://eqbench.com/creative_writing_longform.html"},{"id":"writing-comparison","name":"Lech Mazur Writing comparison","source":"Lech Mazur","models":56,"latest_model_date":"2026-10-23","url":"https://github.com/lechmazur/writing"},{"id":"livebench-writing-20260625","name":"LiveBench: Instruction Following","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"note":"Creative prose, storytelling, and following constraints when rewriting or summarizing. Brand voice and factual accuracy need your own evaluation.","limited_models":126,"models":235},{"id":"reading","name":"Reading & documents","icon":"book","color":"#b8cfa0","scope":"Understand documents and extract the details you need.","benchmarks":[{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":39,"latest_model_date":"2026-10-13","url":"https://surgehq.ai/benchmarks/gdp-pdf"},{"id":"roboflow-reading","name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","models":52,"latest_model_date":"2026-10-26","url":"https://playground.roboflow.com/evals"},{"id":"livebench-reading-20260625","name":"LiveBench: Language + Data Analysis","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"},{"id":"extractbench","name":"ExtractBench: document extraction","source":"LlamaIndex","models":27,"latest_model_date":"2026-09-10","url":"https://github.com/run-llama/ExtractBench"}],"note":"PDF reasoning, document extraction, language comprehension, and structured tables. Full RAG pipelines, million-token recall, and citation accuracy are not measured.","limited_models":60,"models":106},{"id":"reasoning","name":"Reasoning","icon":"path","color":"#d4be8e","scope":"Work through unfamiliar problems and connect the clues.","benchmarks":[{"id":"simplebench","name":"SimpleBench","source":"Epoch AI","models":89,"latest_model_date":"2026-08-12","url":"https://epoch.ai/benchmarks"},{"id":"arc-agi-2","name":"ARC-AGI-2","source":"Epoch AI","models":107,"latest_model_date":"2026-10-17","url":"https://epoch.ai/benchmarks"},{"id":"chess-puzzles","name":"Chess Puzzles","source":"Epoch AI","models":101,"latest_model_date":"2026-11-02","url":"https://epoch.ai/benchmarks"},{"id":"mystery-game-puzzles","name":"Mystery Game Puzzles","source":"Epoch AI","models":89,"latest_model_date":"2026-10-26","url":"https://epoch.ai/benchmarks"},{"id":"livebench-reasoning-20260625","name":"LiveBench: Reasoning","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":103,"models":213},{"id":"math","name":"Math","icon":"math","color":"#8fc9c8","scope":"Solve quantitative problems, from competitions to proofs.","benchmarks":[{"id":"otis-mock-aime-2024-2025","name":"OTIS Mock AIME 2024-2025","source":"Epoch AI","models":224,"latest_model_date":"2026-11-02","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tiers-1-3-v2-private","name":"FrontierMath-Tiers-1-3-v2-Private","source":"Epoch AI","models":91,"latest_model_date":"2026-10-26","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tier-4-v2-private","name":"FrontierMath-Tier-4-v2-Private","source":"Epoch AI","models":73,"latest_model_date":"2026-10-26","url":"https://epoch.ai/benchmarks"},{"id":"proofbench","name":"ProofBench","source":"Epoch AI","models":83,"latest_model_date":"2026-10-23","url":"https://epoch.ai/benchmarks"},{"id":"livebench-math-20260625","name":"LiveBench: Mathematics","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":165,"models":266},{"id":"science","name":"Science","icon":"science","color":"#a7bbe0","scope":"Apply scientific knowledge to demanding questions and problems.","benchmarks":[{"id":"gpqa-diamond","name":"GPQA diamond","source":"Epoch AI","models":242,"latest_model_date":"2026-11-02","url":"https://epoch.ai/benchmarks"},{"id":"hle","name":"HLE","source":"Epoch AI","models":47,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"terminal-science","name":"Terminal-Bench Science 0.1","source":"Terminal-Bench Science","models":17,"latest_model_date":"2026-09-22","url":"https://www.terminal-bench-science.ai/"},{"id":"weirdml-v2","name":"WeirdML v2","source":"Håvard Tveit Ihle","models":133,"latest_model_date":"2026-09-04","url":"https://htihle.github.io/weirdml.html"}],"note":"Includes graduate science, cross-subject expert knowledge, research workflows, and machine-learning experiments. It isn't a measure of factual reliability in every field.","limited_models":222,"models":316},{"id":"vision","name":"Visual reasoning","icon":"eye","color":"#d3a78d","scope":"Read visual clues in photographs and diagrams.","benchmarks":[{"id":"roboflow-vision","name":"Vision Evals (low effort)","source":"Roboflow","models":52,"latest_model_date":"2026-10-26","url":"https://playground.roboflow.com/evals"},{"id":"blueprint-2","name":"Blueprint-Bench 2","source":"Andon Labs","models":37,"latest_model_date":"2026-10-26","url":"https://andonlabs.com/evals/blueprint-bench-2"},{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":39,"latest_model_date":"2026-10-13","url":"https://surgehq.ai/benchmarks/gdp-pdf"}],"note":"Image understanding, spatial reconstruction, and visual document reasoning. This does not measure image generation or video.","limited_models":28,"models":68},{"id":"computer","name":"Computer use","icon":"cursor","color":"#b9afdf","scope":"Complete tasks using software, files, and a terminal.","benchmarks":[{"id":"terminal-4","name":"Terminal-Bench 4.0","source":"Terminal-Bench","models":23,"latest_model_date":"2026-09-29","url":"https://www.tbench.ai/leaderboard/terminal-bench/4.0"},{"id":"apex-agents","name":"APEX-Agents","source":"Epoch AI","models":56,"latest_model_date":"2026-10-26","url":"https://epoch.ai/benchmarks"},{"id":"automationbench-1-0-6","name":"AutomationBench 1.0.6","source":"Zapier","models":53,"latest_model_date":"2026-10-13","url":"https://zapier.com/benchmarks"}],"note":"Terminal tasks, professional app work, and business workflows through APIs. Agent setups differ; scores describe the tested model and tools together.","limited_models":48,"models":84}],"weights":{"coding":{"code":3,"reasoning":1},"documents":{"reading":3,"vision":1}},"views":{"coding":{"leaders":[{"id":"catalog:anthropic/claude-fable-5-1","aliases":["catalog:anthropic/claude-fable-5-1","claudefable5.1","claudefable51"],"name":"Claude Fable 5.1","evidence":{"code":{"index":85.37693812116325,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.5091,"configuration":"claude-fable-5-1_medium","effort":"medium","agent":"claude-code","settings":{"Reasoning effort":"medium"},"date":"2026-09-01","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["claudefable5.1"],"index":68.03375261457094,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.8824,"configuration":"claude-fable-5-1_unknown","effort":"unknown","agent":"OpenHands","settings":{},"date":"2026-09-01","source_url":"https://gso-bench.github.io/index.html","standard_error":null,"aliases":["claudefable5.1"],"index":119.99102949976543,"benchmark":"gso-bench","benchmark_name":"GSO-Bench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.733,"configuration":"claude-fable-5-1_high","effort":"high","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.0917,"aliases":["claudefable5.1"],"index":75.48759977131522,"benchmark":"mirrorcode","benchmark_name":"MirrorCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":76.21783333333333,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"code_generation":90.141,"code_completion":82.609,"javascript":68.182,"typescript":60.0,"python":70.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":77.9953705990014,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":7},"reasoning":{"index":76.53655014665856,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.9,"configuration":"claude-fable-5-1_xhigh","effort":"xhigh","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["claudefable5.1"],"index":78.77746091045526,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.48,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.05021167315686782,"aliases":["claudefable5.1"],"index":75.77919514495562,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.58,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.04960449637488583,"aliases":["claudefable5.1"],"index":83.60692947155106,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":91.69225,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":100.0,"logic_with_navigation":86.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":67.98261505967226,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":5}},"catalog_id":"anthropic/claude-fable-5-1","open_weights":false,"date":"2026-09-01","score":83.17,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":8,"benchmark_total":12,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:deepseek/deepseek-v4.1-flash","aliases":["catalog:deepseek/deepseek-v4.1-flash","deepseekv4.1flash"],"name":"DeepSeek V4.1 Flash","evidence":{"code":{"index":72.20686917156539,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.4387,"configuration":"deepseek-v4.1-flash_high","effort":"high","agent":"devin-cli","settings":{"Reasoning effort":"high"},"date":"2026-09-09","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["deepseekv4.1flash"],"index":60.58668443023381,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":78.65483333333333,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"code_generation":77.465,"code_completion":82.609,"javascript":81.818,"typescript":60.0,"python":90.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":83.82705391289696,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":2,"total":7},"reasoning":{"index":68.15771622327182,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.7291666666666666,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["deepseekv4.1flash"],"index":70.85778871102224,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.48,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.05021167315686782,"aliases":["deepseekv4.1flash"],"index":75.77919514495562,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.43,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.0497569851956243,"aliases":["deepseekv4.1flash"],"index":69.11319659587228,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":86.69225,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":98.0,"logic_with_navigation":68.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":56.88068444123716,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":5}},"catalog_id":"deepseek/deepseek-v4.1-flash","open_weights":true,"date":"2026-09-10","score":71.19,"rank":6,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":12,"coverage":100,"missing":[]}]},"documents":{"leaders":[{"id":"catalog:openai/gpt-6.1-sol","aliases":["catalog:openai/gpt-6.1-sol","gpt6.1sol"],"name":"GPT-6.1 Sol","evidence":{"reading":{"index":72.91046220002745,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":89.965,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:05Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97},"aliases":["gpt6.1sol"],"index":56.716124419470546,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-26T21:16:44Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":86.39266666666667,"configuration":"gpt-6.1-sol-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":84.378,"typos":86.0,"consecutive_events":90.555,"tablejoin":57.423,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6.1sol"],"index":78.21351628114098,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":4},"vision":{"index":75.74850060198646,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":85.47333333333333,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:29Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97,"counting":78.83,"identification":89.58,"reasoning":83.66,"detection":80.84},"aliases":["gpt6.1sol"],"index":73.23607058698475,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-26T21:16:44Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":0.432,"configuration":"GPT-6.1 Sol","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["gpt6.1sol"],"index":70.20768531950382,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"openai/gpt-6.1-sol","open_weights":false,"date":"2026-09-29","score":73.62,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":5,"benchmark_total":6,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:moonshotai/kimi-k3","aliases":["catalog:moonshotai/kimi-k3","kimik3"],"name":"Kimi K3","evidence":{"reading":{"index":58.11538925262577,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":19.0,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":52.800665762384504,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"kimik3","name":"Kimi K3","score":88.77000000000001,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-23T23:46:57Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54},"aliases":["kimik3"],"index":52.20359634536896,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-26T21:16:44Z"},{"id":"kimik3","name":"Kimi K3","score":82.13083333333333,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":72.584,"typos":84.0,"consecutive_events":89.758,"tablejoin":48.404,"tablereformat":98.039},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":66.53930499003656,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null},{"id":"kimik3","name":"Kimi K3","score":83.17,"configuration":"Kimi K3","source_url":"https://github.com/run-llama/ExtractBench","standard_error":null,"agent":"ExtractBench publisher extraction pipeline","settings":{"Metric":"Unified value F1, mean over 370 documents (0–100)","Short (≤10 pages) F1":94.64,"Medium (11–50 pages) F1":69.64,"Long (>50 pages) F1":4.99},"aliases":["kimik3"],"index":60.91798991271306,"benchmark":"extractbench","benchmark_name":"ExtractBench: document extraction","source":"LlamaIndex","benchmark_url":"https://github.com/run-llama/ExtractBench","published_at":null}],"available":4,"total":4},"vision":{"index":49.33501491382376,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":66.50666666666667,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-25T19:51:00Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54,"counting":45.95,"identification":81.25,"reasoning":42.38,"detection":51.92},"aliases":["kimik3"],"index":38.89508622727415,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-26T21:16:44Z"},{"id":"kimik3","name":"Kimi K3","score":0.295,"configuration":"Kimi K3","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["kimik3"],"index":56.30929275181264,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"kimik3","name":"Kimi K3","score":19.0,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":52.800665762384504,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"moonshotai/kimi-k3","open_weights":true,"date":"2026-07-16","score":55.92,"rank":18,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":6,"coverage":100,"missing":[]}]}}},"after":{"method":"cadence-personal-4","checked_at":"2026-11-16T14:10:18.166525+00:00","revision":"05e577695317fb385e65bcfe6585eb142151b2459188b9e8fddeffc47db7d6cb","sources":[{"id":"epoch","ok":true,"checked_at":"2026-11-16T14:10:07.271773+00:00","updated_at":"2026-11-16T14:10:07.271773+00:00"},{"id":"writing-comparison","ok":true,"checked_at":"2026-11-16T14:10:07.275949+00:00","updated_at":"2026-11-16T14:10:07.275949+00:00"},{"id":"eq-writing","ok":true,"checked_at":"2026-11-16T14:10:07.279871+00:00","updated_at":"2026-11-16T14:10:07.279871+00:00"},{"id":"eq-longform","ok":true,"checked_at":"2026-11-16T14:10:07.281992+00:00","updated_at":"2026-11-16T14:10:07.281992+00:00"},{"id":"roboflow","ok":true,"checked_at":"2026-11-16T14:10:07.452753+00:00","updated_at":"2026-11-16T14:10:07.452753+00:00"},{"id":"blueprint","ok":true,"checked_at":"2026-11-16T14:10:07.517616+00:00","updated_at":"2026-11-16T14:10:07.517616+00:00"},{"id":"gdp-pdf","ok":true,"checked_at":"2026-11-16T14:10:07.627742+00:00","updated_at":"2026-11-16T14:10:07.627742+00:00"},{"id":"terminal-4","ok":false,"checked_at":"2026-11-16T14:10:07.713948+00:00","updated_at":"2026-11-09T16:10:13.314852+00:00","error":"ValueError"},{"id":"terminal-science","ok":true,"checked_at":"2026-11-16T14:10:07.738237+00:00","updated_at":"2026-11-16T14:10:07.738237+00:00"},{"id":"weirdml","ok":true,"checked_at":"2026-11-16T14:10:07.943310+00:00","updated_at":"2026-11-16T14:10:07.943310+00:00"},{"id":"livebench","ok":true,"checked_at":"2026-11-16T14:10:07.998090+00:00","updated_at":"2026-11-16T14:10:07.998090+00:00"},{"id":"extractbench","ok":true,"checked_at":"2026-11-16T14:10:08.088599+00:00","updated_at":"2026-11-16T14:10:08.088599+00:00"},{"id":"automationbench","ok":true,"checked_at":"2026-11-16T14:10:08.207086+00:00","updated_at":"2026-11-16T14:10:08.207086+00:00"}],"capabilities":[{"id":"code","name":"Coding","icon":"code","color":"#9ab8e8","scope":"Build software, fix bugs, and work across a codebase.","benchmarks":[{"id":"swe-bench-verified","name":"SWE-Bench verified","source":"Epoch AI","models":32,"latest_model_date":"2026-06-13","url":"https://epoch.ai/benchmarks"},{"id":"aider-polyglot","name":"Aider polyglot","source":"Epoch AI","models":55,"latest_model_date":"2025-12-01","url":"https://epoch.ai/benchmarks"},{"id":"deepswe","name":"DeepSWE","source":"Epoch AI","models":26,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"frontiercode","name":"FrontierCode","source":"Epoch AI","models":46,"latest_model_date":"2026-10-13","url":"https://epoch.ai/benchmarks"},{"id":"gso-bench","name":"GSO-Bench","source":"Epoch AI","models":35,"latest_model_date":"2026-09-04","url":"https://epoch.ai/benchmarks"},{"id":"mirrorcode","name":"MirrorCode","source":"Epoch AI","models":39,"latest_model_date":"2026-10-26","url":"https://epoch.ai/benchmarks"},{"id":"livebench-code-20260625","name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":86,"models":155},{"id":"writing","name":"Writing","icon":"pen","color":"#cfacd4","scope":"Shape ideas into clear, engaging prose.","benchmarks":[{"id":"eq-writing","name":"EQ-Bench Creative Writing v3","source":"EQ-Bench","models":146,"latest_model_date":"2026-10-23","url":"https://eqbench.com/creative_writing.html"},{"id":"eq-longform","name":"EQ-Bench Longform Writing","source":"EQ-Bench","models":143,"latest_model_date":"2026-10-23","url":"https://eqbench.com/creative_writing_longform.html"},{"id":"writing-comparison","name":"Lech Mazur Writing comparison","source":"Lech Mazur","models":60,"latest_model_date":"2026-11-10","url":"https://github.com/lechmazur/writing"},{"id":"livebench-writing-20260625","name":"LiveBench: Instruction Following","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"note":"Creative prose, storytelling, and following constraints when rewriting or summarizing. Brand voice and factual accuracy need your own evaluation.","limited_models":130,"models":239},{"id":"reading","name":"Reading & documents","icon":"book","color":"#b8cfa0","scope":"Understand documents and extract the details you need.","benchmarks":[{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":21,"latest_model_date":"2026-11-10","url":"https://surgehq.ai/benchmarks/gdp-pdf"},{"id":"roboflow-reading","name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","models":56,"latest_model_date":"2026-11-12","url":"https://playground.roboflow.com/evals"},{"id":"livebench-reading-20260625","name":"LiveBench: Language + Data Analysis","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"},{"id":"extractbench","name":"ExtractBench: document extraction","source":"LlamaIndex","models":27,"latest_model_date":"2026-09-10","url":"https://github.com/run-llama/ExtractBench"}],"note":"PDF reasoning, document extraction, language comprehension, and structured tables. Full RAG pipelines, million-token recall, and citation accuracy are not measured.","limited_models":67,"models":110},{"id":"reasoning","name":"Reasoning","icon":"path","color":"#d4be8e","scope":"Work through unfamiliar problems and connect the clues.","benchmarks":[{"id":"simplebench","name":"SimpleBench","source":"Epoch AI","models":111,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"arc-agi-2","name":"ARC-AGI-2","source":"Epoch AI","models":110,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"chess-puzzles","name":"Chess Puzzles","source":"Epoch AI","models":103,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"mystery-game-puzzles","name":"Mystery Game Puzzles","source":"Epoch AI","models":91,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"livebench-reasoning-20260625","name":"LiveBench: Reasoning","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":106,"models":221},{"id":"math","name":"Math","icon":"math","color":"#8fc9c8","scope":"Solve quantitative problems, from competitions to proofs.","benchmarks":[{"id":"otis-mock-aime-2024-2025","name":"OTIS Mock AIME 2024-2025","source":"Epoch AI","models":226,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tiers-1-3-v2-private","name":"FrontierMath-Tiers-1-3-v2-Private","source":"Epoch AI","models":93,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"frontiermath-tier-4-v2-private","name":"FrontierMath-Tier-4-v2-Private","source":"Epoch AI","models":75,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"proofbench","name":"ProofBench","source":"Epoch AI","models":84,"latest_model_date":"2026-11-09","url":"https://epoch.ai/benchmarks"},{"id":"livebench-math-20260625","name":"LiveBench: Mathematics","source":"LiveBench","models":67,"latest_model_date":"2026-10-12","url":"https://livebench.ai/"}],"limited_models":165,"models":268},{"id":"science","name":"Science","icon":"science","color":"#a7bbe0","scope":"Apply scientific knowledge to demanding questions and problems.","benchmarks":[{"id":"gpqa-diamond","name":"GPQA diamond","source":"Epoch AI","models":247,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"hle","name":"HLE","source":"Epoch AI","models":47,"latest_model_date":"2026-10-07","url":"https://epoch.ai/benchmarks"},{"id":"terminal-science","name":"Terminal-Bench Science 0.1","source":"Terminal-Bench Science","models":17,"latest_model_date":"2026-09-22","url":"https://www.terminal-bench-science.ai/"},{"id":"weirdml-v2","name":"WeirdML v2","source":"Håvard Tveit Ihle","models":133,"latest_model_date":"2026-09-04","url":"https://htihle.github.io/weirdml.html"}],"note":"Includes graduate science, cross-subject expert knowledge, research workflows, and machine-learning experiments. It isn't a measure of factual reliability in every field.","limited_models":227,"models":321},{"id":"vision","name":"Visual reasoning","icon":"eye","color":"#d3a78d","scope":"Read visual clues in photographs and diagrams.","benchmarks":[{"id":"roboflow-vision","name":"Vision Evals (low effort)","source":"Roboflow","models":56,"latest_model_date":"2026-11-12","url":"https://playground.roboflow.com/evals"},{"id":"blueprint-2","name":"Blueprint-Bench 2","source":"Andon Labs","models":38,"latest_model_date":"2026-11-10","url":"https://andonlabs.com/evals/blueprint-bench-2"},{"id":"gdp-pdf","name":"GDP.pdf (Surge AI)","source":"Surge AI","models":21,"latest_model_date":"2026-11-10","url":"https://surgehq.ai/benchmarks/gdp-pdf"}],"note":"Image understanding, spatial reconstruction, and visual document reasoning. This does not measure image generation or video.","limited_models":40,"models":73},{"id":"computer","name":"Computer use","icon":"cursor","color":"#b9afdf","scope":"Complete tasks using software, files, and a terminal.","benchmarks":[{"id":"terminal-4","name":"Terminal-Bench 4.0","source":"Terminal-Bench","models":23,"latest_model_date":"2026-09-29","url":"https://www.tbench.ai/leaderboard/terminal-bench/4.0"},{"id":"apex-agents","name":"APEX-Agents","source":"Epoch AI","models":61,"latest_model_date":"2026-11-10","url":"https://epoch.ai/benchmarks"},{"id":"automationbench-1-0-6","name":"AutomationBench 1.0.6","source":"Zapier","models":59,"latest_model_date":"2026-11-12","url":"https://zapier.com/benchmarks"}],"note":"Terminal tasks, professional app work, and business workflows through APIs. Agent setups differ; scores describe the tested model and tools together.","limited_models":57,"models":94}],"weights":{"coding":{"code":3,"reasoning":1},"documents":{"reading":3,"vision":1}},"views":{"coding":{"leaders":[{"id":"catalog:anthropic/claude-fable-5-1","aliases":["catalog:anthropic/claude-fable-5-1","claudefable5.1","claudefable51"],"name":"Claude Fable 5.1","evidence":{"code":{"index":88.59885092966647,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.5091,"configuration":"claude-fable-5-1_medium","effort":"medium","agent":"claude-code","settings":{"Reasoning effort":"medium"},"date":"2026-09-01","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["claudefable5.1"],"index":68.03375261457094,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.8824,"configuration":"claude-fable-5-1_unknown","effort":"unknown","agent":"OpenHands","settings":{},"date":"2026-09-01","source_url":"https://gso-bench.github.io/index.html","standard_error":null,"aliases":["claudefable5.1"],"index":119.99102949976543,"benchmark":"gso-bench","benchmark_name":"GSO-Bench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.9333333333333333,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.03917131009286688,"aliases":["claudefable5.1"],"index":88.37525100532807,"benchmark":"mirrorcode","benchmark_name":"MirrorCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":76.21783333333333,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"code_generation":90.141,"code_completion":82.609,"javascript":68.182,"typescript":60.0,"python":70.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":77.9953705990014,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":4,"total":7},"reasoning":{"index":77.55695860245187,"limited":false,"components":[{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.866,"configuration":"claude-fable-5-1_unknown","effort":"unknown","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://simple-bench.com/","standard_error":null,"aliases":["claudefable5.1"],"index":81.63859242562519,"benchmark":"simplebench","benchmark_name":"SimpleBench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.9,"configuration":"claude-fable-5-1_xhigh","effort":"xhigh","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["claudefable5.1"],"index":78.77746091045526,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.48,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.05021167315686782,"aliases":["claudefable5.1"],"index":75.77919514495562,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":0.58,"configuration":"claude-fable-5-1_max","effort":"max","agent":null,"settings":{},"date":"2026-09-01","source_url":"https://epoch.ai/benchmarks","standard_error":0.04960449637488583,"aliases":["claudefable5.1"],"index":83.60692947155106,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"claudefable5.1","name":"Claude Fable 5.1","score":91.69225,"configuration":"claude-fable-5-1-max-effort","source_url":"https://livebench.ai/","standard_error":null,"effort":"max-effort","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":100.0,"logic_with_navigation":86.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["claudefable5.1","claudefable51"],"index":67.98261505967226,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":5,"total":5}},"catalog_id":"anthropic/claude-fable-5-1","open_weights":false,"date":"2026-09-01","score":85.84,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":9,"benchmark_total":12,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:deepseek/deepseek-v4.1-flash","aliases":["catalog:deepseek/deepseek-v4.1-flash","deepseekv4.1flash"],"name":"DeepSeek V4.1 Flash","evidence":{"code":{"index":71.40148910830797,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.4387,"configuration":"deepseek-v4.1-flash_high","effort":"high","agent":"devin-cli","settings":{"Reasoning effort":"high"},"date":"2026-09-09","source_url":"https://cognition.com/frontiercode","standard_error":null,"aliases":["deepseekv4.1flash"],"index":60.58668443023381,"benchmark":"frontiercode","benchmark_name":"FrontierCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.6444444444444445,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.11369999821616648,"aliases":["deepseekv4.1flash"],"index":69.79072898179317,"benchmark":"mirrorcode","benchmark_name":"MirrorCode","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":78.65483333333333,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"code_generation":77.465,"code_completion":82.609,"javascript":81.818,"typescript":60.0,"python":90.0},"settings":{"Release":"2026-06-25","Categories":"Coding, Agentic Coding","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":83.82705391289696,"benchmark":"livebench-code-20260625","benchmark_name":"LiveBench: Coding + Agentic Coding","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":7},"reasoning":{"index":67.55256227147073,"limited":false,"components":[{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.667,"configuration":"deepseek-v4.1-flash_unknown","effort":"unknown","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://simple-bench.com/","standard_error":null,"aliases":["deepseekv4.1flash"],"index":65.13194646426638,"benchmark":"simplebench","benchmark_name":"SimpleBench","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.7291666666666666,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":null,"aliases":["deepseekv4.1flash"],"index":70.85778871102224,"benchmark":"arc-agi-2","benchmark_name":"ARC-AGI-2","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.48,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.05021167315686782,"aliases":["deepseekv4.1flash"],"index":75.77919514495562,"benchmark":"chess-puzzles","benchmark_name":"Chess Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":0.43,"configuration":"deepseek-v4.1-flash_max","effort":"max","agent":null,"settings":{},"date":"2026-09-09","source_url":"https://epoch.ai/benchmarks","standard_error":0.0497569851956243,"aliases":["deepseekv4.1flash"],"index":69.11319659587228,"benchmark":"mystery-game-puzzles","benchmark_name":"Mystery Game Puzzles","source":"Epoch AI","benchmark_url":"https://epoch.ai/benchmarks","published_at":null},{"id":"deepseekv4.1flash","name":"DeepSeek V4.1 Flash","score":86.69225,"configuration":"deepseek-v4.1-flash-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"theory_of_mind":80.769,"zebra_puzzle":100.0,"spatial":98.0,"logic_with_navigation":68.0},"settings":{"Release":"2026-06-25","Categories":"Reasoning","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["deepseekv4.1flash"],"index":56.88068444123716,"benchmark":"livebench-reasoning-20260625","benchmark_name":"LiveBench: Reasoning","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":5,"total":5}},"catalog_id":"deepseek/deepseek-v4.1-flash","open_weights":true,"date":"2026-09-10","score":70.44,"rank":7,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":8,"benchmark_total":12,"coverage":100,"missing":[]}]},"documents":{"leaders":[{"id":"catalog:meta/muse-ember-1.0","aliases":["catalog:meta/muse-ember-1.0","museember","museember1.0"],"name":"Muse Ember","evidence":{"reading":{"index":88.51875210910288,"limited":false,"components":[{"id":"museember","name":"Muse Ember","score":49.2,"configuration":"Muse Ember 1.0 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["museember","museember1.0"],"index":111.31520452113494,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"museember","name":"Muse Ember","score":92.35,"configuration":"muse-ember:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-11-10T07:44:28Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"Muse Ember"},"subscores":{"ocr":93.29,"extraction":91.41},"aliases":["museember"],"index":65.7222996970708,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"}],"available":2,"total":4},"vision":{"index":90.04105944172542,"limited":false,"components":[{"id":"museember","name":"Muse Ember","score":83.005,"configuration":"muse-ember:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-11-10T07:44:52Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"Muse Ember"},"subscores":{"ocr":93.29,"extraction":91.41,"counting":79.73,"identification":91.67,"reasoning":78.15,"detection":63.78},"aliases":["museember"],"index":68.76691436231592,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"},{"id":"museember","name":"Muse Ember","score":49.2,"configuration":"Muse Ember 1.0 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["museember","museember1.0"],"index":111.31520452113494,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":2,"total":3}},"catalog_id":"meta/muse-ember-1.0","open_weights":false,"date":"2026-11-09","score":88.9,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":3,"benchmark_total":6,"coverage":100,"missing":[]}],"open_leaders":[{"id":"catalog:moonshotai/kimi-k3","aliases":["catalog:moonshotai/kimi-k3","kimik3"],"name":"Kimi K3","evidence":{"reading":{"index":68.19074029717882,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":39.8,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":93.10206994059672,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"kimik3","name":"Kimi K3","score":88.77000000000001,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-23T23:46:57Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54},"aliases":["kimik3"],"index":52.20359634536896,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"},{"id":"kimik3","name":"Kimi K3","score":82.13083333333333,"configuration":"kimi-k3","source_url":"https://livebench.ai/","standard_error":null,"effort":null,"agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":72.584,"typos":84.0,"consecutive_events":89.758,"tablejoin":48.404,"tablereformat":98.039},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["kimik3"],"index":66.53930499003656,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null},{"id":"kimik3","name":"Kimi K3","score":83.17,"configuration":"Kimi K3","source_url":"https://github.com/run-llama/ExtractBench","standard_error":null,"agent":"ExtractBench publisher extraction pipeline","settings":{"Metric":"Unified value F1, mean over 370 documents (0–100)","Short (≤10 pages) F1":94.64,"Medium (11–50 pages) F1":69.64,"Long (>50 pages) F1":4.99},"aliases":["kimik3"],"index":60.91798991271306,"benchmark":"extractbench","benchmark_name":"ExtractBench: document extraction","source":"LlamaIndex","benchmark_url":"https://github.com/run-llama/ExtractBench","published_at":null}],"available":4,"total":4},"vision":{"index":62.76881630656117,"limited":false,"components":[{"id":"kimik3","name":"Kimi K3","score":66.50666666666667,"configuration":"kimi-k3:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-07-25T19:51:00Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"Kimi K3"},"subscores":{"ocr":93.0,"extraction":84.54,"counting":45.95,"identification":81.25,"reasoning":42.38,"detection":51.92},"aliases":["kimik3"],"index":38.89508622727415,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"},{"id":"kimik3","name":"Kimi K3","score":0.295,"configuration":"Kimi K3","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["kimik3"],"index":56.30929275181264,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"kimik3","name":"Kimi K3","score":39.8,"configuration":"Kimi K3 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["kimik3"],"index":93.10206994059672,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"moonshotai/kimi-k3","open_weights":true,"date":"2026-07-16","score":66.84,"rank":13,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":6,"benchmark_total":6,"coverage":100,"missing":[]}]}}},"changes":[{"guide":"documents","lane":"leaders","before":[{"id":"catalog:openai/gpt-6.1-sol","aliases":["catalog:openai/gpt-6.1-sol","gpt6.1sol"],"name":"GPT-6.1 Sol","evidence":{"reading":{"index":72.91046220002745,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":89.965,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:05Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97},"aliases":["gpt6.1sol"],"index":56.716124419470546,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-26T21:16:44Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":86.39266666666667,"configuration":"gpt-6.1-sol-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":84.378,"typos":86.0,"consecutive_events":90.555,"tablejoin":57.423,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6.1sol"],"index":78.21351628114098,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":4},"vision":{"index":75.74850060198646,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":85.47333333333333,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:29Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97,"counting":78.83,"identification":89.58,"reasoning":83.66,"detection":80.84},"aliases":["gpt6.1sol"],"index":73.23607058698475,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-10-26T21:16:44Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":0.432,"configuration":"GPT-6.1 Sol","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["gpt6.1sol"],"index":70.20768531950382,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":35.0,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":83.80174589947083,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"openai/gpt-6.1-sol","open_weights":false,"date":"2026-09-29","score":73.62,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":5,"benchmark_total":6,"coverage":100,"missing":[]}],"after":[{"id":"catalog:meta/muse-ember-1.0","aliases":["catalog:meta/muse-ember-1.0","museember","museember1.0"],"name":"Muse Ember","evidence":{"reading":{"index":88.51875210910288,"limited":false,"components":[{"id":"museember","name":"Muse Ember","score":49.2,"configuration":"Muse Ember 1.0 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["museember","museember1.0"],"index":111.31520452113494,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"museember","name":"Muse Ember","score":92.35,"configuration":"muse-ember:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-11-10T07:44:28Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"Muse Ember"},"subscores":{"ocr":93.29,"extraction":91.41},"aliases":["museember"],"index":65.7222996970708,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"}],"available":2,"total":4},"vision":{"index":90.04105944172542,"limited":false,"components":[{"id":"museember","name":"Muse Ember","score":83.005,"configuration":"muse-ember:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-11-10T07:44:52Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"Muse Ember"},"subscores":{"ocr":93.29,"extraction":91.41,"counting":79.73,"identification":91.67,"reasoning":78.15,"detection":63.78},"aliases":["museember"],"index":68.76691436231592,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"},{"id":"museember","name":"Muse Ember","score":49.2,"configuration":"Muse Ember 1.0 (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["museember","museember1.0"],"index":111.31520452113494,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":2,"total":3}},"catalog_id":"meta/muse-ember-1.0","open_weights":false,"date":"2026-11-09","score":88.9,"rank":1,"delta":null,"shared_delta":null,"shared_tests":0,"limited":[],"benchmark_count":3,"benchmark_total":6,"coverage":100,"missing":[]}],"comparisons":[{"previous":"GPT-6.1 Sol","candidate":"Muse Ember","delta":0.49,"tests":3,"current_previous":{"id":"catalog:openai/gpt-6.1-sol","aliases":["catalog:openai/gpt-6.1-sol","gpt6.1sol"],"name":"GPT-6.1 Sol","evidence":{"reading":{"index":83.6316690807698,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":51.6,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":115.96536654169788,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":89.965,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:05Z","settings":{"Tasks":"ocr, extraction","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97},"aliases":["gpt6.1sol"],"index":56.716124419470546,"benchmark":"roboflow-reading","benchmark_name":"Vision Evals: OCR & extraction (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":86.39266666666667,"configuration":"gpt-6.1-sol-max","source_url":"https://livebench.ai/","standard_error":null,"effort":"max","agent":"LiveBench published evaluation","subscores":{"connections":100.0,"plot_unscrambling":84.378,"typos":86.0,"consecutive_events":90.555,"tablejoin":57.423,"tablereformat":100.0},"settings":{"Release":"2026-06-25","Categories":"Language, Data Analysis","Metric":"Equal category means of objective task scores (0–100)"},"aliases":["gpt6.1sol"],"index":78.21351628114098,"benchmark":"livebench-reading-20260625","benchmark_name":"LiveBench: Language + Data Analysis","source":"LiveBench","benchmark_url":"https://livebench.ai/","published_at":null}],"available":3,"total":4},"vision":{"index":86.46970748272882,"limited":false,"components":[{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":85.47333333333333,"configuration":"gpt-6.1-sol:low","source_url":"https://playground.roboflow.com/evals","standard_error":null,"effort":"low","agent":"Roboflow VLM Exam","evaluated_at":"2026-09-29T17:52:29Z","settings":{"Tasks":"ocr, extraction, counting, identification, reasoning, detection","Judge":"gemini-3.5-flash","Publisher label":"GPT-6.1 Sol"},"subscores":{"ocr":91.96,"extraction":87.97,"counting":78.83,"identification":89.58,"reasoning":83.66,"detection":80.84},"aliases":["gpt6.1sol"],"index":73.23607058698475,"benchmark":"roboflow-vision","benchmark_name":"Vision Evals (low effort)","source":"Roboflow","benchmark_url":"https://playground.roboflow.com/evals","published_at":"2026-11-12T23:58:06Z"},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":0.432,"configuration":"GPT-6.1 Sol","source_url":"https://andonlabs.com/evals/blueprint-bench-2","standard_error":null,"agent":"Andon Labs persistent-notepad agent","settings":{"Metric":"Connectivity similarity, random baseline normalized to 0"},"aliases":["gpt6.1sol"],"index":70.20768531950382,"benchmark":"blueprint-2","benchmark_name":"Blueprint-Bench 2","source":"Andon Labs","benchmark_url":"https://andonlabs.com/evals/blueprint-bench-2","published_at":null},{"id":"gpt6.1sol","name":"GPT-6.1 Sol","score":51.6,"configuration":"GPT 6.1 Sol (Max reasoning)","source_url":"https://surgehq.ai/benchmarks/gdp-pdf","standard_error":null,"effort":"max reasoning","agent":"Surge AI document reasoning evaluation","settings":{"Metric":"Publisher leaderboard score (%)"},"aliases":["gpt6.1sol"],"index":115.96536654169788,"benchmark":"gdp-pdf","benchmark_name":"GDP.pdf (Surge AI)","source":"Surge AI","benchmark_url":"https://surgehq.ai/benchmarks/gdp-pdf","published_at":null}],"available":3,"total":3}},"catalog_id":"openai/gpt-6.1-sol","open_weights":false,"date":"2026-09-29","score":84.34,"rank":3,"delta":0.0,"shared_delta":0.0,"shared_tests":5,"limited":[],"benchmark_count":5,"benchmark_total":6,"coverage":100,"missing":[]}}]}]}