{"schemaVersion":1,"selectionPolicy":"/llms.txt","batches":[{"id":4,"recordedAt":"2026-11-10T01:18:20.032Z","kind":"update","summary":"179 dataset changes recorded","changeCount":179,"changes":[{"id":45,"type":"benchmark","entityId":"hle-full","action":"updated","label":"Humanity’s Last Exam","modelId":null,"benchmarkSlug":"humanitys-last-exam","source":"https://lastexam.ai/","previous":{"benchmark_homepage_url":"https://lastexam.ai/","category":"Knowledge and reasoning","created_at":"2026-11-02T18:16:18.837Z","description":"Published HLE accuracy. Tool access and settings are recorded per result. Not HLE-Diamond, HLE-Rolling or HLE-Verified.","display_name":"Humanity’s Last Exam","higher_is_better":1,"id":"hle-full","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Humanity's Last Exam","official_url":"https://lastexam.ai/","page_slug":"humanitys-last-exam","summary":"Humanity’s Last Exam evaluates advanced knowledge and reasoning across a broad range of academic subjects using expert-written questions. It includes text and multimodal problems designed to challenge frontier AI systems. Results identify whether the full exam or a text-only subset was used.","tracking_status":"tracked","unit":"%","version":"Stable HLE — full set"},"current":{"benchmark_homepage_url":"https://lastexam.ai/","category":"Knowledge and reasoning","created_at":"2026-11-02T18:16:18.837Z","description":"Published HLE accuracy. Tool access and settings are recorded per result. Not HLE-Diamond, HLE-Rolling or HLE-Verified.","display_name":"Humanity’s Last Exam","higher_is_better":1,"id":"hle-full","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Humanity's Last Exam","official_url":"https://lastexam.ai/","page_slug":"humanitys-last-exam","summary":"Tests whether a model can work through difficult expert-level questions, helping you choose an assistant for demanding research and technical analysis. Full-exam and text-only results are separate, and exam accuracy does not measure client-project completion.","tracking_status":"tracked","unit":"%","version":"Stable HLE — full set"}},{"id":46,"type":"benchmark","entityId":"hle-text","action":"updated","label":"Humanity’s Last Exam","modelId":null,"benchmarkSlug":"humanitys-last-exam","source":"https://lastexam.ai/","previous":{"benchmark_homepage_url":"https://lastexam.ai/","category":"Knowledge and reasoning","created_at":"2026-11-02T18:16:18.837Z","description":"Published HLE accuracy. Tool access and settings are recorded per result. Not HLE-Diamond, HLE-Rolling or HLE-Verified.","display_name":"Humanity’s Last Exam","higher_is_better":1,"id":"hle-text","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Humanity's Last Exam","official_url":"https://lastexam.ai/","page_slug":"humanitys-last-exam","summary":"Humanity’s Last Exam evaluates advanced knowledge and reasoning across a broad range of academic subjects using expert-written questions. It includes text and multimodal problems designed to challenge frontier AI systems. Results identify whether the full exam or a text-only subset was used.","tracking_status":"tracked","unit":"%","version":"Stable HLE — text-only subset"},"current":{"benchmark_homepage_url":"https://lastexam.ai/","category":"Knowledge and reasoning","created_at":"2026-11-02T18:16:18.837Z","description":"Published HLE accuracy. Tool access and settings are recorded per result. Not HLE-Diamond, HLE-Rolling or HLE-Verified.","display_name":"Humanity’s Last Exam","higher_is_better":1,"id":"hle-text","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Humanity's Last Exam","official_url":"https://lastexam.ai/","page_slug":"humanitys-last-exam","summary":"Tests whether a model can work through difficult expert-level questions, helping you choose an assistant for demanding research and technical analysis. Full-exam and text-only results are separate, and exam accuracy does not measure client-project completion.","tracking_status":"tracked","unit":"%","version":"Stable HLE — text-only subset"}},{"id":47,"type":"benchmark","entityId":"hle-unspecified","action":"updated","label":"Humanity’s Last Exam","modelId":null,"benchmarkSlug":"humanitys-last-exam","source":"https://lastexam.ai/","previous":{"benchmark_homepage_url":"https://lastexam.ai/","category":"Knowledge and reasoning","created_at":"2026-11-02T18:16:18.837Z","description":"Published HLE accuracy. Tool access and settings are recorded per result. Not HLE-Diamond, HLE-Rolling or HLE-Verified.","display_name":"Humanity’s Last Exam","higher_is_better":1,"id":"hle-unspecified","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Humanity's Last Exam","official_url":"https://lastexam.ai/","page_slug":"humanitys-last-exam","summary":"Humanity’s Last Exam evaluates advanced knowledge and reasoning across a broad range of academic subjects using expert-written questions. It includes text and multimodal problems designed to challenge frontier AI systems. Results identify whether the full exam or a text-only subset was used.","tracking_status":"tracked","unit":"%","version":"HLE — source does not specify subset"},"current":{"benchmark_homepage_url":"https://lastexam.ai/","category":"Knowledge and reasoning","created_at":"2026-11-02T18:16:18.837Z","description":"Published HLE accuracy. Tool access and settings are recorded per result. Not HLE-Diamond, HLE-Rolling or HLE-Verified.","display_name":"Humanity’s Last Exam","higher_is_better":1,"id":"hle-unspecified","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Humanity's Last Exam","official_url":"https://lastexam.ai/","page_slug":"humanitys-last-exam","summary":"Tests whether a model can work through difficult expert-level questions, helping you choose an assistant for demanding research and technical analysis. Full-exam and text-only results are separate, and exam accuracy does not measure client-project completion.","tracking_status":"tracked","unit":"%","version":"HLE — source does not specify subset"}},{"id":48,"type":"benchmark","entityId":"deepswe-v1-1","action":"updated","label":"DeepSWE","modelId":null,"benchmarkSlug":"deepswe","source":"https://deepswe.datacurve.ai/","previous":{"benchmark_homepage_url":"https://deepswe.datacurve.ai/","category":"Coding agents","created_at":"2026-11-02T19:49:15.742Z","description":"Official v1.1 leaderboard. Best reported effort per model; scores rounded by source.","display_name":"DeepSWE","higher_is_better":1,"id":"deepswe-v1-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"DeepSWE","official_url":"https://deepswe.datacurve.ai/","page_slug":"deepswe","summary":"DeepSWE evaluates coding agents on original, long-horizon software engineering tasks. It tests whether an agent can understand a codebase, make changes, and complete engineering work beyond short coding questions.","tracking_status":"tracked","unit":"%","version":"v1.1"},"current":{"benchmark_homepage_url":"https://deepswe.datacurve.ai/","category":"Coding agents","created_at":"2026-11-02T19:49:15.742Z","description":"Official v1.1 leaderboard. Best reported effort per model; scores rounded by source.","display_name":"DeepSWE","higher_is_better":1,"id":"deepswe-v1-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"DeepSWE","official_url":"https://deepswe.datacurve.ai/","page_slug":"deepswe","summary":"Tests whether an agent can understand a codebase and complete substantial engineering changes, helping you assess which model could take on more development work while you focus on requirements and review.","tracking_status":"tracked","unit":"%","version":"v1.1"}},{"id":49,"type":"benchmark","entityId":"terminal-bench-4-0","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-02T19:49:15.743Z","description":"Official leaderboard results; native agent harnesses and efforts vary, as recorded per report.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-4-0","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"4.0"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-02T19:49:15.743Z","description":"Official leaderboard results; native agent harnesses and efforts vary, as recorded per report.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-4-0","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"4.0"}},{"id":50,"type":"benchmark","entityId":"automationbench-aa-1-0-6","action":"updated","label":"AutomationBench-AA","modelId":null,"benchmarkSlug":"automationbench-aa","source":"https://artificialanalysis.ai/evaluations/automationbench-aa","previous":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/automationbench-aa","category":"Work automations","created_at":"2026-11-02T19:49:15.743Z","description":"Objective-level score, with zero credit for a task that violates a guardrail. 50-turn cap.","display_name":"AutomationBench-AA","higher_is_better":1,"id":"automationbench-aa-1-0-6","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Objectives completed without guardrail violations","minimum_score":0,"name":"AutomationBench-AA","official_url":"https://artificialanalysis.ai/evaluations/automationbench-aa","page_slug":"automationbench-aa","summary":"AutomationBench-AA evaluates agents working across simulated business applications. Its score measures the share of task objectives completed without guardrail violations. This objective-level metric is separate from the strict task-completion score on Zapier’s hosted AutomationBench leaderboard.","tracking_status":"tracked","unit":"%","version":"1.0.6 / AA private 657"},"current":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/automationbench-aa","category":"Work automations","created_at":"2026-11-02T19:49:15.743Z","description":"Objective-level score, with zero credit for a task that violates a guardrail. 50-turn cap.","display_name":"AutomationBench-AA","higher_is_better":1,"id":"automationbench-aa-1-0-6","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Objectives completed without guardrail violations","minimum_score":0,"name":"AutomationBench-AA","official_url":"https://artificialanalysis.ai/evaluations/automationbench-aa","page_slug":"automationbench-aa","summary":"Tests how much of a business workflow an agent can complete without breaking its rules, helping you identify opportunities to reduce manual work across apps. Scores measure objectives completed, not the percentage of entire workflows successfully finished.","tracking_status":"tracked","unit":"%","version":"1.0.6 / AA private 657"}},{"id":51,"type":"benchmark","entityId":"mcp-atlas-public-apr2026","action":"updated","label":"MCP-Atlas","modelId":null,"benchmarkSlug":"mcp-atlas","source":"https://labs.scale.com/leaderboard/mcp_atlas","previous":{"benchmark_homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","category":"Tool use","created_at":"2026-11-02T19:49:15.743Z","description":"April 2026 protocol. Public-subset scores only; all-task scores are kept separate.","display_name":"MCP-Atlas","higher_is_better":1,"id":"mcp-atlas-public-apr2026","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"MCP-Atlas","official_url":"https://labs.scale.com/leaderboard/mcp_atlas","page_slug":"mcp-atlas","summary":"MCP-Atlas evaluates real-world tool use through the Model Context Protocol. It tests tool discovery, choosing correct arguments, recovering from errors, and combining tool outputs to complete multi-step workflows. The stored results below cover the public subset under the April 2026 protocol.","tracking_status":"tracked","unit":"%","version":"April 2026 / public 500"},"current":{"benchmark_homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","category":"Tool use","created_at":"2026-11-02T19:49:15.743Z","description":"April 2026 protocol. Public-subset scores only; all-task scores are kept separate.","display_name":"MCP-Atlas","higher_is_better":1,"id":"mcp-atlas-public-apr2026","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"MCP-Atlas","official_url":"https://labs.scale.com/leaderboard/mcp_atlas","page_slug":"mcp-atlas","summary":"Tests whether an agent can find and use the right connected tools, helping you assess its ability to complete workflows across your business systems with less manual coordination. These results cover the public April 2026 subset.","tracking_status":"tracked","unit":"%","version":"April 2026 / public 500"}},{"id":52,"type":"benchmark","entityId":"osworld-2-1-full","action":"updated","label":"OSWorld","modelId":null,"benchmarkSlug":"osworld","source":"https://osworld-v2.xlang.ai/","previous":{"benchmark_homepage_url":"https://osworld-v2.xlang.ai/","category":"Computer use","created_at":"2026-11-02T19:49:15.744Z","description":"3 models reported on v2.1. Best effort per model; older releases are excluded.","display_name":"OSWorld","higher_is_better":1,"id":"osworld-2-1-full","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Binary accuracy","minimum_score":0,"name":"OSWorld","official_url":"https://osworld-v2.xlang.ai/","page_slug":"osworld","summary":"OSWorld evaluates computer-use agents on practical tasks in a desktop environment. It tests whether agents can interact with applications and complete multi-step work through a graphical interface. Results below are restricted to the stored 2.1 full-set evaluation.","tracking_status":"tracked","unit":"%","version":"2.1 / full set"},"current":{"benchmark_homepage_url":"https://osworld-v2.xlang.ai/","category":"Computer use","created_at":"2026-11-02T19:49:15.744Z","description":"3 models reported on v2.1. Best effort per model; older releases are excluded.","display_name":"OSWorld","higher_is_better":1,"id":"osworld-2-1-full","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Binary accuracy","minimum_score":0,"name":"OSWorld","official_url":"https://osworld-v2.xlang.ai/","page_slug":"osworld","summary":"Tests whether an agent can carry out work through desktop applications, helping you assess which computer tasks you might delegate instead of doing every click yourself. Full-set, offline-subset, partial-credit and strict-success results measure different things.","tracking_status":"tracked","unit":"%","version":"2.1 / full set"}},{"id":53,"type":"benchmark","entityId":"browsecomp-kimi31-comparison","action":"updated","label":"BrowseComp","modelId":null,"benchmarkSlug":"browsecomp","source":"https://openai.com/index/browsecomp/","previous":{"benchmark_homepage_url":"https://openai.com/index/browsecomp/","category":"Web research","created_at":"2026-11-02T19:49:15.744Z","description":"Shared setup: 200K context compaction, 500-step cap, temperature 1.","display_name":"BrowseComp","higher_is_better":1,"id":"browsecomp-kimi31-comparison","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"BrowseComp","official_url":"https://www.kimi.ai/blog/kimi-k3-1-pro","page_slug":"browsecomp","summary":"BrowseComp evaluates web research agents on questions with hard-to-find but verifiable answers. It tests persistent browsing, search, and the ability to connect information from multiple sources. Tool access, search budgets, and context management can differ across published comparison reports.","tracking_status":"tracked","unit":"%","version":"Kimi K3.1 comparison / 200K compaction"},"current":{"benchmark_homepage_url":"https://openai.com/index/browsecomp/","category":"Web research","created_at":"2026-11-02T19:49:15.744Z","description":"Shared setup: 200K context compaction, 500-step cap, temperature 1.","display_name":"BrowseComp","higher_is_better":1,"id":"browsecomp-kimi31-comparison","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"BrowseComp","official_url":"https://www.kimi.ai/blog/kimi-k3-1-pro","page_slug":"browsecomp","summary":"Tests whether an agent can find hard-to-locate, verifiable information online, helping you choose a model for client research that would otherwise require persistent manual searching. Search tools and budgets differ across evaluation setups.","tracking_status":"tracked","unit":"%","version":"Kimi K3.1 comparison / 200K compaction"}},{"id":54,"type":"benchmark","entityId":"gpqa-diamond-aa","action":"updated","label":"GPQA Diamond","modelId":null,"benchmarkSlug":"gpqa-diamond","source":"https://github.com/idavidrein/gpqa","previous":{"benchmark_homepage_url":"https://github.com/idavidrein/gpqa","category":"Best in Reasoning","created_at":"2026-11-02T22:06:11.752Z","description":"Independent GPQA Diamond results. Best reported effort per model; full precision retained.","display_name":"GPQA Diamond","higher_is_better":1,"id":"gpqa-diamond-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"GPQA","official_url":"https://artificialanalysis.ai/evaluations/gpqa-diamond","page_slug":"gpqa-diamond","summary":"GPQA Diamond tests graduate-level scientific knowledge and reasoning with difficult multiple-choice questions in biology, chemistry, and physics. Diamond is the 198-question subset selected for high-quality expert agreement. The results here use the independently reported Diamond evaluation.","tracking_status":"tracked","unit":"%","version":"Diamond / Artificial Analysis"},"current":{"benchmark_homepage_url":"https://github.com/idavidrein/gpqa","category":"Best in Reasoning","created_at":"2026-11-02T22:06:11.752Z","description":"Independent GPQA Diamond results. Best reported effort per model; full precision retained.","display_name":"GPQA Diamond","higher_is_better":1,"id":"gpqa-diamond-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"GPQA","official_url":"https://artificialanalysis.ai/evaluations/gpqa-diamond","page_slug":"gpqa-diamond","summary":"Tests difficult scientific questions in biology, chemistry and physics, helping you choose a model to support technical research and check complex explanations. Results cover the 198-question Diamond subset, not hands-on laboratory work.","tracking_status":"tracked","unit":"%","version":"Diamond / Artificial Analysis"}},{"id":55,"type":"benchmark","entityId":"deepswe-openai-astra","action":"updated","label":"DeepSWE","modelId":null,"benchmarkSlug":"deepswe","source":"https://deepswe.datacurve.ai/","previous":{"benchmark_homepage_url":"https://deepswe.datacurve.ai/","category":"Best in Coding","created_at":"2026-11-02T22:06:11.752Z","description":"Provider-reported comparison cohort; kept separate from rounded official board.","display_name":"DeepSWE","higher_is_better":1,"id":"deepswe-openai-astra","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"DeepSWE","official_url":"https://openai.com/index/gpt-6-astra/","page_slug":"deepswe","summary":"DeepSWE evaluates coding agents on original, long-horizon software engineering tasks. It tests whether an agent can understand a codebase, make changes, and complete engineering work beyond short coding questions.","tracking_status":"tracked","unit":"%","version":"v1.1 / OpenAI Astra launch comparison"},"current":{"benchmark_homepage_url":"https://deepswe.datacurve.ai/","category":"Best in Coding","created_at":"2026-11-02T22:06:11.752Z","description":"Provider-reported comparison cohort; kept separate from rounded official board.","display_name":"DeepSWE","higher_is_better":1,"id":"deepswe-openai-astra","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"DeepSWE","official_url":"https://openai.com/index/gpt-6-astra/","page_slug":"deepswe","summary":"Tests whether an agent can understand a codebase and complete substantial engineering changes, helping you assess which model could take on more development work while you focus on requirements and review.","tracking_status":"tracked","unit":"%","version":"v1.1 / OpenAI Astra launch comparison"}},{"id":56,"type":"benchmark","entityId":"browsecomp-openai-astra","action":"updated","label":"BrowseComp","modelId":null,"benchmarkSlug":"browsecomp","source":"https://openai.com/index/browsecomp/","previous":{"benchmark_homepage_url":"https://openai.com/index/browsecomp/","category":"Best in Web Research","created_at":"2026-11-02T22:06:11.753Z","description":"One published comparison cohort. Do not combine with the Moonshot 200K/500-step track.","display_name":"BrowseComp","higher_is_better":1,"id":"browsecomp-openai-astra","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"BrowseComp","official_url":"https://openai.com/index/gpt-6-astra/","page_slug":"browsecomp","summary":"BrowseComp evaluates web research agents on questions with hard-to-find but verifiable answers. It tests persistent browsing, search, and the ability to connect information from multiple sources. Tool access, search budgets, and context management can differ across published comparison reports.","tracking_status":"tracked","unit":"%","version":"OpenAI Astra launch comparison / September 2026"},"current":{"benchmark_homepage_url":"https://openai.com/index/browsecomp/","category":"Best in Web Research","created_at":"2026-11-02T22:06:11.753Z","description":"One published comparison cohort. Do not combine with the Moonshot 200K/500-step track.","display_name":"BrowseComp","higher_is_better":1,"id":"browsecomp-openai-astra","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"BrowseComp","official_url":"https://openai.com/index/gpt-6-astra/","page_slug":"browsecomp","summary":"Tests whether an agent can find hard-to-locate, verifiable information online, helping you choose a model for client research that would otherwise require persistent manual searching. Search tools and budgets differ across evaluation setups.","tracking_status":"tracked","unit":"%","version":"OpenAI Astra launch comparison / September 2026"}},{"id":57,"type":"benchmark","entityId":"automationbench-zapier-1-0-6","action":"updated","label":"AutomationBench","modelId":null,"benchmarkSlug":"automationbench","source":"https://zapier.com/benchmarks","previous":{"benchmark_homepage_url":"https://zapier.com/benchmarks","category":"Best in Work Automations","created_at":"2026-11-02T22:06:11.753Z","description":"Zapier official board, not AutomationBench-AA objective scores or the public 600-task dataset. Per-model default fallbacks are disclosed; explicit two-model fallback combinations are stored separately.","display_name":"AutomationBench","higher_is_better":1,"id":"automationbench-zapier-1-0-6","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict task completion","minimum_score":0,"name":"AutomationBench","official_url":"https://zapier.com/benchmarks","page_slug":"automationbench","summary":"AutomationBench tests agents on realistic business workflows across applications and APIs, covering sales, marketing, operations, support, finance, and HR. It evaluates whether agents can coordinate multi-step work and complete tasks. Zapier’s hosted leaderboard measures strict task completion on a private held-out set.","tracking_status":"tracked","unit":"%","version":"1.0.6 / Zapier held-out private set"},"current":{"benchmark_homepage_url":"https://zapier.com/benchmarks","category":"Best in Work Automations","created_at":"2026-11-02T22:06:11.753Z","description":"Zapier official board, not AutomationBench-AA objective scores or the public 600-task dataset. Per-model default fallbacks are disclosed; explicit two-model fallback combinations are stored separately.","display_name":"AutomationBench","higher_is_better":1,"id":"automationbench-zapier-1-0-6","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict task completion","minimum_score":0,"name":"AutomationBench","official_url":"https://zapier.com/benchmarks","page_slug":"automationbench","summary":"Tests whether an agent can finish business workflows across apps and APIs, helping you assess what sales, support or operations work you could automate to spend more time with clients. The hosted Zapier track measures complete tasks on a private test set.","tracking_status":"tracked","unit":"%","version":"1.0.6 / Zapier held-out private set"}},{"id":58,"type":"benchmark","entityId":"gdpval-aa-v2-1","action":"updated","label":"GDPval-AA","modelId":null,"benchmarkSlug":"gdpval-aa","source":"https://artificialanalysis.ai/evaluations/gdpval-aa","previous":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","category":"Best in Knowledge Work","created_at":"2026-11-02T22:06:11.753Z","description":"Independent v2.1 Elo ratings, not percentages or v2.0 ratings. Best reported effort per model.","display_name":"GDPval-AA","higher_is_better":1,"id":"gdpval-aa-v2-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":null,"metric":"Elo","minimum_score":null,"name":"GDPval-AA","official_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","page_slug":"gdpval-aa","summary":"GDPval-AA evaluates models on professional knowledge-work tasks from OpenAI’s GDPval dataset, including creating documents, spreadsheets, slides, and other work products. Agents use tools through the Stirrup framework. Results are Elo ratings from blind pairwise comparisons, rather than percentage accuracy.","tracking_status":"tracked","unit":"Elo","version":"2.1"},"current":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","category":"Best in Knowledge Work","created_at":"2026-11-02T22:06:11.753Z","description":"Independent v2.1 Elo ratings, not percentages or v2.0 ratings. Best reported effort per model.","display_name":"GDPval-AA","higher_is_better":1,"id":"gdpval-aa-v2-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":null,"metric":"Elo","minimum_score":null,"name":"GDPval-AA","official_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","page_slug":"gdpval-aa","summary":"Compares the quality of professional work products such as documents, spreadsheets and slides, helping you choose a model for preparing client deliverables. Elo is a relative quality rating, not a completion percentage or a measure of time saved.","tracking_status":"tracked","unit":"Elo","version":"2.1"}},{"id":59,"type":"benchmark","entityId":"swe-verified-open-reports","action":"updated","label":"SWE-bench Verified","modelId":null,"benchmarkSlug":"swe-bench-verified","source":"https://www.swebench.com/","previous":{"benchmark_homepage_url":"https://www.swebench.com/","category":"Best in Coding","created_at":"2026-11-02T22:18:13.327Z","description":"Provider-reported scores; agent harnesses differ. Every row retains its harness and original source.","display_name":"SWE-bench Verified","higher_is_better":1,"id":"swe-verified-open-reports","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Resolved","minimum_score":0,"name":"SWE-bench","official_url":"https://www.swebench.com/","page_slug":"swe-bench-verified","summary":"SWE-bench evaluates software engineering agents on resolving real issues in existing code repositories. The Verified subset contains human-validated tasks. Scores report the proportion of issues resolved; agent harnesses and evaluation settings are retained with each published model result.","tracking_status":"tracked","unit":"%","version":"Verified / published open-model reports"},"current":{"benchmark_homepage_url":"https://www.swebench.com/","category":"Best in Coding","created_at":"2026-11-02T22:18:13.327Z","description":"Provider-reported scores; agent harnesses differ. Every row retains its harness and original source.","display_name":"SWE-bench Verified","higher_is_better":1,"id":"swe-verified-open-reports","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Resolved","minimum_score":0,"name":"SWE-bench","official_url":"https://www.swebench.com/","page_slug":"swe-bench-verified","summary":"Tests whether an agent can fix real GitHub issues, helping you choose a model for bug fixes and maintenance so you can focus on reviewing changes and serving clients. The Verified set contains 500 human-validated issues; agent setups remain separate.","tracking_status":"tracked","unit":"%","version":"Verified / published open-model reports"}},{"id":60,"type":"benchmark","entityId":"terminal-bench-2-1-aa","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Best in Terminal Use","created_at":"2026-11-02T22:18:13.327Z","description":"Independent AA evaluation of Terminal-Bench 2.1. Never mix 2.0 or 4.0 scores.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"Terminal-Bench","official_url":"https://artificialanalysis.ai/evaluations/terminalbench-2-1","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"2.1 / Artificial Analysis"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Best in Terminal Use","created_at":"2026-11-02T22:18:13.327Z","description":"Independent AA evaluation of Terminal-Bench 2.1. Never mix 2.0 or 4.0 scores.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"Terminal-Bench","official_url":"https://artificialanalysis.ai/evaluations/terminalbench-2-1","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"2.1 / Artificial Analysis"}},{"id":61,"type":"benchmark","entityId":"browsecomp-open-reports","action":"updated","label":"BrowseComp","modelId":null,"benchmarkSlug":"browsecomp","source":"https://openai.com/index/browsecomp/","previous":{"benchmark_homepage_url":"https://openai.com/index/browsecomp/","category":"Best in Web Research","created_at":"2026-11-02T22:18:13.327Z","description":"Provider-reported scores; browsing tools, budgets and context-management settings differ. This ranking is not a matched-harness comparison.","display_name":"BrowseComp","higher_is_better":1,"id":"browsecomp-open-reports","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"BrowseComp","official_url":"https://openai.com/index/browsecomp/","page_slug":"browsecomp","summary":"BrowseComp evaluates web research agents on questions with hard-to-find but verifiable answers. It tests persistent browsing, search, and the ability to connect information from multiple sources. Tool access, search budgets, and context management can differ across published comparison reports.","tracking_status":"tracked","unit":"%","version":"Published open-model reports"},"current":{"benchmark_homepage_url":"https://openai.com/index/browsecomp/","category":"Best in Web Research","created_at":"2026-11-02T22:18:13.327Z","description":"Provider-reported scores; browsing tools, budgets and context-management settings differ. This ranking is not a matched-harness comparison.","display_name":"BrowseComp","higher_is_better":1,"id":"browsecomp-open-reports","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"BrowseComp","official_url":"https://openai.com/index/browsecomp/","page_slug":"browsecomp","summary":"Tests whether an agent can find hard-to-locate, verifiable information online, helping you choose a model for client research that would otherwise require persistent manual searching. Search tools and budgets differ across evaluation setups.","tracking_status":"tracked","unit":"%","version":"Published open-model reports"}},{"id":62,"type":"benchmark","entityId":"terminal-bench-2-0-open-reports","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-02T22:18:13.327Z","description":"Archived 2.0 scores; excluded from the 2.1 card.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-0-open-reports","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"2.0 / published open-model reports"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-02T22:18:13.327Z","description":"Archived 2.0 scores; excluded from the 2.1 card.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-0-open-reports","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"2.0 / published open-model reports"}},{"id":63,"type":"benchmark","entityId":"terminal-bench-2-1-provider-open","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-02T22:18:13.327Z","description":"Provider harnesses differ; excluded from independent AA track.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-provider-open","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"2.1 / published open-model reports"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-02T22:18:13.327Z","description":"Provider harnesses differ; excluded from independent AA track.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-provider-open","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"2.1 / published open-model reports"}},{"id":64,"type":"benchmark","entityId":"automationbench-1-0-6-provider-open","action":"updated","label":"AutomationBench","modelId":null,"benchmarkSlug":"automationbench","source":"https://zapier.com/benchmarks","previous":{"benchmark_homepage_url":"https://zapier.com/benchmarks","category":"Work automations","created_at":"2026-11-02T22:18:13.327Z","description":"Not AA objective scoring or Zapier current leaderboard; dataset identity not established.","display_name":"AutomationBench","higher_is_better":1,"id":"automationbench-1-0-6-provider-open","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict task completion","minimum_score":0,"name":"AutomationBench","official_url":"https://z.ai/blog/glm-5.3-flash","page_slug":"automationbench","summary":"AutomationBench tests agents on realistic business workflows across applications and APIs, covering sales, marketing, operations, support, finance, and HR. It evaluates whether agents can coordinate multi-step work and complete tasks. Zapier’s hosted leaderboard measures strict task completion on a private held-out set.","tracking_status":"tracked","unit":"%","version":"1.0.6 / Z.ai provider report"},"current":{"benchmark_homepage_url":"https://zapier.com/benchmarks","category":"Work automations","created_at":"2026-11-02T22:18:13.327Z","description":"Not AA objective scoring or Zapier current leaderboard; dataset identity not established.","display_name":"AutomationBench","higher_is_better":1,"id":"automationbench-1-0-6-provider-open","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict task completion","minimum_score":0,"name":"AutomationBench","official_url":"https://z.ai/blog/glm-5.3-flash","page_slug":"automationbench","summary":"Tests whether an agent can finish business workflows across apps and APIs, helping you assess what sales, support or operations work you could automate to spend more time with clients. The hosted Zapier track measures complete tasks on a private test set.","tracking_status":"tracked","unit":"%","version":"1.0.6 / Z.ai provider report"}},{"id":65,"type":"benchmark","entityId":"lifeval-2026-11-02","action":"updated","label":"LifeVal","modelId":null,"benchmarkSlug":"lifeval","source":"https://openai.com/index/lifeval/","previous":{"benchmark_homepage_url":"https://openai.com/index/lifeval/","category":"Personal assistance","created_at":"2026-11-02T23:50:46.980Z","description":"Average attempts within each task, then average all 169 task means equally. No willingness-to-pay weighting. Reward multiplied by 100; published display values rounded to one decimal place.","display_name":"LifeVal","higher_is_better":1,"id":"lifeval-2026-11-02","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Unweighted average reward","minimum_score":0,"name":"LifeVal","official_url":"https://openai.com/index/lifeval/","page_slug":"lifeval","summary":"LifeVal evaluates personal assistants on everyday responsibilities, including caregiving, household administration, shopping, education, personal finances, and accessing government services. Scores are unweighted average rewards across 169 tasks, expressed as percentages, rather than a strict task-completion rate. The launch results cover five OpenAI models.","tracking_status":"tracked","unit":"%","version":"November 2026"},"current":{"benchmark_homepage_url":"https://openai.com/index/lifeval/","category":"Personal assistance","created_at":"2026-11-02T23:50:46.980Z","description":"Average attempts within each task, then average all 169 task means equally. No willingness-to-pay weighting. Reward multiplied by 100; published display values rounded to one decimal place.","display_name":"LifeVal","higher_is_better":1,"id":"lifeval-2026-11-02","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Unweighted average reward","minimum_score":0,"name":"LifeVal","official_url":"https://openai.com/index/lifeval/","page_slug":"lifeval","summary":"Tests help with everyday responsibilities such as shopping, household administration and personal finances, helping you assess an assistant for reducing routine life admin. Scores are average rewards across 169 tasks, not the percentage of tasks fully completed.","tracking_status":"tracked","unit":"%","version":"November 2026"}},{"id":66,"type":"benchmark","entityId":"mypcbench-canonical-100-step","action":"updated","label":"MyPCBench","modelId":null,"benchmarkSlug":"mypcbench","source":"https://mypcbench.com/leaderboard","previous":{"benchmark_homepage_url":"https://mypcbench.com/leaderboard","category":"Personal tasks across apps","created_at":"2026-11-03T00:01:47.491Z","description":"MyPCBench evaluates assistants on 184 tasks across a personal desktop with 17 simulated web apps. Tasks involve personal records, travel, shopping, calendars, and coordinating work across applications. These results use the canonical computer-and-bash setup with a 100-step limit and report the percentage of tasks where every rubric item passes.","display_name":"MyPCBench","higher_is_better":1,"id":"mypcbench-canonical-100-step","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Perfect-task rate","minimum_score":0,"name":"MyPCBench","official_url":"https://mypcbench.com/leaderboard","page_slug":"mypcbench","summary":"MyPCBench evaluates assistants on 184 tasks across a personal desktop with 17 simulated web apps. Tasks involve personal records, travel, shopping, calendars, and coordinating work across applications. These results use the canonical computer-and-bash setup with a 100-step limit and report the percentage of tasks where every rubric item passes.","tracking_status":"tracked","unit":"%","version":"184 tasks · 100 steps"},"current":{"benchmark_homepage_url":"https://mypcbench.com/leaderboard","category":"Personal tasks across apps","created_at":"2026-11-03T00:01:47.491Z","description":"MyPCBench evaluates assistants on 184 tasks across a personal desktop with 17 simulated web apps. Tasks involve personal records, travel, shopping, calendars, and coordinating work across applications. These results use the canonical computer-and-bash setup with a 100-step limit and report the percentage of tasks where every rubric item passes.","display_name":"MyPCBench","higher_is_better":1,"id":"mypcbench-canonical-100-step","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Perfect-task rate","minimum_score":0,"name":"MyPCBench","official_url":"https://mypcbench.com/leaderboard","page_slug":"mypcbench","summary":"Tests whether an assistant can complete personal desktop tasks across calendars, travel, shopping and records, helping you assess what routine coordination you could delegate. A task passes only when every rubric item is satisfied within the 100-step setup.","tracking_status":"tracked","unit":"%","version":"184 tasks · 100 steps"}},{"id":67,"type":"benchmark","entityId":"appworld-test-challenge-tgc","action":"updated","label":"AppWorld","modelId":null,"benchmarkSlug":"appworld","source":"https://appworld.dev/appworld/leaderboard","previous":{"benchmark_homepage_url":"https://appworld.dev/appworld/leaderboard","category":"Everyday app automation","created_at":"2026-11-03T00:01:47.493Z","description":"AppWorld evaluates agents completing everyday tasks across nine simulated apps through code and APIs. Tasks include coordinating messages, shopping, and personal information. Scores here measure Task Goal Completion on the Test-Challenge split: the percentage of tasks for which all checks pass. Agent methods are retained with each result.","display_name":"AppWorld","higher_is_better":1,"id":"appworld-test-challenge-tgc","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task Goal Completion","minimum_score":0,"name":"AppWorld","official_url":"https://appworld.dev/appworld/leaderboard","page_slug":"appworld","summary":"AppWorld evaluates agents completing everyday tasks across nine simulated apps through code and APIs. Tasks include coordinating messages, shopping, and personal information. Scores here measure Task Goal Completion on the Test-Challenge split: the percentage of tasks for which all checks pass. Agent methods are retained with each result.","tracking_status":"tracked","unit":"%","version":"Test-Challenge"},"current":{"benchmark_homepage_url":"https://appworld.dev/appworld/leaderboard","category":"Everyday app automation","created_at":"2026-11-03T00:01:47.493Z","description":"AppWorld evaluates agents completing everyday tasks across nine simulated apps through code and APIs. Tasks include coordinating messages, shopping, and personal information. Scores here measure Task Goal Completion on the Test-Challenge split: the percentage of tasks for which all checks pass. Agent methods are retained with each result.","display_name":"AppWorld","higher_is_better":1,"id":"appworld-test-challenge-tgc","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task Goal Completion","minimum_score":0,"name":"AppWorld","official_url":"https://appworld.dev/appworld/leaderboard","page_slug":"appworld","summary":"Tests whether an agent can complete everyday tasks across simulated apps using code and APIs, helping you choose an assistant for reducing repetitive app-to-app work. Test-Challenge scores require all task checks to pass.","tracking_status":"tracked","unit":"%","version":"Test-Challenge"}},{"id":68,"type":"benchmark","entityId":"gaia2-cli-openclaw-2026-4-1","action":"updated","label":"Gaia2","modelId":null,"benchmarkSlug":"gaia2","source":"https://huggingface.co/spaces/meta-agents-research-environments/leaderboard","previous":{"benchmark_homepage_url":"https://huggingface.co/spaces/meta-agents-research-environments/leaderboard","category":"Planning and coordination","created_at":"2026-11-03T00:01:47.494Z","description":"Gaia2 evaluates assistants coordinating tasks across simulated email, calendars, shopping, contacts, and other apps. It tests execution, search, adaptation, ambiguity handling, and time constraints. This ranking uses the Gaia2-CLI leaderboard with the OpenClaw 2026.4.1 harness and reports overall pass@1. The older vanilla, noise, and agent-to-agent results are separate evaluations.","display_name":"Gaia2","higher_is_better":1,"id":"gaia2-cli-openclaw-2026-4-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"Gaia2","official_url":"https://huggingface.co/spaces/meta-agents-research-environments/leaderboard","page_slug":"gaia2","summary":"Gaia2 evaluates assistants coordinating tasks across simulated email, calendars, shopping, contacts, and other apps. It tests execution, search, adaptation, ambiguity handling, and time constraints. This ranking uses the Gaia2-CLI leaderboard with the OpenClaw 2026.4.1 harness and reports overall pass@1. The older vanilla, noise, and agent-to-agent results are separate evaluations.","tracking_status":"tracked","unit":"%","version":"CLI · OpenClaw 2026.4.1"},"current":{"benchmark_homepage_url":"https://huggingface.co/spaces/meta-agents-research-environments/leaderboard","category":"Planning and coordination","created_at":"2026-11-03T00:01:47.494Z","description":"Gaia2 evaluates assistants coordinating tasks across simulated email, calendars, shopping, contacts, and other apps. It tests execution, search, adaptation, ambiguity handling, and time constraints. This ranking uses the Gaia2-CLI leaderboard with the OpenClaw 2026.4.1 harness and reports overall pass@1. The older vanilla, noise, and agent-to-agent results are separate evaluations.","display_name":"Gaia2","higher_is_better":1,"id":"gaia2-cli-openclaw-2026-4-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"Gaia2","official_url":"https://huggingface.co/spaces/meta-agents-research-environments/leaderboard","page_slug":"gaia2","summary":"Tests whether an assistant can coordinate work across email, calendars, shopping and other apps, helping you assess how much routine planning and execution you could hand off. These results use the CLI and OpenClaw setup; other setups are separate.","tracking_status":"tracked","unit":"%","version":"CLI · OpenClaw 2026.4.1"}},{"id":69,"type":"benchmark","entityId":"pi-bench-overall-proc","action":"updated","label":"π-Bench","modelId":null,"benchmarkSlug":"pi-bench","source":"https://simplified-reasoning.github.io/Pi-Bench/","previous":{"benchmark_homepage_url":"https://simplified-reasoning.github.io/Pi-Bench/","category":"Proactive assistance","created_at":"2026-11-03T00:01:47.494Z","description":"π-Bench evaluates assistants across 100 multi-turn tasks in persistent workspaces. It tests whether an agent notices unstated preferences and requirements, asks useful questions, and carries context across sessions. Scores here use overall Proactivity (PROC), the share of hidden intents resolved proactively. Completeness is a separate metric and is retained with the source results.","display_name":"π-Bench","higher_is_better":1,"id":"pi-bench-overall-proc","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Proactivity (PROC)","minimum_score":0,"name":"π-Bench","official_url":"https://simplified-reasoning.github.io/Pi-Bench/","page_slug":"pi-bench","summary":"π-Bench evaluates assistants across 100 multi-turn tasks in persistent workspaces. It tests whether an agent notices unstated preferences and requirements, asks useful questions, and carries context across sessions. Scores here use overall Proactivity (PROC), the share of hidden intents resolved proactively. Completeness is a separate metric and is retained with the source results.","tracking_status":"tracked","unit":"%","version":"Overall · PROC"},"current":{"benchmark_homepage_url":"https://simplified-reasoning.github.io/Pi-Bench/","category":"Proactive assistance","created_at":"2026-11-03T00:01:47.494Z","description":"π-Bench evaluates assistants across 100 multi-turn tasks in persistent workspaces. It tests whether an agent notices unstated preferences and requirements, asks useful questions, and carries context across sessions. Scores here use overall Proactivity (PROC), the share of hidden intents resolved proactively. Completeness is a separate metric and is retained with the source results.","display_name":"π-Bench","higher_is_better":1,"id":"pi-bench-overall-proc","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Proactivity (PROC)","minimum_score":0,"name":"π-Bench","official_url":"https://simplified-reasoning.github.io/Pi-Bench/","page_slug":"pi-bench","summary":"Tests whether an assistant notices needs you have not explicitly stated, helping you choose one that needs less repeated prompting and remembers context across sessions. The score measures hidden intents handled proactively, not overall task completion.","tracking_status":"tracked","unit":"%","version":"Overall · PROC"}},{"id":70,"type":"benchmark","entityId":"tau2-standard-overall","action":"updated","label":"τ²-bench","modelId":null,"benchmarkSlug":"tau-bench","source":"https://taubench.com/leaderboard?benchmark=core","previous":{"benchmark_homepage_url":"https://taubench.com/leaderboard?benchmark=core","category":"Bookings and customer service","created_at":"2026-11-03T00:01:47.494Z","description":"τ²-bench evaluates conversational agents completing customer-service tasks in airline, retail, and telecom environments. Agents use tools, follow policies, and coordinate with a simulated user who can also act. Scores here are overall pass@1 from the Standard leaderboard, using GPT-5.2 as the user simulator. Custom, legacy, voice, and banking evaluations are separate.","display_name":"τ²-bench","higher_is_better":1,"id":"tau2-standard-overall","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"τ²-bench","official_url":"https://taubench.com/leaderboard?benchmark=core","page_slug":"tau-bench","summary":"τ²-bench evaluates conversational agents completing customer-service tasks in airline, retail, and telecom environments. Agents use tools, follow policies, and coordinate with a simulated user who can also act. Scores here are overall pass@1 from the Standard leaderboard, using GPT-5.2 as the user simulator. Custom, legacy, voice, and banking evaluations are separate.","tracking_status":"tracked","unit":"%","version":"Standard · Overall"},"current":{"benchmark_homepage_url":"https://taubench.com/leaderboard?benchmark=core","category":"Bookings and customer service","created_at":"2026-11-03T00:01:47.494Z","description":"τ²-bench evaluates conversational agents completing customer-service tasks in airline, retail, and telecom environments. Agents use tools, follow policies, and coordinate with a simulated user who can also act. Scores here are overall pass@1 from the Standard leaderboard, using GPT-5.2 as the user simulator. Custom, legacy, voice, and banking evaluations are separate.","display_name":"τ²-bench","higher_is_better":1,"id":"tau2-standard-overall","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Pass@1","minimum_score":0,"name":"τ²-bench","official_url":"https://taubench.com/leaderboard?benchmark=core","page_slug":"tau-bench","summary":"Tests whether a customer-service agent can resolve requests while using tools and following company policies, helping you assess opportunities to reduce manual support work. These results use the Standard airline, retail and telecom evaluation.","tracking_status":"tracked","unit":"%","version":"Standard · Overall"}},{"id":71,"type":"benchmark","entityId":"hle-diamond-aa","action":"updated","label":"HLE-Diamond","modelId":null,"benchmarkSlug":"hle-diamond","source":"https://lastexam.ai/","previous":{"benchmark_homepage_url":"https://lastexam.ai/","category":"advanced knowledge and reasoning","created_at":"2026-11-03T01:02:06.543Z","description":"HLE-Diamond is a revised 1,000-question version of Humanity’s Last Exam, split into 500 reasoning and 500 knowledge questions. These results use the official prompts without tools. Scores are separate from the original Humanity’s Last Exam.","display_name":"HLE-Diamond","higher_is_better":1,"id":"hle-diamond-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"HLE-Diamond","official_url":"https://artificialanalysis.ai/evaluations/hle-diamond","page_slug":"hle-diamond","summary":"HLE-Diamond is a revised 1,000-question version of Humanity’s Last Exam, split into 500 reasoning and 500 knowledge questions. These results use the official prompts without tools. Scores are separate from the original Humanity’s Last Exam.","tracking_status":"tracked","unit":"%","version":"1,000 questions / no tools"},"current":{"benchmark_homepage_url":"https://lastexam.ai/","category":"advanced knowledge and reasoning","created_at":"2026-11-03T01:02:06.543Z","description":"HLE-Diamond is a revised 1,000-question version of Humanity’s Last Exam, split into 500 reasoning and 500 knowledge questions. These results use the official prompts without tools. Scores are separate from the original Humanity’s Last Exam.","display_name":"HLE-Diamond","higher_is_better":1,"id":"hle-diamond-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"HLE-Diamond","official_url":"https://artificialanalysis.ai/evaluations/hle-diamond","page_slug":"hle-diamond","summary":"Tests difficult knowledge and reasoning questions, helping you choose a model for demanding analysis before you trust it with client research. This 1,000-question evaluation uses no tools and is separate from the original Humanity's Last Exam.","tracking_status":"tracked","unit":"%","version":"1,000 questions / no tools"}},{"id":72,"type":"benchmark","entityId":"gdp-pdf-aa","action":"updated","label":"GDP.pdf","modelId":null,"benchmarkSlug":"gdp-pdf","source":"https://surgehq.ai/benchmarks/gdp-pdf","previous":{"benchmark_homepage_url":"https://surgehq.ai/benchmarks/gdp-pdf","category":"professional document reasoning","created_at":"2026-11-03T01:02:06.543Z","description":"GDP.pdf tests reasoning over long professional documents across ten domains. The All-pass score measures attempts satisfying every task criterion. These results use the independent five-attempt evaluation with LiteParse text and page images for models supporting images; other implementations remain separate.","display_name":"GDP.pdf","higher_is_better":1,"id":"gdp-pdf-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"GDP.pdf","official_url":"https://artificialanalysis.ai/evaluations/gdp-pdf","page_slug":"gdp-pdf","summary":"GDP.pdf tests reasoning over long professional documents across ten domains. The All-pass score measures attempts satisfying every task criterion. These results use the independent five-attempt evaluation with LiteParse text and page images for models supporting images; other implementations remain separate.","tracking_status":"tracked","unit":"%","version":"100 tasks / All-pass / five attempts"},"current":{"benchmark_homepage_url":"https://surgehq.ai/benchmarks/gdp-pdf","category":"professional document reasoning","created_at":"2026-11-03T01:02:06.543Z","description":"GDP.pdf tests reasoning over long professional documents across ten domains. The All-pass score measures attempts satisfying every task criterion. These results use the independent five-attempt evaluation with LiteParse text and page images for models supporting images; other implementations remain separate.","display_name":"GDP.pdf","higher_is_better":1,"id":"gdp-pdf-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"GDP.pdf","official_url":"https://artificialanalysis.ai/evaluations/gdp-pdf","page_slug":"gdp-pdf","summary":"Tests reasoning over long professional documents, helping you assess which model could reduce the time you spend working through source material for client projects. All-pass scores require every criterion to be satisfied; implementations are kept separate.","tracking_status":"tracked","unit":"%","version":"100 tasks / All-pass / five attempts"}},{"id":73,"type":"benchmark","entityId":"aa-briefcase-v1-1","action":"updated","label":"AA-Briefcase","modelId":null,"benchmarkSlug":"aa-briefcase","source":"https://artificialanalysis.ai/evaluations/aa-briefcase","previous":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","category":"business deliverables","created_at":"2026-11-03T01:02:06.544Z","description":"AA-Briefcase evaluates agents producing spreadsheets, presentations, memos and other professional deliverables across 91 tasks. Its combined Elo incorporates rubric performance, analytical quality and presentation. Each task runs independently, and the public Lite example is excluded.","display_name":"AA-Briefcase","higher_is_better":1,"id":"aa-briefcase-v1-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":null,"metric":"Combined Elo","minimum_score":0,"name":"AA-Briefcase","official_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","page_slug":"aa-briefcase","summary":"AA-Briefcase evaluates agents producing spreadsheets, presentations, memos and other professional deliverables across 91 tasks. Its combined Elo incorporates rubric performance, analytical quality and presentation. Each task runs independently, and the public Lite example is excluded.","tracking_status":"tracked","unit":"Elo","version":"1.1 / combined Elo"},"current":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","category":"business deliverables","created_at":"2026-11-03T01:02:06.544Z","description":"AA-Briefcase evaluates agents producing spreadsheets, presentations, memos and other professional deliverables across 91 tasks. Its combined Elo incorporates rubric performance, analytical quality and presentation. Each task runs independently, and the public Lite example is excluded.","display_name":"AA-Briefcase","higher_is_better":1,"id":"aa-briefcase-v1-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":null,"metric":"Combined Elo","minimum_score":0,"name":"AA-Briefcase","official_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","page_slug":"aa-briefcase","summary":"Tests professional deliverables such as spreadsheets, presentations and memos, helping you choose a model for complex client projects with many linked requirements. Rubric scores measure requirements met; Elo also reflects analytical and presentation quality, not hours saved.","tracking_status":"tracked","unit":"Elo","version":"1.1 / combined Elo"}},{"id":74,"type":"benchmark","entityId":"ifbench-aa","action":"updated","label":"IFBench","modelId":null,"benchmarkSlug":"ifbench","source":"https://github.com/allenai/IFBench","previous":{"benchmark_homepage_url":"https://github.com/allenai/IFBench","category":"precise instruction following","created_at":"2026-11-03T01:02:06.544Z","description":"IFBench tests whether models follow precise output requirements and generalize to 58 diverse, verifiable constraint types. It measures instruction-following reliability rather than completion of a full agent workflow.","display_name":"IFBench","higher_is_better":1,"id":"ifbench-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"IFBench","official_url":"https://artificialanalysis.ai/evaluations/ifbench","page_slug":"ifbench","summary":"IFBench tests whether models follow precise output requirements and generalize to 58 diverse, verifiable constraint types. It measures instruction-following reliability rather than completion of a full agent workflow.","tracking_status":"tracked","unit":"%","version":"58 constraint types"},"current":{"benchmark_homepage_url":"https://github.com/allenai/IFBench","category":"precise instruction following","created_at":"2026-11-03T01:02:06.544Z","description":"IFBench tests whether models follow precise output requirements and generalize to 58 diverse, verifiable constraint types. It measures instruction-following reliability rather than completion of a full agent workflow.","display_name":"IFBench","higher_is_better":1,"id":"ifbench-aa","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"IFBench","official_url":"https://artificialanalysis.ai/evaluations/ifbench","page_slug":"ifbench","summary":"Tests whether a model follows detailed formatting and output instructions, helping you reduce rework when client deliverables must meet a precise brief. It measures instruction following rather than completion of an entire project.","tracking_status":"tracked","unit":"%","version":"58 constraint types"}},{"id":75,"type":"benchmark","entityId":"livebench-2-agentic","action":"updated","label":"LiveBench: Agentic","modelId":null,"benchmarkSlug":"livebench-agentic","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"long-running agentic work","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates long-running agentic work on its October 25, 2026 release. This suite contains 18 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Agentic","higher_is_better":1,"id":"livebench-2-agentic","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Agentic","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-agentic","summary":"LiveBench 2.0 evaluates long-running agentic work on its October 25, 2026 release. This suite contains 18 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"long-running agentic work","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates long-running agentic work on its October 25, 2026 release. This suite contains 18 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Agentic","higher_is_better":1,"id":"livebench-2-agentic","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Agentic","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-agentic","summary":"Tests extended tasks that require an agent to keep working toward a goal, helping you assess which model could handle longer assignments with less step-by-step direction. Compare results from the same dated task set.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"}},{"id":76,"type":"benchmark","entityId":"livebench-2-design","action":"updated","label":"LiveBench: Design","modelId":null,"benchmarkSlug":"livebench-design","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"document and presentation creation","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates document and presentation creation on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Design","higher_is_better":1,"id":"livebench-2-design","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Design","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-design","summary":"LiveBench 2.0 evaluates document and presentation creation on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"document and presentation creation","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates document and presentation creation on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Design","higher_is_better":1,"id":"livebench-2-design","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Design","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-design","summary":"Tests document and presentation creation, helping you choose a model for preparing client-facing materials with less manual production work. Compare the same dated task set; the score does not measure turnaround time.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"}},{"id":77,"type":"benchmark","entityId":"livebench-2-coding","action":"updated","label":"LiveBench: Coding","modelId":null,"benchmarkSlug":"livebench-coding","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"coding with LiveBench","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates coding with LiveBench on its October 25, 2026 release. This suite contains 29 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Coding","higher_is_better":1,"id":"livebench-2-coding","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Coding","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-coding","summary":"LiveBench 2.0 evaluates coding with LiveBench on its October 25, 2026 release. This suite contains 29 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"coding with LiveBench","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates coding with LiveBench on its October 25, 2026 release. This suite contains 29 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Coding","higher_is_better":1,"id":"livebench-2-coding","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Coding","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-coding","summary":"Tests coding tasks, helping you compare models for implementation work that you would otherwise write or debug yourself. Compare results from the same dated task set and review generated changes before delivery.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"}},{"id":78,"type":"benchmark","entityId":"livebench-2-browser-use","action":"updated","label":"LiveBench: Browser Use","modelId":null,"benchmarkSlug":"livebench-browser-use","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"browser workflows","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates browser workflows on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Browser Use","higher_is_better":1,"id":"livebench-2-browser-use","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Browser Use","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-browser-use","summary":"LiveBench 2.0 evaluates browser workflows on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"browser workflows","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates browser workflows on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Browser Use","higher_is_better":1,"id":"livebench-2-browser-use","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Browser Use","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-browser-use","summary":"Tests browser workflows, helping you choose an agent for repetitive online tasks so you can spend less time navigating websites yourself. Results from different dated task sets are kept separate.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"}},{"id":79,"type":"benchmark","entityId":"livebench-2-data-analysis","action":"updated","label":"LiveBench: Data Analysis","modelId":null,"benchmarkSlug":"livebench-data-analysis","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"data analysis","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates data analysis on its October 25, 2026 release. This suite contains 150 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Data Analysis","higher_is_better":1,"id":"livebench-2-data-analysis","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Data Analysis","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-data-analysis","summary":"LiveBench 2.0 evaluates data analysis on its October 25, 2026 release. This suite contains 150 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"data analysis","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates data analysis on its October 25, 2026 release. This suite contains 150 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Data Analysis","higher_is_better":1,"id":"livebench-2-data-analysis","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Data Analysis","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-data-analysis","summary":"Tests data analysis, helping you choose a model to support turning client data into useful findings with less manual processing. Compare the same dated task set; a higher score does not guarantee correct analysis on your data.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"}},{"id":80,"type":"benchmark","entityId":"livebench-2-research","action":"updated","label":"LiveBench: Research","modelId":null,"benchmarkSlug":"livebench-research","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"research reports","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates research reports on its October 25, 2026 release. This suite contains 22 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Research","higher_is_better":1,"id":"livebench-2-research","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Research","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-research","summary":"LiveBench 2.0 evaluates research reports on its October 25, 2026 release. This suite contains 22 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"research reports","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates research reports on its October 25, 2026 release. This suite contains 22 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Research","higher_is_better":1,"id":"livebench-2-research","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Research","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-research","summary":"Tests research report tasks, helping you compare assistants for gathering and organizing material before advising clients. Results are specific to each dated task set and do not measure time saved.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"}},{"id":81,"type":"benchmark","entityId":"livebench-2-reasoning","action":"updated","label":"LiveBench: Reasoning","modelId":null,"benchmarkSlug":"livebench-reasoning","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"reasoning with LiveBench","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates reasoning with LiveBench on its October 25, 2026 release. This suite contains 202 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Reasoning","higher_is_better":1,"id":"livebench-2-reasoning","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Reasoning","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-reasoning","summary":"LiveBench 2.0 evaluates reasoning with LiveBench on its October 25, 2026 release. This suite contains 202 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"reasoning with LiveBench","created_at":"2026-11-03T01:02:06.544Z","description":"LiveBench 2.0 evaluates reasoning with LiveBench on its October 25, 2026 release. This suite contains 202 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","display_name":"LiveBench: Reasoning","higher_is_better":1,"id":"livebench-2-reasoning","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Reasoning","official_url":"https://livebench.ai/#/release/2026-10-25","page_slug":"livebench-reasoning","summary":"Tests reasoning problems, helping you compare models for working through complex questions and checking your approach. This suite measures reasoning performance rather than completion of a business workflow.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-25"}},{"id":82,"type":"benchmark","entityId":"terminal-bench-4-0-aa","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"terminal use","created_at":"2026-11-03T01:02:32.052Z","description":"Terminal-Bench 4.0 evaluated with the mini-swe-agent harness, 66 tasks and three repeats. Native-agent leaderboard results remain a separate evaluation track.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-4-0-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"4.0 / mini-swe-agent"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"terminal use","created_at":"2026-11-03T01:02:32.052Z","description":"Terminal-Bench 4.0 evaluated with the mini-swe-agent harness, 66 tasks and three repeats. Native-agent leaderboard results remain a separate evaluation track.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-4-0-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://artificialanalysis.ai/evaluations/terminalbench-4-0","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"4.0 / mini-swe-agent"}},{"id":83,"type":"benchmark","entityId":"livebench-2-agentic-2026-10-14","action":"updated","label":"LiveBench: Agentic","modelId":null,"benchmarkSlug":"livebench-agentic","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"long-running agentic work","created_at":"2026-11-03T02:14:08.943Z","description":"LiveBench 2.0 agentic suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","display_name":"LiveBench: Agentic","higher_is_better":1,"id":"livebench-2-agentic-2026-10-14","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Agentic","official_url":"https://livebench.ai/#/release/2026-10-14","page_slug":"livebench-agentic","summary":"LiveBench 2.0 agentic suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-14"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"long-running agentic work","created_at":"2026-11-03T02:14:08.943Z","description":"LiveBench 2.0 agentic suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","display_name":"LiveBench: Agentic","higher_is_better":1,"id":"livebench-2-agentic-2026-10-14","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Agentic","official_url":"https://livebench.ai/#/release/2026-10-14","page_slug":"livebench-agentic","summary":"Tests extended tasks that require an agent to keep working toward a goal, helping you assess which model could handle longer assignments with less step-by-step direction. Compare results from the same dated task set.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-14"}},{"id":84,"type":"benchmark","entityId":"livebench-2-coding-2026-10-14","action":"updated","label":"LiveBench: Coding","modelId":null,"benchmarkSlug":"livebench-coding","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"coding with LiveBench","created_at":"2026-11-03T02:14:08.943Z","description":"LiveBench 2.0 coding suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","display_name":"LiveBench: Coding","higher_is_better":1,"id":"livebench-2-coding-2026-10-14","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Coding","official_url":"https://livebench.ai/#/release/2026-10-14","page_slug":"livebench-coding","summary":"LiveBench 2.0 coding suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-14"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"coding with LiveBench","created_at":"2026-11-03T02:14:08.943Z","description":"LiveBench 2.0 coding suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","display_name":"LiveBench: Coding","higher_is_better":1,"id":"livebench-2-coding-2026-10-14","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Coding","official_url":"https://livebench.ai/#/release/2026-10-14","page_slug":"livebench-coding","summary":"Tests coding tasks, helping you compare models for implementation work that you would otherwise write or debug yourself. Compare results from the same dated task set and review generated changes before delivery.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-14"}},{"id":85,"type":"benchmark","entityId":"livebench-2-browser-use-2026-10-14","action":"updated","label":"LiveBench: Browser Use","modelId":null,"benchmarkSlug":"livebench-browser-use","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"browser workflows","created_at":"2026-11-03T02:23:57.213Z","description":"LiveBench 2.0 browser-use suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","display_name":"LiveBench: Browser Use","higher_is_better":1,"id":"livebench-2-browser-use-2026-10-14","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Browser Use","official_url":"https://livebench.ai/#/release/2026-10-14","page_slug":"livebench-browser-use","summary":"LiveBench 2.0 browser-use suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-14"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"browser workflows","created_at":"2026-11-03T02:23:57.213Z","description":"LiveBench 2.0 browser-use suite on the October 14, 2026 release. Historical results, kept separate from the October 25 release used by Agent Index. Source-specific model effort labels are retained; no effort is inferred.","display_name":"LiveBench: Browser Use","higher_is_better":1,"id":"livebench-2-browser-use-2026-10-14","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Browser Use","official_url":"https://livebench.ai/#/release/2026-10-14","page_slug":"livebench-browser-use","summary":"Tests browser workflows, helping you choose an agent for repetitive online tasks so you can spend less time navigating websites yourself. Results from different dated task sets are kept separate.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-10-14"}},{"id":86,"type":"benchmark","entityId":"swe-verified-mini-swe-agent-2-0-0","action":"updated","label":"SWE-bench Verified","modelId":null,"benchmarkSlug":"swe-bench-verified","source":"https://www.swebench.com/","previous":{"benchmark_homepage_url":"https://www.swebench.com/","category":"Coding agents","created_at":"2026-11-05T05:17:23.095Z","description":"Official 500-task SWE-bench Verified results using mini-SWE-agent 2.0.0. One attempt per task; model effort is preserved per report.","display_name":"SWE-bench Verified","higher_is_better":1,"id":"swe-verified-mini-swe-agent-2-0-0","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Resolved","minimum_score":0,"name":"SWE-bench","official_url":"https://www.swebench.com/","page_slug":"swe-bench-verified","summary":"SWE-bench Verified measures whether an agent can resolve real GitHub issues across 500 human-validated tasks. The primary track uses mini-SWE-agent 2.0.0; provider-reported setups remain separate.","tracking_status":"tracked","unit":"%","version":"Verified / mini-SWE-agent 2.0.0 / Bash Only"},"current":{"benchmark_homepage_url":"https://www.swebench.com/","category":"Coding agents","created_at":"2026-11-05T05:17:23.095Z","description":"Official 500-task SWE-bench Verified results using mini-SWE-agent 2.0.0. One attempt per task; model effort is preserved per report.","display_name":"SWE-bench Verified","higher_is_better":1,"id":"swe-verified-mini-swe-agent-2-0-0","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Resolved","minimum_score":0,"name":"SWE-bench","official_url":"https://www.swebench.com/","page_slug":"swe-bench-verified","summary":"Tests whether an agent can fix real GitHub issues, helping you choose a model for bug fixes and maintenance so you can focus on reviewing changes and serving clients. The Verified set contains 500 human-validated issues; agent setups remain separate.","tracking_status":"tracked","unit":"%","version":"Verified / mini-SWE-agent 2.0.0 / Bash Only"}},{"id":87,"type":"benchmark","entityId":"terminal-bench-2-1-cognition-launch","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-06T00:26:15.623Z","description":"Cognition-reported Terminal-Bench 2.1 results; agent harness and effort not specified.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-cognition-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"2.1 / Cognition SWE-2 Flash launch"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-06T00:26:15.623Z","description":"Cognition-reported Terminal-Bench 2.1 results; agent harness and effort not specified.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-cognition-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"2.1 / Cognition SWE-2 Flash launch"}},{"id":88,"type":"benchmark","entityId":"deep-finance-2026-11-05","action":"updated","label":"DeepFinance","modelId":null,"benchmarkSlug":"deep-finance","source":"https://www.juliusresearch.ai/deep-finance","previous":{"benchmark_homepage_url":"https://www.juliusresearch.ai/deep-finance","category":"Knowledge work","created_at":"2026-11-06T02:51:28.790Z","description":"DeepFinance tests long-horizon private equity and investment banking work: valuation, due diligence, underwriting and financial modeling. Agents work across data-room files and are graded on deliverables, workflow and judgment. Scores average rubric criteria satisfied across 21 tasks and 10 attempts per task. Native agent harnesses differ by model.","display_name":"DeepFinance","higher_is_better":1,"id":"deep-finance-2026-11-05","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Mean rubric score","minimum_score":0,"name":"DeepFinance","official_url":"https://www.juliusresearch.ai/deep-finance","page_slug":"deep-finance","summary":"DeepFinance tests long-horizon private equity and investment banking work: valuation, due diligence, underwriting and financial modeling. Agents work across data-room files and are graded on deliverables, workflow and judgment. Scores average rubric criteria satisfied across 21 tasks and 10 attempts per task. Native agent harnesses differ by model.","tracking_status":"tracked","unit":"%","version":"November 5, 2026 / native agent harnesses"},"current":{"benchmark_homepage_url":"https://www.juliusresearch.ai/deep-finance","category":"Knowledge work","created_at":"2026-11-06T02:51:28.790Z","description":"DeepFinance tests long-horizon private equity and investment banking work: valuation, due diligence, underwriting and financial modeling. Agents work across data-room files and are graded on deliverables, workflow and judgment. Scores average rubric criteria satisfied across 21 tasks and 10 attempts per task. Native agent harnesses differ by model.","display_name":"DeepFinance","higher_is_better":1,"id":"deep-finance-2026-11-05","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Mean rubric score","minimum_score":0,"name":"DeepFinance","official_url":"https://www.juliusresearch.ai/deep-finance","page_slug":"deep-finance","summary":"Tests private equity and investment banking work such as valuation, due diligence and financial modeling, helping you assess AI support for demanding client analysis. Scores reflect deliverables, workflow and judgment in model-specific agent setups.","tracking_status":"tracked","unit":"%","version":"November 5, 2026 / native agent harnesses"}},{"id":89,"type":"benchmark","entityId":"terminal-bench-2-1-inkling2-pi","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-08T23:23:39.765Z","description":"Thinking Machines model-card score using the Pi harness. Separate from Terminus and mini-swe-agent.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-inkling2-pi","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"2.1 / Inkling 2 Small / Pi harness"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-08T23:23:39.765Z","description":"Thinking Machines model-card score using the Pi harness. Separate from Terminus and mini-swe-agent.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-inkling2-pi","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"2.1 / Inkling 2 Small / Pi harness"}},{"id":90,"type":"benchmark","entityId":"terminal-bench-2-1-inkling2-terminus","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-08T23:23:39.765Z","description":"Thinking Machines model-card score using Terminus. Separate from the Pi-harness score.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-inkling2-terminus","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"2.1 / Inkling 2 Small / Terminus"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-08T23:23:39.765Z","description":"Thinking Machines model-card score using Terminus. Separate from the Pi-harness score.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-inkling2-terminus","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"2.1 / Inkling 2 Small / Terminus"}},{"id":91,"type":"benchmark","entityId":"terminal-bench-2-1-ripple-launch","action":"updated","label":"Terminal-Bench","modelId":null,"benchmarkSlug":"terminal-bench","source":"https://www.tbench.ai/","previous":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-08T23:23:39.765Z","description":"Ricursive reports 80.52 in its headline and 80.5 in its rounded benchmark table; Terminus 2, maximum effort, BF16.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-ripple-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Terminal-Bench evaluates agents on challenging tasks performed through a terminal. It tests multi-step command-line work, problem solving, and the ability to use tools to complete practical tasks. Reported scores depend on the benchmark release and agent harness.","tracking_status":"tracked","unit":"%","version":"2.1 / Ripple launch / Terminus 2"},"current":{"benchmark_homepage_url":"https://www.tbench.ai/","category":"Terminal use","created_at":"2026-11-08T23:23:39.765Z","description":"Ricursive reports 80.52 in its headline and 80.5 in its rounded benchmark table; Terminus 2, maximum effort, BF16.","display_name":"Terminal-Bench","higher_is_better":1,"id":"terminal-bench-2-1-ripple-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"Terminal-Bench","official_url":"https://www.tbench.ai/","page_slug":"terminal-bench","summary":"Tests whether an agent can complete multi-step command-line tasks, helping you choose a model for development, debugging and technical operations with less manual execution. Compare the same release and agent setup.","tracking_status":"tracked","unit":"%","version":"2.1 / Ripple launch / Terminus 2"}},{"id":92,"type":"benchmark","entityId":"gdp-pdf-inkling2-launch","action":"updated","label":"GDP.pdf","modelId":null,"benchmarkSlug":"gdp-pdf","source":"https://surgehq.ai/benchmarks/gdp-pdf","previous":{"benchmark_homepage_url":"https://surgehq.ai/benchmarks/gdp-pdf","category":"professional document reasoning","created_at":"2026-11-08T23:23:39.765Z","description":"GDP.pdf score transcribed from model card. Attempt count and All-pass protocol are not established in this source; excluded from the pinned independent track.","display_name":"GDP.pdf","higher_is_better":1,"id":"gdp-pdf-inkling2-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"GDP.pdf","official_url":"https://artificialanalysis.ai/evaluations/gdp-pdf","page_slug":"gdp-pdf","summary":"GDP.pdf tests reasoning over long professional documents across ten domains. The All-pass score measures attempts satisfying every task criterion. These results use the independent five-attempt evaluation with LiteParse text and page images for models supporting images; other implementations remain separate.","tracking_status":"tracked","unit":"%","version":"Inkling 2 Small launch / protocol unspecified"},"current":{"benchmark_homepage_url":"https://surgehq.ai/benchmarks/gdp-pdf","category":"professional document reasoning","created_at":"2026-11-08T23:23:39.765Z","description":"GDP.pdf score transcribed from model card. Attempt count and All-pass protocol are not established in this source; excluded from the pinned independent track.","display_name":"GDP.pdf","higher_is_better":1,"id":"gdp-pdf-inkling2-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"GDP.pdf","official_url":"https://artificialanalysis.ai/evaluations/gdp-pdf","page_slug":"gdp-pdf","summary":"Tests reasoning over long professional documents, helping you assess which model could reduce the time you spend working through source material for client projects. All-pass scores require every criterion to be satisfied; implementations are kept separate.","tracking_status":"tracked","unit":"%","version":"Inkling 2 Small launch / protocol unspecified"}},{"id":93,"type":"benchmark","entityId":"hle-diamond-inkling2-launch","action":"updated","label":"HLE-Diamond","modelId":null,"benchmarkSlug":"hle-diamond","source":"https://lastexam.ai/","previous":{"benchmark_homepage_url":"https://lastexam.ai/","category":"advanced knowledge and reasoning","created_at":"2026-11-08T23:23:39.765Z","description":"HLE-Diamond model-card result. Do not infer tools or independent protocol from the benchmark name.","display_name":"HLE-Diamond","higher_is_better":1,"id":"hle-diamond-inkling2-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"HLE-Diamond","official_url":"https://artificialanalysis.ai/evaluations/hle-diamond","page_slug":"hle-diamond","summary":"HLE-Diamond is a revised 1,000-question version of Humanity’s Last Exam, split into 500 reasoning and 500 knowledge questions. These results use the official prompts without tools. Scores are separate from the original Humanity’s Last Exam.","tracking_status":"tracked","unit":"%","version":"Inkling 2 Small launch / tool setting unspecified"},"current":{"benchmark_homepage_url":"https://lastexam.ai/","category":"advanced knowledge and reasoning","created_at":"2026-11-08T23:23:39.765Z","description":"HLE-Diamond model-card result. Do not infer tools or independent protocol from the benchmark name.","display_name":"HLE-Diamond","higher_is_better":1,"id":"hle-diamond-inkling2-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"HLE-Diamond","official_url":"https://artificialanalysis.ai/evaluations/hle-diamond","page_slug":"hle-diamond","summary":"Tests difficult knowledge and reasoning questions, helping you choose a model for demanding analysis before you trust it with client research. This 1,000-question evaluation uses no tools and is separate from the original Humanity's Last Exam.","tracking_status":"tracked","unit":"%","version":"Inkling 2 Small launch / tool setting unspecified"}},{"id":94,"type":"benchmark","entityId":"mcp-atlas-ripple-launch","action":"updated","label":"MCP-Atlas","modelId":null,"benchmarkSlug":"mcp-atlas","source":"https://labs.scale.com/leaderboard/mcp_atlas","previous":{"benchmark_homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","category":"Tool use","created_at":"2026-11-08T23:23:39.765Z","description":"MCP Atlas score in Ricursive model card; public versus full subset is not established, so this is not the public-500 track.","display_name":"MCP-Atlas","higher_is_better":1,"id":"mcp-atlas-ripple-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"MCP-Atlas","official_url":"https://labs.scale.com/leaderboard/mcp_atlas","page_slug":"mcp-atlas","summary":"MCP-Atlas evaluates real-world tool use through the Model Context Protocol. It tests tool discovery, choosing correct arguments, recovering from errors, and combining tool outputs to complete multi-step workflows. The stored results below cover the public subset under the April 2026 protocol.","tracking_status":"tracked","unit":"%","version":"Ripple launch / subset unspecified"},"current":{"benchmark_homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","category":"Tool use","created_at":"2026-11-08T23:23:39.765Z","description":"MCP Atlas score in Ricursive model card; public versus full subset is not established, so this is not the public-500 track.","display_name":"MCP-Atlas","higher_is_better":1,"id":"mcp-atlas-ripple-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"MCP-Atlas","official_url":"https://labs.scale.com/leaderboard/mcp_atlas","page_slug":"mcp-atlas","summary":"Tests whether an agent can find and use the right connected tools, helping you assess its ability to complete workflows across your business systems with less manual coordination. These results cover the public April 2026 subset.","tracking_status":"tracked","unit":"%","version":"Ripple launch / subset unspecified"}},{"id":95,"type":"benchmark","entityId":"automationbench-ripple-public","action":"updated","label":"AutomationBench","modelId":null,"benchmarkSlug":"automationbench","source":"https://zapier.com/benchmarks","previous":{"benchmark_homepage_url":"https://zapier.com/benchmarks","category":"Work automations","created_at":"2026-11-08T23:23:39.765Z","description":"Ricursive's public AutomationBench score. Do not merge with the held-out private or known-version tracks.","display_name":"AutomationBench","higher_is_better":1,"id":"automationbench-ripple-public","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict task completion","minimum_score":0,"name":"AutomationBench","official_url":"https://z.ai/blog/glm-5.3-flash","page_slug":"automationbench","summary":"AutomationBench tests agents on realistic business workflows across applications and APIs, covering sales, marketing, operations, support, finance, and HR. It evaluates whether agents can coordinate multi-step work and complete tasks. Zapier’s hosted leaderboard measures strict task completion on a private held-out set.","tracking_status":"tracked","unit":"%","version":"Ripple launch / public / version unspecified"},"current":{"benchmark_homepage_url":"https://zapier.com/benchmarks","category":"Work automations","created_at":"2026-11-08T23:23:39.765Z","description":"Ricursive's public AutomationBench score. Do not merge with the held-out private or known-version tracks.","display_name":"AutomationBench","higher_is_better":1,"id":"automationbench-ripple-public","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict task completion","minimum_score":0,"name":"AutomationBench","official_url":"https://z.ai/blog/glm-5.3-flash","page_slug":"automationbench","summary":"Tests whether an agent can finish business workflows across apps and APIs, helping you assess what sales, support or operations work you could automate to spend more time with clients. The hosted Zapier track measures complete tasks on a private test set.","tracking_status":"tracked","unit":"%","version":"Ripple launch / public / version unspecified"}},{"id":96,"type":"benchmark","entityId":"omnidocbench","action":"updated","label":"OmniDocBench","modelId":null,"benchmarkSlug":"omnidocbench","source":"https://github.com/opendatalab/OmniDocBench","previous":{"benchmark_homepage_url":"https://github.com/opendatalab/OmniDocBench","category":"Knowledge work","created_at":"2026-11-08T23:23:39.765Z","description":"OmniDocBench evaluates document parsing across varied layouts, including text, tables and formulas. This result is the publisher's reported v1.5 score.","display_name":"OmniDocBench","higher_is_better":1,"id":"omnidocbench","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Reported score","minimum_score":0,"name":"OmniDocBench","official_url":"https://github.com/opendatalab/OmniDocBench","page_slug":"omnidocbench","summary":"OmniDocBench evaluates document parsing across varied layouts, including text, tables and formulas. This result is the publisher's reported v1.5 score.","tracking_status":"tracked","unit":"%","version":"1.5 / Ant Ling launch"},"current":{"benchmark_homepage_url":"https://github.com/opendatalab/OmniDocBench","category":"Knowledge work","created_at":"2026-11-08T23:23:39.765Z","description":"OmniDocBench evaluates document parsing across varied layouts, including text, tables and formulas. This result is the publisher's reported v1.5 score.","display_name":"OmniDocBench","higher_is_better":1,"id":"omnidocbench","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Reported score","minimum_score":0,"name":"OmniDocBench","official_url":"https://github.com/opendatalab/OmniDocBench","page_slug":"omnidocbench","summary":"Tests extracting text, tables and formulas from documents, helping you choose a model for turning client files into usable data with less manual copying. These results use the reported version 1.5 evaluation.","tracking_status":"tracked","unit":"%","version":"1.5 / Ant Ling launch"}},{"id":97,"type":"benchmark","entityId":"zerobench","action":"updated","label":"ZeroBench","modelId":null,"benchmarkSlug":"zerobench","source":"https://zerobench.github.io/","previous":{"benchmark_homepage_url":"https://zerobench.github.io/","category":"reasoning","created_at":"2026-11-08T23:23:39.765Z","description":"ZeroBench evaluates challenging visual reasoning. This result uses the subquestion track, not full-question accuracy.","display_name":"ZeroBench","higher_is_better":1,"id":"zerobench","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Subquestion accuracy","minimum_score":0,"name":"ZeroBench","official_url":"https://zerobench.github.io/","page_slug":"zerobench","summary":"ZeroBench evaluates challenging visual reasoning. This result uses the subquestion track, not full-question accuracy.","tracking_status":"tracked","unit":"%","version":"Sub / Ant Ling launch"},"current":{"benchmark_homepage_url":"https://zerobench.github.io/","category":"reasoning","created_at":"2026-11-08T23:23:39.765Z","description":"ZeroBench evaluates challenging visual reasoning. This result uses the subquestion track, not full-question accuracy.","display_name":"ZeroBench","higher_is_better":1,"id":"zerobench","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Subquestion accuracy","minimum_score":0,"name":"ZeroBench","official_url":"https://zerobench.github.io/","page_slug":"zerobench","summary":"Tests difficult visual reasoning, helping you assess which model can interpret complex visual material before using it in client analysis. These scores measure the subquestion track, not complete-question accuracy.","tracking_status":"tracked","unit":"%","version":"Sub / Ant Ling launch"}},{"id":98,"type":"benchmark","entityId":"vision2web","action":"updated","label":"Vision2Web","modelId":null,"benchmarkSlug":"vision2web","source":"https://vision2web-bench.github.io/","previous":{"benchmark_homepage_url":"https://vision2web-bench.github.io/","category":"Coding agents","created_at":"2026-11-08T23:23:39.765Z","description":"Vision2Web evaluates agents building functional websites from visual prototypes and requirements. Launch-report scores retain their publisher's setup.","display_name":"Vision2Web","higher_is_better":1,"id":"vision2web","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Reported score","minimum_score":0,"name":"Vision2Web","official_url":"https://vision2web-bench.github.io/","page_slug":"vision2web","summary":"Vision2Web evaluates agents building functional websites from visual prototypes and requirements. Launch-report scores retain their publisher's setup.","tracking_status":"tracked","unit":"%","version":"Ant Ling launch / protocol unspecified"},"current":{"benchmark_homepage_url":"https://vision2web-bench.github.io/","category":"Coding agents","created_at":"2026-11-08T23:23:39.765Z","description":"Vision2Web evaluates agents building functional websites from visual prototypes and requirements. Launch-report scores retain their publisher's setup.","display_name":"Vision2Web","higher_is_better":1,"id":"vision2web","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Reported score","minimum_score":0,"name":"Vision2Web","official_url":"https://vision2web-bench.github.io/","page_slug":"vision2web","summary":"Tests building functional websites from visual prototypes and requirements, helping you compare models for turning a client's design into working code. Scores retain the publisher's evaluation setup.","tracking_status":"tracked","unit":"%","version":"Ant Ling launch / protocol unspecified"}},{"id":99,"type":"benchmark","entityId":"exploitgym","action":"updated","label":"ExploitGym","modelId":null,"benchmarkSlug":"exploitgym","source":"https://openai.com/index/introducing-gpt-6-cyber/","previous":{"benchmark_homepage_url":"https://openai.com/index/introducing-gpt-6-cyber/","category":"Coding agents","created_at":"2026-11-08T23:23:39.765Z","description":"ExploitGym evaluates security agents discovering and exploiting vulnerabilities in controlled software environments. Results here are from OpenAI's GPT-6 Cyber launch evaluation.","display_name":"ExploitGym","higher_is_better":1,"id":"exploitgym","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"ExploitGym","official_url":"https://openai.com/index/introducing-gpt-6-cyber/","page_slug":"exploitgym","summary":"ExploitGym evaluates security agents discovering and exploiting vulnerabilities in controlled software environments. Results here are from OpenAI's GPT-6 Cyber launch evaluation.","tracking_status":"tracked","unit":"%","version":"OpenAI GPT-6 Cyber launch"},"current":{"benchmark_homepage_url":"https://openai.com/index/introducing-gpt-6-cyber/","category":"Coding agents","created_at":"2026-11-08T23:23:39.765Z","description":"ExploitGym evaluates security agents discovering and exploiting vulnerabilities in controlled software environments. Results here are from OpenAI's GPT-6 Cyber launch evaluation.","display_name":"ExploitGym","higher_is_better":1,"id":"exploitgym","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"ExploitGym","official_url":"https://openai.com/index/introducing-gpt-6-cyber/","page_slug":"exploitgym","summary":"Tests finding and exploiting vulnerabilities in controlled software environments, helping authorized security teams assess AI support for testing and remediation work. These launch-reported results do not measure security across an entire client system.","tracking_status":"tracked","unit":"%","version":"OpenAI GPT-6 Cyber launch"}},{"id":100,"type":"benchmark","entityId":"remoteexploitbench","action":"updated","label":"RemoteExploitBench","modelId":null,"benchmarkSlug":"remoteexploitbench","source":"https://openai.com/index/introducing-gpt-6-cyber/","previous":{"benchmark_homepage_url":"https://openai.com/index/introducing-gpt-6-cyber/","category":"Coding agents","created_at":"2026-11-08T23:23:39.766Z","description":"RemoteExploitBench evaluates exploit development against known vulnerabilities in remotely deployed applications in a controlled environment.","display_name":"RemoteExploitBench","higher_is_better":1,"id":"remoteexploitbench","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"RemoteExploitBench","official_url":"https://openai.com/index/introducing-gpt-6-cyber/","page_slug":"remoteexploitbench","summary":"RemoteExploitBench evaluates exploit development against known vulnerabilities in remotely deployed applications in a controlled environment.","tracking_status":"tracked","unit":"%","version":"OpenAI GPT-6 Cyber launch"},"current":{"benchmark_homepage_url":"https://openai.com/index/introducing-gpt-6-cyber/","category":"Coding agents","created_at":"2026-11-08T23:23:39.766Z","description":"RemoteExploitBench evaluates exploit development against known vulnerabilities in remotely deployed applications in a controlled environment.","display_name":"RemoteExploitBench","higher_is_better":1,"id":"remoteexploitbench","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Task success","minimum_score":0,"name":"RemoteExploitBench","official_url":"https://openai.com/index/introducing-gpt-6-cyber/","page_slug":"remoteexploitbench","summary":"Tests exploit development against known vulnerabilities in controlled, remotely deployed apps, helping authorized security teams assess how AI could support validation of security findings. Results do not establish real-world audit coverage.","tracking_status":"tracked","unit":"%","version":"OpenAI GPT-6 Cyber launch"}},{"id":101,"type":"benchmark","entityId":"frontiercode-2-0","action":"updated","label":"FrontierCode","modelId":null,"benchmarkSlug":"frontiercode","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":{"benchmark_homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Evaluates mergeable software changes across 132 repository tasks using functional checks and rubrics. Native agent harnesses and reasoning efforts are recorded per report.","display_name":"FrontierCode","higher_is_better":1,"id":"frontiercode-2-0","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"FrontierCode","official_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","page_slug":"frontiercode","summary":"Evaluates mergeable software changes across 132 repository tasks using functional checks and rubrics. Native agent harnesses and reasoning efforts are recorded per report.","tracking_status":"tracked","unit":"%","version":"2.0 / Cognition native agents"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Evaluates mergeable software changes across 132 repository tasks using functional checks and rubrics. Native agent harnesses and reasoning efforts are recorded per report.","display_name":"FrontierCode","higher_is_better":1,"id":"frontiercode-2-0","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"FrontierCode","official_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","page_slug":"frontiercode","summary":"Tests whether an agent can produce software changes that meet functional and review requirements, helping you assess how much implementation work you could delegate before human code review. Scores cover 132 tasks and depend on the agent setup and effort level.","tracking_status":"tracked","unit":"%","version":"2.0 / Cognition native agents"}},{"id":102,"type":"benchmark","entityId":"frontiercode-2-0-aa","action":"updated","label":"FrontierCode","modelId":null,"benchmarkSlug":"frontiercode","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":{"benchmark_homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Independent evaluation, three repeats and mini-swe-agent; distinct from native CLI runs.","display_name":"FrontierCode","higher_is_better":1,"id":"frontiercode-2-0-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"FrontierCode","official_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","page_slug":"frontiercode","summary":"Evaluates mergeable software changes across 132 repository tasks using functional checks and rubrics. Native agent harnesses and reasoning efforts are recorded per report.","tracking_status":"tracked","unit":"%","version":"2.0 / independent mini-swe-agent"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Independent evaluation, three repeats and mini-swe-agent; distinct from native CLI runs.","display_name":"FrontierCode","higher_is_better":1,"id":"frontiercode-2-0-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"FrontierCode","official_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","page_slug":"frontiercode","summary":"Tests whether an agent can produce software changes that meet functional and review requirements, helping you assess how much implementation work you could delegate before human code review. Scores cover 132 tasks and depend on the agent setup and effort level.","tracking_status":"tracked","unit":"%","version":"2.0 / independent mini-swe-agent"}},{"id":103,"type":"benchmark","entityId":"frontierswe-v2","action":"updated","label":"FrontierSWE","modelId":null,"benchmarkSlug":"frontierswe","source":"https://www.frontierswe.com/blog/v2","previous":{"benchmark_homepage_url":"https://www.frontierswe.com/blog/v2","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures ultra-long-horizon software engineering and research across 34 tasks, with a 20-hour budget per trial and five trials per task.","display_name":"FrontierSWE","higher_is_better":1,"id":"frontierswe-v2","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"FrontierSWE","official_url":"https://www.frontierswe.com/blog/v2","page_slug":"frontierswe","summary":"Measures ultra-long-horizon software engineering and research across 34 tasks, with a 20-hour budget per trial and five trials per task.","tracking_status":"tracked","unit":"%","version":"2 / Proximal harness"},"current":{"benchmark_homepage_url":"https://www.frontierswe.com/blog/v2","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures ultra-long-horizon software engineering and research across 34 tasks, with a 20-hour budget per trial and five trials per task.","display_name":"FrontierSWE","higher_is_better":1,"id":"frontierswe-v2","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"FrontierSWE","official_url":"https://www.frontierswe.com/blog/v2","page_slug":"frontierswe","summary":"Tests substantial engineering and research assignments, helping you assess models for complex client work that extends beyond a quick code change. Each of the 34 tasks has a 20-hour trial budget; scores do not translate directly into billable hours saved.","tracking_status":"tracked","unit":"%","version":"2 / Proximal harness"}},{"id":104,"type":"benchmark","entityId":"terminal-bench-science-0-1-native","action":"updated","label":"Terminal-Bench-Science","modelId":null,"benchmarkSlug":"terminal-bench-science","source":"https://doi.org/10.5281/zenodo.22110253","previous":{"benchmark_homepage_url":"https://doi.org/10.5281/zenodo.22110253","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures scientific research workflows across 70 terminal tasks. These native Claude Code runs use cached resources and offline search; repeated-trial scores remain distinct from other harnesses.","display_name":"Terminal-Bench-Science","higher_is_better":1,"id":"terminal-bench-science-0-1-native","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Terminal-Bench-Science","official_url":"https://doi.org/10.5281/zenodo.22110253","page_slug":"terminal-bench-science","summary":"Measures scientific research workflows across 70 terminal tasks. These native Claude Code runs use cached resources and offline search; repeated-trial scores remain distinct from other harnesses.","tracking_status":"tracked","unit":"%","version":"0.1 / Claude Code / offline"},"current":{"benchmark_homepage_url":"https://doi.org/10.5281/zenodo.22110253","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures scientific research workflows across 70 terminal tasks. These native Claude Code runs use cached resources and offline search; repeated-trial scores remain distinct from other harnesses.","display_name":"Terminal-Bench-Science","higher_is_better":1,"id":"terminal-bench-science-0-1-native","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Terminal-Bench-Science","official_url":"https://doi.org/10.5281/zenodo.22110253","page_slug":"terminal-bench-science","summary":"Tests computational research workflows, helping you assess what analysis and execution you could delegate so you can focus on interpreting results. Native-agent and independent setups remain separate, and success does not establish a scientific discovery's validity.","tracking_status":"tracked","unit":"%","version":"0.1 / Claude Code / offline"}},{"id":105,"type":"benchmark","entityId":"terminal-bench-science-0-1-aa","action":"updated","label":"Terminal-Bench-Science","modelId":null,"benchmarkSlug":"terminal-bench-science","source":"https://doi.org/10.5281/zenodo.22110253","previous":{"benchmark_homepage_url":"https://doi.org/10.5281/zenodo.22110253","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Independent 70-task evaluation with three repeats in mini-swe-agent.","display_name":"Terminal-Bench-Science","higher_is_better":1,"id":"terminal-bench-science-0-1-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Terminal-Bench-Science","official_url":"https://doi.org/10.5281/zenodo.22110253","page_slug":"terminal-bench-science","summary":"Measures scientific research workflows across 70 terminal tasks. These native Claude Code runs use cached resources and offline search; repeated-trial scores remain distinct from other harnesses.","tracking_status":"tracked","unit":"%","version":"0.1 / independent mini-swe-agent"},"current":{"benchmark_homepage_url":"https://doi.org/10.5281/zenodo.22110253","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Independent 70-task evaluation with three repeats in mini-swe-agent.","display_name":"Terminal-Bench-Science","higher_is_better":1,"id":"terminal-bench-science-0-1-aa","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Terminal-Bench-Science","official_url":"https://doi.org/10.5281/zenodo.22110253","page_slug":"terminal-bench-science","summary":"Tests computational research workflows, helping you assess what analysis and execution you could delegate so you can focus on interpreting results. Native-agent and independent setups remain separate, and success does not establish a scientific discovery's validity.","tracking_status":"tracked","unit":"%","version":"0.1 / independent mini-swe-agent"}},{"id":106,"type":"benchmark","entityId":"osworld-2-1-offline-partial-anthropic","action":"updated","label":"OSWorld","modelId":null,"benchmarkSlug":"osworld","source":"https://osworld-v2.xlang.ai/","previous":{"benchmark_homepage_url":"https://osworld-v2.xlang.ai/","category":"Computer use","created_at":"2026-11-09T23:48:06.259Z","description":"Official 82-task offline subset, five attempts, 500 action steps and 1080p. Partial credit is not strict task success or full-108 performance.","display_name":"OSWorld","higher_is_better":1,"id":"osworld-2-1-offline-partial-anthropic","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Partial-credit score","minimum_score":0,"name":"OSWorld","official_url":"https://osworld-v2.xlang.ai/","page_slug":"osworld","summary":"OSWorld evaluates computer-use agents on practical tasks in a desktop environment. It tests whether agents can interact with applications and complete multi-step work through a graphical interface. Results below are restricted to the stored 2.1 full-set evaluation.","tracking_status":"tracked","unit":"%","version":"2.1 / offline 82 / partial credit"},"current":{"benchmark_homepage_url":"https://osworld-v2.xlang.ai/","category":"Computer use","created_at":"2026-11-09T23:48:06.259Z","description":"Official 82-task offline subset, five attempts, 500 action steps and 1080p. Partial credit is not strict task success or full-108 performance.","display_name":"OSWorld","higher_is_better":1,"id":"osworld-2-1-offline-partial-anthropic","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Partial-credit score","minimum_score":0,"name":"OSWorld","official_url":"https://osworld-v2.xlang.ai/","page_slug":"osworld","summary":"Tests whether an agent can carry out work through desktop applications, helping you assess which computer tasks you might delegate instead of doing every click yourself. Full-set, offline-subset, partial-credit and strict-success results measure different things.","tracking_status":"tracked","unit":"%","version":"2.1 / offline 82 / partial credit"}},{"id":107,"type":"benchmark","entityId":"osworld-2-1-offline-strict-anthropic","action":"updated","label":"OSWorld","modelId":null,"benchmarkSlug":"osworld","source":"https://osworld-v2.xlang.ai/","previous":{"benchmark_homepage_url":"https://osworld-v2.xlang.ai/","category":"Computer use","created_at":"2026-11-09T23:48:06.259Z","description":"Official 82-task offline subset; every checkpoint must pass. Five attempts, 500 action steps and 1080p.","display_name":"OSWorld","higher_is_better":1,"id":"osworld-2-1-offline-strict-anthropic","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict pass rate","minimum_score":0,"name":"OSWorld","official_url":"https://osworld-v2.xlang.ai/","page_slug":"osworld","summary":"OSWorld evaluates computer-use agents on practical tasks in a desktop environment. It tests whether agents can interact with applications and complete multi-step work through a graphical interface. Results below are restricted to the stored 2.1 full-set evaluation.","tracking_status":"tracked","unit":"%","version":"2.1 / offline 82 / strict pass"},"current":{"benchmark_homepage_url":"https://osworld-v2.xlang.ai/","category":"Computer use","created_at":"2026-11-09T23:48:06.259Z","description":"Official 82-task offline subset; every checkpoint must pass. Five attempts, 500 action steps and 1080p.","display_name":"OSWorld","higher_is_better":1,"id":"osworld-2-1-offline-strict-anthropic","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Strict pass rate","minimum_score":0,"name":"OSWorld","official_url":"https://osworld-v2.xlang.ai/","page_slug":"osworld","summary":"Tests whether an agent can carry out work through desktop applications, helping you assess which computer tasks you might delegate instead of doing every click yourself. Full-set, offline-subset, partial-credit and strict-success results measure different things.","tracking_status":"tracked","unit":"%","version":"2.1 / offline 82 / strict pass"}},{"id":108,"type":"benchmark","entityId":"aa-briefcase-v1-1-rubric","action":"updated","label":"AA-Briefcase","modelId":null,"benchmarkSlug":"aa-briefcase","source":"https://artificialanalysis.ai/evaluations/aa-briefcase","previous":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","category":"business deliverables","created_at":"2026-11-09T23:48:06.259Z","description":"AA-Briefcase evaluates agents producing spreadsheets, presentations, memos and other professional deliverables across 91 tasks. Its combined Elo incorporates rubric performance, analytical quality and presentation. Each task runs independently, and the public Lite example is excluded.","display_name":"AA-Briefcase","higher_is_better":1,"id":"aa-briefcase-v1-1-rubric","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Rubric pass rate","minimum_score":0,"name":"AA-Briefcase","official_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","page_slug":"aa-briefcase","summary":"AA-Briefcase evaluates agents producing spreadsheets, presentations, memos and other professional deliverables across 91 tasks. Its combined Elo incorporates rubric performance, analytical quality and presentation. Each task runs independently, and the public Lite example is excluded.","tracking_status":"tracked","unit":"%","version":"1.1 / rubric pass rate"},"current":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","category":"business deliverables","created_at":"2026-11-09T23:48:06.259Z","description":"AA-Briefcase evaluates agents producing spreadsheets, presentations, memos and other professional deliverables across 91 tasks. Its combined Elo incorporates rubric performance, analytical quality and presentation. Each task runs independently, and the public Lite example is excluded.","display_name":"AA-Briefcase","higher_is_better":1,"id":"aa-briefcase-v1-1-rubric","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Rubric pass rate","minimum_score":0,"name":"AA-Briefcase","official_url":"https://artificialanalysis.ai/evaluations/aa-briefcase","page_slug":"aa-briefcase","summary":"Tests professional deliverables such as spreadsheets, presentations and memos, helping you choose a model for complex client projects with many linked requirements. Rubric scores measure requirements met; Elo also reflects analytical and presentation quality, not hours saved.","tracking_status":"tracked","unit":"%","version":"1.1 / rubric pass rate"}},{"id":109,"type":"benchmark","entityId":"wandr-anthropic-relaxed","action":"updated","label":"WANDR","modelId":null,"benchmarkSlug":"wandr","source":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","previous":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"web research","created_at":"2026-11-09T23:48:06.259Z","description":"Evaluates wide web research and structured fact collection across 500 tasks. This modified offline-search evaluation uses soft F1 and differs from Perplexity’s live-web harness.","display_name":"WANDR","higher_is_better":1,"id":"wandr-anthropic-relaxed","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Soft F1","minimum_score":0,"name":"WANDR","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"wandr","summary":"Evaluates wide web research and structured fact collection across 500 tasks. This modified offline-search evaluation uses soft F1 and differs from Perplexity’s live-web harness.","tracking_status":"tracked","unit":"%","version":"Anthropic offline / relaxed soft F1"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"web research","created_at":"2026-11-09T23:48:06.259Z","description":"Evaluates wide web research and structured fact collection across 500 tasks. This modified offline-search evaluation uses soft F1 and differs from Perplexity’s live-web harness.","display_name":"WANDR","higher_is_better":1,"id":"wandr-anthropic-relaxed","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Soft F1","minimum_score":0,"name":"WANDR","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"wandr","summary":"Tests collecting and organizing facts across a broad research assignment, helping you assess AI support for client research that involves many separate lookups. This offline-search soft-F1 evaluation differs from live-web runs and is not a project-completion percentage.","tracking_status":"tracked","unit":"%","version":"Anthropic offline / relaxed soft F1"}},{"id":110,"type":"benchmark","entityId":"chartography","action":"updated","label":"Chartography","modelId":null,"benchmarkSlug":"chartography","source":"https://surgehq.ai/blog/chartography","previous":{"benchmark_homepage_url":"https://surgehq.ai/blog/chartography","category":"reasoning","created_at":"2026-11-09T23:48:06.259Z","description":"Tests reading specialized professional charts across 100 tasks. Tool-free and tool-assisted results are recorded separately in the report settings.","display_name":"Chartography","higher_is_better":1,"id":"chartography","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Chartography","official_url":"https://surgehq.ai/blog/chartography","page_slug":"chartography","summary":"Tests reading specialized professional charts across 100 tasks. Tool-free and tool-assisted results are recorded separately in the report settings.","tracking_status":"tracked","unit":"%","version":"Anthropic implementation / Gemini 3.5 Flash grader"},"current":{"benchmark_homepage_url":"https://surgehq.ai/blog/chartography","category":"reasoning","created_at":"2026-11-09T23:48:06.259Z","description":"Tests reading specialized professional charts across 100 tasks. Tool-free and tool-assisted results are recorded separately in the report settings.","display_name":"Chartography","higher_is_better":1,"id":"chartography","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"Chartography","official_url":"https://surgehq.ai/blog/chartography","page_slug":"chartography","summary":"Tests reading professional charts accurately, helping you choose a model to support client reports and presentations with less manual interpretation. Results cover 100 tasks; tool-free and tool-assisted settings are recorded separately.","tracking_status":"tracked","unit":"%","version":"Anthropic implementation / Gemini 3.5 Flash grader"}},{"id":111,"type":"benchmark","entityId":"remote-labor-index","action":"updated","label":"Remote Labor Index","modelId":null,"benchmarkSlug":"remote-labor-index","source":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","previous":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Knowledge work","created_at":"2026-11-09T23:48:06.259Z","description":"Measures completion of real freelance projects against human professional deliverables. Human evaluators judge whether the agent’s work is at least as good as the reference.","display_name":"Remote Labor Index","higher_is_better":1,"id":"remote-labor-index","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Automation rate","minimum_score":0,"name":"Remote Labor Index","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"remote-labor-index","summary":"Measures completion of real freelance projects against human professional deliverables. Human evaluators judge whether the agent’s work is at least as good as the reference.","tracking_status":"tracked","unit":"%","version":"240 freelance projects"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Knowledge work","created_at":"2026-11-09T23:48:06.259Z","description":"Measures completion of real freelance projects against human professional deliverables. Human evaluators judge whether the agent’s work is at least as good as the reference.","display_name":"Remote Labor Index","higher_is_better":1,"id":"remote-labor-index","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Automation rate","minimum_score":0,"name":"Remote Labor Index","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"remote-labor-index","summary":"Tests whether an agent can complete real freelance projects to a human professional's standard, helping you assess what client work you could delegate to make more time for your clients. The score measures projects completed, not the share of your workload or hours you will save.","tracking_status":"tracked","unit":"%","version":"240 freelance projects"}},{"id":112,"type":"benchmark","entityId":"swe-bench-pro-anthropic-2026-11-09","action":"updated","label":"SWE-bench Pro","modelId":null,"benchmarkSlug":"swe-bench-pro","source":"https://arxiv.org/abs/2509.16941","previous":{"benchmark_homepage_url":"https://arxiv.org/abs/2509.16941","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Long-horizon repository issues; the system card does not establish a version, so this is not labeled v2 Hard.","display_name":"SWE-bench Pro","higher_is_better":1,"id":"swe-bench-pro-anthropic-2026-11-09","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"SWE-bench Pro","official_url":"https://arxiv.org/abs/2509.16941","page_slug":"swe-bench-pro","summary":"Long-horizon repository issues; the system card does not establish a version, so this is not labeled v2 Hard.","tracking_status":"tracked","unit":"%","version":"Anthropic November 2026 / five trials"},"current":{"benchmark_homepage_url":"https://arxiv.org/abs/2509.16941","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Long-horizon repository issues; the system card does not establish a version, so this is not labeled v2 Hard.","display_name":"SWE-bench Pro","higher_is_better":1,"id":"swe-bench-pro-anthropic-2026-11-09","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"SWE-bench Pro","official_url":"https://arxiv.org/abs/2509.16941","page_slug":"swe-bench-pro","summary":"Tests substantial software repository issues, helping you assess which model could support complex fixes and implementation work for clients. The source does not establish a specific Pro release, so these scores are not labeled v2 Hard.","tracking_status":"tracked","unit":"%","version":"Anthropic November 2026 / five trials"}},{"id":113,"type":"benchmark","entityId":"swe-bench-multilingual-anthropic-2026-11-09","action":"updated","label":"SWE-bench Multilingual","modelId":null,"benchmarkSlug":"swe-bench-multilingual","source":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","previous":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"300 software tasks across nine programming languages.","display_name":"SWE-bench Multilingual","higher_is_better":1,"id":"swe-bench-multilingual-anthropic-2026-11-09","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"SWE-bench Multilingual","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"swe-bench-multilingual","summary":"300 software tasks across nine programming languages.","tracking_status":"tracked","unit":"%","version":"Anthropic November 2026 / five trials"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"300 software tasks across nine programming languages.","display_name":"SWE-bench Multilingual","higher_is_better":1,"id":"swe-bench-multilingual-anthropic-2026-11-09","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"SWE-bench Multilingual","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"swe-bench-multilingual","summary":"Tests software fixes across nine programming languages, helping you assess models for supporting client codebases beyond a single language. Results cover 300 tasks and do not measure developer productivity directly.","tracking_status":"tracked","unit":"%","version":"Anthropic November 2026 / five trials"}},{"id":114,"type":"benchmark","entityId":"swe-bench-multimodal-anthropic-2026-11-09","action":"updated","label":"SWE-bench Multimodal","modelId":null,"benchmarkSlug":"swe-bench-multimodal","source":"https://arxiv.org/abs/2410.03859","previous":{"benchmark_homepage_url":"https://arxiv.org/abs/2410.03859","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Software issues with screenshots and other visual context.","display_name":"SWE-bench Multimodal","higher_is_better":1,"id":"swe-bench-multimodal-anthropic-2026-11-09","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"SWE-bench Multimodal","official_url":"https://arxiv.org/abs/2410.03859","page_slug":"swe-bench-multimodal","summary":"Software issues with screenshots and other visual context.","tracking_status":"tracked","unit":"%","version":"Anthropic November 2026 / five trials"},"current":{"benchmark_homepage_url":"https://arxiv.org/abs/2410.03859","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Software issues with screenshots and other visual context.","display_name":"SWE-bench Multimodal","higher_is_better":1,"id":"swe-bench-multimodal-anthropic-2026-11-09","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"SWE-bench Multimodal","official_url":"https://arxiv.org/abs/2410.03859","page_slug":"swe-bench-multimodal","summary":"Tests software issues that include screenshots and visual context, helping you assess which model could connect a client's visible problem to the code changes needed to fix it.","tracking_status":"tracked","unit":"%","version":"Anthropic November 2026 / five trials"}},{"id":115,"type":"benchmark","entityId":"robodojo-sim","action":"updated","label":"RoboDojo Sim","modelId":null,"benchmarkSlug":"robodojo-sim","source":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","previous":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Evaluates agents on simulated robotics tasks. This is Anthropic’s internal run at highest available effort, not an independently reproduced result.","display_name":"RoboDojo Sim","higher_is_better":1,"id":"robodojo-sim","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"RoboDojo Sim","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"robodojo-sim","summary":"Evaluates agents on simulated robotics tasks. This is Anthropic’s internal run at highest available effort, not an independently reproduced result.","tracking_status":"tracked","unit":"%","version":"Anthropic internal simulation"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Evaluates agents on simulated robotics tasks. This is Anthropic’s internal run at highest available effort, not an independently reproduced result.","display_name":"RoboDojo Sim","higher_is_better":1,"id":"robodojo-sim","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"RoboDojo Sim","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"robodojo-sim","summary":"Tests controlling a simulated robot from visual information, helping you assess progress toward automating physical tasks. These are internal simulation results, not evidence of reliable operation in a client's workplace.","tracking_status":"tracked","unit":"%","version":"Anthropic internal simulation"}},{"id":116,"type":"benchmark","entityId":"cyscenariobench-anthropic","action":"updated","label":"CyScenarioBench","modelId":null,"benchmarkSlug":"cyscenariobench","source":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","previous":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures planning and execution across multi-stage security scenarios. Labs use different private subsets; the Anthropic 10-challenge cohort must not be compared as an identical test to another lab’s score.","display_name":"CyScenarioBench","higher_is_better":1,"id":"cyscenariobench-anthropic","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Overall solve rate","minimum_score":0,"name":"CyScenarioBench","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"cyscenariobench","summary":"Measures planning and execution across multi-stage security scenarios. Labs use different private subsets; the Anthropic 10-challenge cohort must not be compared as an identical test to another lab’s score.","tracking_status":"tracked","unit":"%","version":"Anthropic 10-challenge subset / prerelease"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures planning and execution across multi-stage security scenarios. Labs use different private subsets; the Anthropic 10-challenge cohort must not be compared as an identical test to another lab’s score.","display_name":"CyScenarioBench","higher_is_better":1,"id":"cyscenariobench-anthropic","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Overall solve rate","minimum_score":0,"name":"CyScenarioBench","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"cyscenariobench","summary":"Tests multi-stage cybersecurity scenarios, helping authorized security teams assess an agent's ability to plan and execute complex assessments. Labs use different private task sets, and safeguards settings are recorded with each report.","tracking_status":"tracked","unit":"%","version":"Anthropic 10-challenge subset / prerelease"}},{"id":117,"type":"benchmark","entityId":"cve-rediscovery-2026-10","action":"updated","label":"CVE Rediscovery","modelId":null,"benchmarkSlug":"cve-rediscovery","source":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","previous":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures rediscovery of documented security vulnerabilities from vulnerable repository snapshots, without internet access. This cohort contains 1,829 October 2026 advisories.","display_name":"CVE Rediscovery","higher_is_better":1,"id":"cve-rediscovery-2026-10","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"CVE Rediscovery","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"cve-rediscovery","summary":"Measures rediscovery of documented security vulnerabilities from vulnerable repository snapshots, without internet access. This cohort contains 1,829 October 2026 advisories.","tracking_status":"tracked","unit":"%","version":"October 2026 / 1829 advisories"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Coding agents","created_at":"2026-11-09T23:48:06.259Z","description":"Measures rediscovery of documented security vulnerabilities from vulnerable repository snapshots, without internet access. This cohort contains 1,829 October 2026 advisories.","display_name":"CVE Rediscovery","higher_is_better":1,"id":"cve-rediscovery-2026-10","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"CVE Rediscovery","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"cve-rediscovery","summary":"Tests finding previously documented vulnerabilities from vulnerable code snapshots, helping authorized security teams assess AI support for code audits. This 1,829-advisory evaluation does not establish how many unknown vulnerabilities it would find in a client's system.","tracking_status":"tracked","unit":"%","version":"October 2026 / 1829 advisories"}},{"id":118,"type":"benchmark","entityId":"de-novo-binder-design-24h","action":"updated","label":"De novo binder design","modelId":null,"benchmarkSlug":"de-novo-binder-design-24h","source":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","previous":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Knowledge work","created_at":"2026-11-09T23:48:06.259Z","description":"Measures autonomous protein binder design with a 24-hour budget per target, protein-design tools and one H200 GPU. This is a computational evaluation, not evidence of experimentally validated binding.","display_name":"De novo binder design","higher_is_better":1,"id":"de-novo-binder-design-24h","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Reported design score","minimum_score":0,"name":"De novo binder design","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"de-novo-binder-design-24h","summary":"Measures autonomous protein binder design with a 24-hour budget per target, protein-design tools and one H200 GPU. This is a computational evaluation, not evidence of experimentally validated binding.","tracking_status":"tracked","unit":"%","version":"15 targets / 24 hours / H200"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","category":"Knowledge work","created_at":"2026-11-09T23:48:06.259Z","description":"Measures autonomous protein binder design with a 24-hour budget per target, protein-design tools and one H200 GPU. This is a computational evaluation, not evidence of experimentally validated binding.","display_name":"De novo binder design","higher_is_better":1,"id":"de-novo-binder-design-24h","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Reported design score","minimum_score":0,"name":"De novo binder design","official_url":"https://www.anthropic.com/document/claude-fable-5-2-mythos-5-2-system-card","page_slug":"de-novo-binder-design-24h","summary":"Tests computational protein-binder design within a fixed time and hardware budget, helping research teams assess AI support for generating candidates. Scores do not establish experimentally validated binding or clinical usefulness.","tracking_status":"tracked","unit":"%","version":"15 targets / 24 hours / H200"}},{"id":119,"type":"benchmark","entityId":"livebench-2-agentic-2026-11-06","action":"updated","label":"LiveBench: Agentic","modelId":null,"benchmarkSlug":"livebench-agentic","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"long-running agentic work","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Agentic","higher_is_better":1,"id":"livebench-2-agentic-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Agentic","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-agentic","summary":"LiveBench 2.0 evaluates long-running agentic work on its October 25, 2026 release. This suite contains 18 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"long-running agentic work","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Agentic","higher_is_better":1,"id":"livebench-2-agentic-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Agentic","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-agentic","summary":"Tests extended tasks that require an agent to keep working toward a goal, helping you assess which model could handle longer assignments with less step-by-step direction. Compare results from the same dated task set.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"}},{"id":120,"type":"benchmark","entityId":"livebench-2-coding-2026-11-06","action":"updated","label":"LiveBench: Coding","modelId":null,"benchmarkSlug":"livebench-coding","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"coding with LiveBench","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Coding","higher_is_better":1,"id":"livebench-2-coding-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Coding","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-coding","summary":"LiveBench 2.0 evaluates coding with LiveBench on its October 25, 2026 release. This suite contains 29 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"coding with LiveBench","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Coding","higher_is_better":1,"id":"livebench-2-coding-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Coding","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-coding","summary":"Tests coding tasks, helping you compare models for implementation work that you would otherwise write or debug yourself. Compare results from the same dated task set and review generated changes before delivery.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"}},{"id":121,"type":"benchmark","entityId":"livebench-2-browser-use-2026-11-06","action":"updated","label":"LiveBench: Browser Use","modelId":null,"benchmarkSlug":"livebench-browser-use","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"browser workflows","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Browser Use","higher_is_better":1,"id":"livebench-2-browser-use-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Browser Use","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-browser-use","summary":"LiveBench 2.0 evaluates browser workflows on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"browser workflows","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Browser Use","higher_is_better":1,"id":"livebench-2-browser-use-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Browser Use","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-browser-use","summary":"Tests browser workflows, helping you choose an agent for repetitive online tasks so you can spend less time navigating websites yourself. Results from different dated task sets are kept separate.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"}},{"id":122,"type":"benchmark","entityId":"livebench-2-reasoning-2026-11-06","action":"updated","label":"LiveBench: Reasoning","modelId":null,"benchmarkSlug":"livebench-reasoning","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"reasoning with LiveBench","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Reasoning","higher_is_better":1,"id":"livebench-2-reasoning-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Reasoning","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-reasoning","summary":"LiveBench 2.0 evaluates reasoning with LiveBench on its October 25, 2026 release. This suite contains 202 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"reasoning with LiveBench","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Reasoning","higher_is_better":1,"id":"livebench-2-reasoning-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Reasoning","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-reasoning","summary":"Tests reasoning problems, helping you compare models for working through complex questions and checking your approach. This suite measures reasoning performance rather than completion of a business workflow.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"}},{"id":123,"type":"benchmark","entityId":"livebench-2-data-analysis-2026-11-06","action":"updated","label":"LiveBench: Data Analysis","modelId":null,"benchmarkSlug":"livebench-data-analysis","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"data analysis","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Data Analysis","higher_is_better":1,"id":"livebench-2-data-analysis-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Data Analysis","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-data-analysis","summary":"LiveBench 2.0 evaluates data analysis on its October 25, 2026 release. This suite contains 150 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"data analysis","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Data Analysis","higher_is_better":1,"id":"livebench-2-data-analysis-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Data Analysis","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-data-analysis","summary":"Tests data analysis, helping you choose a model to support turning client data into useful findings with less manual processing. Compare the same dated task set; a higher score does not guarantee correct analysis on your data.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"}},{"id":124,"type":"benchmark","entityId":"livebench-2-research-2026-11-06","action":"updated","label":"LiveBench: Research","modelId":null,"benchmarkSlug":"livebench-research","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"research reports","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Research","higher_is_better":1,"id":"livebench-2-research-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Research","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-research","summary":"LiveBench 2.0 evaluates research reports on its October 25, 2026 release. This suite contains 22 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"research reports","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Research","higher_is_better":1,"id":"livebench-2-research-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Research","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-research","summary":"Tests research report tasks, helping you compare assistants for gathering and organizing material before advising clients. Results are specific to each dated task set and do not measure time saved.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"}},{"id":125,"type":"benchmark","entityId":"livebench-2-design-2026-11-06","action":"updated","label":"LiveBench: Design","modelId":null,"benchmarkSlug":"livebench-design","source":"https://livebench.ai/#/methodology","previous":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"document and presentation creation","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Design","higher_is_better":1,"id":"livebench-2-design-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Design","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-design","summary":"LiveBench 2.0 evaluates document and presentation creation on its October 25, 2026 release. This suite contains 25 tasks or questions, scored from 0 to 100 in one run per model. Models use their highest supported effort unless a different setting is reported. This is an individual task-suite score, separate from LiveBench’s weighted overall average.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"},"current":{"benchmark_homepage_url":"https://livebench.ai/#/methodology","category":"document and presentation creation","created_at":"2026-11-09T23:48:06.259Z","description":"November 6, 2026 task-set release. Kept separate from October 25 and October 14; no model effort inferred.","display_name":"LiveBench: Design","higher_is_better":1,"id":"livebench-2-design-2026-11-06","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Accuracy","minimum_score":0,"name":"LiveBench: Design","official_url":"https://livebench.ai/#/release/2026-11-06","page_slug":"livebench-design","summary":"Tests document and presentation creation, helping you choose a model for preparing client-facing materials with less manual production work. Compare the same dated task set; the score does not measure turnaround time.","tracking_status":"tracked","unit":"%","version":"2.0 / 2026-11-06"}},{"id":126,"type":"benchmark","entityId":"cyscenariobench-openai-cyber-launch","action":"updated","label":"CyScenarioBench","modelId":null,"benchmarkSlug":"cyscenariobench","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":{"benchmark_homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","category":"Coding agents","created_at":"2026-11-10T01:04:32.954Z","description":"GPT-6 Cyber launch-table score. Each lab uses unique tasks; not directly comparable to the Anthropic 10-challenge subset.","display_name":"CyScenarioBench","higher_is_better":1,"id":"cyscenariobench-openai-cyber-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Overall solve rate","minimum_score":0,"name":"CyScenarioBench","official_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","page_slug":"cyscenariobench","summary":"GPT-6 Cyber score on its own task set, kept separate from Anthropic tasks.","tracking_status":"tracked","unit":"%","version":"OpenAI GPT-6 Cyber / unique task set"},"current":{"benchmark_homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","category":"Coding agents","created_at":"2026-11-10T01:04:32.954Z","description":"GPT-6 Cyber launch-table score. Each lab uses unique tasks; not directly comparable to the Anthropic 10-challenge subset.","display_name":"CyScenarioBench","higher_is_better":1,"id":"cyscenariobench-openai-cyber-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Overall solve rate","minimum_score":0,"name":"CyScenarioBench","official_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","page_slug":"cyscenariobench","summary":"Tests multi-stage cybersecurity scenarios, helping authorized security teams assess an agent's ability to plan and execute complex assessments. Labs use different private task sets, and safeguards settings are recorded with each report.","tracking_status":"tracked","unit":"%","version":"OpenAI GPT-6 Cyber / unique task set"}},{"id":127,"type":"benchmark","entityId":"cursorbench-4-0","action":"added","label":"CursorBench","modelId":null,"benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_homepage_url":"https://cursor.com/cursorbench","category":"Coding agents","created_at":"2026-11-10T01:17:35.861Z","description":"Ambiguous multi-file coding tasks in Cursor production agent harness; score is not task latency. Current November 9 publisher table includes effort-specific cost, output tokens and steps. Small score differences may be variance.","display_name":"CursorBench","higher_is_better":1,"id":"cursorbench-4-0","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Score","minimum_score":0,"name":"CursorBench","official_url":"https://cursor.com/cursorbench","page_slug":"cursorbench","summary":"Tests whether an agent can complete ambiguous, multi-file coding tasks from real Cursor sessions, helping you choose a model for client development work with less step-by-step direction. Scores, effort levels and task costs are specific to Cursor's agent setup.","tracking_status":"tracked","unit":"%","version":"4.0 / Cursor agent"}},{"id":128,"type":"benchmark","entityId":"real-world-finance-v2","action":"added","label":"Real-World Finance","modelId":null,"benchmarkSlug":"real-world-finance","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_homepage_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","category":"Knowledge work","created_at":"2026-11-10T01:17:35.861Z","description":"Internal financial analysis evaluation: 123 tasks, one attempt per model, three-hour limit, Opus 5.5 model grader. Grader Elo gaps divided by 2.61 and anchored to preserve the earlier scale. Fable missed six tasks; the headline Elo advantage drops from about 66 to 43 points if counted as losses. Not DeepFinance.","display_name":"Real-World Finance","higher_is_better":1,"id":"real-world-finance-v2","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":null,"metric":"Elo","minimum_score":0,"name":"Real-World Finance","official_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","page_slug":"real-world-finance","summary":"Compares financial models, audits and other professional finance deliverables, helping you assess AI support for preparing client analysis. Elo reflects a model grader's preferences across 123 tasks; it is not a completion percentage or a measure of time saved.","tracking_status":"tracked","unit":"Elo","version":"2 / November 2026 grader"}},{"id":129,"type":"benchmark","entityId":"benchcad-2-1-tools","action":"added","label":"BenchCAD","modelId":null,"benchmarkSlug":"benchcad","source":"https://benchcad.com/leaderboard","previous":null,"current":{"benchmark_homepage_url":"https://benchcad.com/leaderboard","category":"Knowledge work","created_at":"2026-11-10T01:17:35.862Z","description":"100 Core CAD tasks; five runs; max adaptive thinking; shell, file editor and image zoom; thinking-block clearing; no round or total-token limit; official scorer. Not directly comparable to standard-harness or older-version scores.","display_name":"BenchCAD","higher_is_better":1,"id":"benchcad-2-1-tools","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Mean partial-credit score","minimum_score":0,"name":"BenchCAD","official_url":"https://benchcad.com/leaderboard","page_slug":"benchcad","summary":"Tests rebuilding parts, assemblies and circuit boards from drawings and images, helping you assess AI support for client design and prototyping work. Scores award partial credit, and tool-assisted runs are separate from standard runs; manufacturing still requires validation.","tracking_status":"tracked","unit":"%","version":"2.1 / Core / with tools"}},{"id":130,"type":"benchmark","entityId":"benchcad-2-1-standard","action":"added","label":"BenchCAD","modelId":null,"benchmarkSlug":"benchcad","source":"https://benchcad.com/leaderboard","previous":null,"current":{"benchmark_homepage_url":"https://benchcad.com/leaderboard","category":"Knowledge work","created_at":"2026-11-10T01:17:35.862Z","description":"100 Core CAD tasks; five runs; max adaptive thinking; Anthropic implementation of standard protocol; render feedback; up to 60 rounds and 500,000 output tokens; official scorer. Without the extra shell/editor/zoom tool setup.","display_name":"BenchCAD","higher_is_better":1,"id":"benchcad-2-1-standard","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Mean partial-credit score","minimum_score":0,"name":"BenchCAD","official_url":"https://benchcad.com/leaderboard","page_slug":"benchcad","summary":"Tests rebuilding parts, assemblies and circuit boards from drawings and images, helping you assess AI support for client design and prototyping work. Scores award partial credit, and tool-assisted runs are separate from standard runs; manufacturing still requires validation.","tracking_status":"tracked","unit":"%","version":"2.1 / Core / standard"}},{"id":131,"type":"benchmark","entityId":"harvey-lab-aa-v1-1","action":"added","label":"Harvey LAB-AA","modelId":null,"benchmarkSlug":"harvey-lab-aa","source":"https://artificialanalysis.ai/evaluations/harvey-lab-aa","previous":null,"current":{"benchmark_homepage_url":"https://artificialanalysis.ai/evaluations/harvey-lab-aa","category":"Knowledge work","created_at":"2026-11-10T01:17:35.862Z","description":"Independent Artificial Analysis Stirrup evaluation on 120 private Harvey tasks; three-judge panel. Every rubric criterion must pass and there must be no material hallucination. Not comparable to v1.0 or ungated all-pass rates.","display_name":"Harvey LAB-AA","higher_is_better":1,"id":"harvey-lab-aa-v1-1","is_primary_track":1,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Hallucination-gated all-pass rate","minimum_score":0,"name":"Harvey LAB-AA","official_url":"https://artificialanalysis.ai/evaluations/harvey-lab-aa","page_slug":"harvey-lab-aa","summary":"Tests whether an agent can produce legal deliverables that meet every requirement without material hallucinations, helping you assess how much drafting and document work you could delegate before professional review. These results use the private 120-task v1.1 evaluation, not earlier LAB scoring.","tracking_status":"tracked","unit":"%","version":"1.1 / private 120 / hallucination-gated"}},{"id":132,"type":"result","entityId":"1389","action":"added","label":"Claude Fable 5.2 · CursorBench","modelId":"claude-fable-5-2","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 32.79, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 352794, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 218, \"tools\": true}","id":1389,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":60,"source_location":"CursorBench 4.0 table: Fable 5.2 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":133,"type":"result","entityId":"1390","action":"added","label":"Claude Opus 5.5 · CursorBench","modelId":"claude-opus-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 13.43, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 218363, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 185, \"tools\": true}","id":1390,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":57.8,"source_location":"CursorBench 4.0 table: Opus 5.5 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":134,"type":"result","entityId":"1391","action":"added","label":"Claude Fable 5.2 · CursorBench","modelId":"claude-fable-5-2","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 9.7, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 99259, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 69, \"tools\": true}","id":1391,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":56.9,"source_location":"CursorBench 4.0 table: Fable 5.2 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":135,"type":"result","entityId":"1392","action":"added","label":"Claude Fable 5.2 · CursorBench","modelId":"claude-fable-5-2","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 15.48, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 161251, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 108, \"tools\": true}","id":1392,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":56.3,"source_location":"CursorBench 4.0 table: Fable 5.2 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":136,"type":"result","entityId":"1393","action":"added","label":"Claude Opus 5.5 · CursorBench","modelId":"claude-opus-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 6.98, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 101083, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 109, \"tools\": true}","id":1393,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":56,"source_location":"CursorBench 4.0 table: Opus 5.5 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":137,"type":"result","entityId":"1394","action":"added","label":"Claude Opus 5.5 · CursorBench","modelId":"claude-opus-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 3.97, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 53078, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 68, \"tools\": true}","id":1394,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":56,"source_location":"CursorBench 4.0 table: Opus 5.5 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":138,"type":"result","entityId":"1395","action":"added","label":"Claude Sonnet 5.5 · CursorBench","modelId":"claude-sonnet-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 7.05, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 271920, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 170, \"tools\": true}","id":1395,"model_id":"claude-sonnet-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":55.5,"source_location":"CursorBench 4.0 table: Sonnet 5.5 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":139,"type":"result","entityId":"1396","action":"added","label":"Claude Fable 5.2 · CursorBench","modelId":"claude-fable-5-2","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.66, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 44103, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 35, \"tools\": true}","id":1396,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":54.8,"source_location":"CursorBench 4.0 table: Fable 5.2 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":140,"type":"result","entityId":"1397","action":"added","label":"Claude Fable 5.2 · CursorBench","modelId":"claude-fable-5-2","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 3.94, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 36616, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 30, \"tools\": true}","id":1397,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":53.2,"source_location":"CursorBench 4.0 table: Fable 5.2 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":141,"type":"result","entityId":"1398","action":"added","label":"Claude Sonnet 5.5 · CursorBench","modelId":"claude-sonnet-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 2.81, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 100158, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 78, \"tools\": true}","id":1398,"model_id":"claude-sonnet-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":53.1,"source_location":"CursorBench 4.0 table: Sonnet 5.5 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":142,"type":"result","entityId":"1399","action":"added","label":"Gemini 4 Argon · CursorBench","modelId":"gemini-4-argon","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.57, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 109500, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 118, \"tools\": true}","id":1399,"model_id":"gemini-4-argon","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":52.9,"source_location":"CursorBench 4.0 table: Gemini 4 Argon High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":143,"type":"result","entityId":"1400","action":"added","label":"Gemini 4 Argon · CursorBench","modelId":"gemini-4-argon","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.39, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 103600, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 115, \"tools\": true}","id":1400,"model_id":"gemini-4-argon","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":52.8,"source_location":"CursorBench 4.0 table: Gemini 4 Argon Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":144,"type":"result","entityId":"1401","action":"added","label":"Claude Opus 5.5 · CursorBench","modelId":"claude-opus-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 2.91, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 37954, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 54, \"tools\": true}","id":1401,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":52.5,"source_location":"CursorBench 4.0 table: Opus 5.5 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":145,"type":"result","entityId":"1402","action":"added","label":"Claude Fable 5.1 · CursorBench","modelId":"claude-fable-5-1","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 17.28, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 117236, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 128, \"tools\": true}","id":1402,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":51.8,"source_location":"CursorBench 4.0 table: Fable 5.1 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":146,"type":"result","entityId":"1403","action":"added","label":"Claude Fable 5.1 · CursorBench","modelId":"claude-fable-5-1","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 13.01, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 87294, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 101, \"tools\": true}","id":1403,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":51.6,"source_location":"CursorBench 4.0 table: Fable 5.1 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":147,"type":"result","entityId":"1404","action":"added","label":"Gemini 4 Argon · CursorBench","modelId":"gemini-4-argon","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.09, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 94200, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 110, \"tools\": true}","id":1404,"model_id":"gemini-4-argon","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":50.6,"source_location":"CursorBench 4.0 table: Gemini 4 Argon Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":148,"type":"result","entityId":"1405","action":"added","label":"Claude Fable 5.1 · CursorBench","modelId":"claude-fable-5-1","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 9.08, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 58438, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 77, \"tools\": true}","id":1405,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":49.2,"source_location":"CursorBench 4.0 table: Fable 5.1 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":149,"type":"result","entityId":"1406","action":"added","label":"Claude Haiku 5.5 · CursorBench","modelId":"claude-haiku-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.12, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 325934, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 163, \"tools\": true}","id":1406,"model_id":"claude-haiku-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":48.4,"source_location":"CursorBench 4.0 table: Haiku 5.5 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":150,"type":"result","entityId":"1407","action":"added","label":"Claude Sonnet 5.5 · CursorBench","modelId":"claude-sonnet-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.2, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 37391, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 41, \"tools\": true}","id":1407,"model_id":"claude-sonnet-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":47.8,"source_location":"CursorBench 4.0 table: Sonnet 5.5 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":151,"type":"result","entityId":"1408","action":"added","label":"DeepSeek V4.1 Flash 1017 · CursorBench","modelId":"deepseek-v4-1-flash-1017","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.98, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 537047, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 187, \"tools\": true}","id":1408,"model_id":"deepseek-v4-1-flash-1017","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":47.7,"source_location":"CursorBench 4.0 table: DeepSeek V4.1 Flash 1017 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":152,"type":"result","entityId":"1409","action":"added","label":"Claude Fable 5.1 · CursorBench","modelId":"claude-fable-5-1","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 7.05, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 45411, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 63, \"tools\": true}","id":1409,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":46.8,"source_location":"CursorBench 4.0 table: Fable 5.1 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":153,"type":"result","entityId":"1410","action":"added","label":"Claude Opus 5 · CursorBench","modelId":"claude-opus-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 11.95, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 85384, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 106, \"tools\": true}","id":1410,"model_id":"claude-opus-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":46.6,"source_location":"CursorBench 4.0 table: Opus 5 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":154,"type":"result","entityId":"1411","action":"added","label":"Grok 4.7 · CursorBench","modelId":"grok-4-7","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 6.01, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 70141, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 88, \"tools\": true}","id":1411,"model_id":"grok-4-7","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":46.3,"source_location":"CursorBench 4.0 table: Grok 4.7 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":155,"type":"result","entityId":"1412","action":"added","label":"DeepSeek V4.1 Flash 1017 · CursorBench","modelId":"deepseek-v4-1-flash-1017","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.8, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 398255, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 189, \"tools\": true}","id":1412,"model_id":"deepseek-v4-1-flash-1017","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":46.3,"source_location":"CursorBench 4.0 table: DeepSeek V4.1 Flash 1017 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":156,"type":"result","entityId":"1413","action":"added","label":"Claude Opus 5 · CursorBench","modelId":"claude-opus-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 11.43, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 80094, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 103, \"tools\": true}","id":1413,"model_id":"claude-opus-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":46.1,"source_location":"CursorBench 4.0 table: Opus 5 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":157,"type":"result","entityId":"1414","action":"added","label":"DeepSeek V4.1 Flash 1017 · CursorBench","modelId":"deepseek-v4-1-flash-1017","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.77, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 385196, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 186, \"tools\": true}","id":1414,"model_id":"deepseek-v4-1-flash-1017","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":45.6,"source_location":"CursorBench 4.0 table: DeepSeek V4.1 Flash 1017 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":158,"type":"result","entityId":"1415","action":"added","label":"Claude Fable 5.1 · CursorBench","modelId":"claude-fable-5-1","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 5.44, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 34795, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 51, \"tools\": true}","id":1415,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":45.1,"source_location":"CursorBench 4.0 table: Fable 5.1 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":159,"type":"result","entityId":"1416","action":"added","label":"DeepSeek V4.1 Flash 1017 · CursorBench","modelId":"deepseek-v4-1-flash-1017","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.6, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 290651, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 161, \"tools\": true}","id":1416,"model_id":"deepseek-v4-1-flash-1017","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":44.9,"source_location":"CursorBench 4.0 table: DeepSeek V4.1 Flash 1017 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":160,"type":"result","entityId":"1417","action":"added","label":"Claude Opus 5 · CursorBench","modelId":"claude-opus-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 9.0, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 61405, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 86, \"tools\": true}","id":1417,"model_id":"claude-opus-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":44.7,"source_location":"CursorBench 4.0 table: Opus 5 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":161,"type":"result","entityId":"1418","action":"added","label":"Claude Haiku 5.5 · CursorBench","modelId":"claude-haiku-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.56, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 143813, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 93, \"tools\": true}","id":1418,"model_id":"claude-haiku-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":44.3,"source_location":"CursorBench 4.0 table: Haiku 5.5 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":162,"type":"result","entityId":"1419","action":"added","label":"Grok 4.7 · CursorBench","modelId":"grok-4-7","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.69, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 56382, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 71, \"tools\": true}","id":1419,"model_id":"grok-4-7","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":43.9,"source_location":"CursorBench 4.0 table: Grok 4.7 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":163,"type":"result","entityId":"1420","action":"added","label":"Claude Opus 5.5 · CursorBench","modelId":"claude-opus-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.17, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 15811, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 28, \"tools\": true}","id":1420,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":43.7,"source_location":"CursorBench 4.0 table: Opus 5.5 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":164,"type":"result","entityId":"1421","action":"added","label":"Claude Opus 5 · CursorBench","modelId":"claude-opus-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 6.94, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 45272, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 72, \"tools\": true}","id":1421,"model_id":"claude-opus-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":43.3,"source_location":"CursorBench 4.0 table: Opus 5 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":165,"type":"result","entityId":"1422","action":"added","label":"GLM-5.3 · CursorBench","modelId":"glm-5-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.862Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 5.05, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 96387, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 166, \"tools\": true}","id":1422,"model_id":"glm-5-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":42.6,"source_location":"CursorBench 4.0 table: GLM 5.3 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":166,"type":"result","entityId":"1423","action":"added","label":"Claude Haiku 5.5 · CursorBench","modelId":"claude-haiku-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.32, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 77057, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 59, \"tools\": true}","id":1423,"model_id":"claude-haiku-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":42.3,"source_location":"CursorBench 4.0 table: Haiku 5.5 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":167,"type":"result","entityId":"1424","action":"added","label":"GPT-5.6 Sol · CursorBench","modelId":"gpt-5-6-sol","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 8.23, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 42944, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 99, \"tools\": true}","id":1424,"model_id":"gpt-5-6-sol","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":41.7,"source_location":"CursorBench 4.0 table: GPT-5.6 Sol Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":168,"type":"result","entityId":"1425","action":"added","label":"Grok 4.7 · CursorBench","modelId":"grok-4-7","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 3.49, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 36683, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 60, \"tools\": true}","id":1425,"model_id":"grok-4-7","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":41.6,"source_location":"CursorBench 4.0 table: Grok 4.7 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":169,"type":"result","entityId":"1426","action":"added","label":"Muse Spark 1.3 · CursorBench","modelId":"muse-spark-1-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 2.64, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 52005, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 98, \"tools\": true}","id":1426,"model_id":"muse-spark-1-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":41.6,"source_location":"CursorBench 4.0 table: Muse Spark 1.3 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":170,"type":"result","entityId":"1427","action":"added","label":"GPT-5.6 Terra · CursorBench","modelId":"gpt-5-6-terra","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 5.14, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 60814, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 107, \"tools\": true}","id":1427,"model_id":"gpt-5-6-terra","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":41.3,"source_location":"CursorBench 4.0 table: GPT-5.6 Terra Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":171,"type":"result","entityId":"1428","action":"added","label":"Claude Opus 5 · CursorBench","modelId":"claude-opus-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.87, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 31995, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 57, \"tools\": true}","id":1428,"model_id":"claude-opus-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":40.7,"source_location":"CursorBench 4.0 table: Opus 5 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":172,"type":"result","entityId":"1429","action":"added","label":"Gemini 3.8 Flash · CursorBench","modelId":"gemini-3-8-flash","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.7, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 162565, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 324, \"tools\": true}","id":1429,"model_id":"gemini-3-8-flash","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":39.6,"source_location":"CursorBench 4.0 table: Gemini 3.8 Flash High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":173,"type":"result","entityId":"1430","action":"added","label":"Claude Sonnet 5.5 · CursorBench","modelId":"claude-sonnet-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.52, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 16036, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 22, \"tools\": true}","id":1430,"model_id":"claude-sonnet-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":39.2,"source_location":"CursorBench 4.0 table: Sonnet 5.5 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":174,"type":"result","entityId":"1431","action":"added","label":"GLM-5.3 · CursorBench","modelId":"glm-5-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 3.24, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 60031, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 114, \"tools\": true}","id":1431,"model_id":"glm-5-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":38,"source_location":"CursorBench 4.0 table: GLM 5.3 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":175,"type":"result","entityId":"1432","action":"added","label":"GPT-5.6 Sol · CursorBench","modelId":"gpt-5-6-sol","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.4, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 24729, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 55, \"tools\": true}","id":1432,"model_id":"gpt-5-6-sol","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":37.7,"source_location":"CursorBench 4.0 table: GPT-5.6 Sol Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":176,"type":"result","entityId":"1433","action":"added","label":"Muse Spark 1.3 · CursorBench","modelId":"muse-spark-1-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 2.1, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 40891, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 83, \"tools\": true}","id":1433,"model_id":"muse-spark-1-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":37.5,"source_location":"CursorBench 4.0 table: Muse Spark 1.3 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":177,"type":"result","entityId":"1434","action":"added","label":"Gemini 3.8 Flash · CursorBench","modelId":"gemini-3-8-flash","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.06, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 128364, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 290, \"tools\": true}","id":1434,"model_id":"gemini-3-8-flash","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":37.3,"source_location":"CursorBench 4.0 table: Gemini 3.8 Flash Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":178,"type":"result","entityId":"1435","action":"added","label":"Claude Haiku 5.5 · CursorBench","modelId":"claude-haiku-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.17, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 42659, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 39, \"tools\": true}","id":1435,"model_id":"claude-haiku-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":36.9,"source_location":"CursorBench 4.0 table: Haiku 5.5 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":179,"type":"result","entityId":"1436","action":"added","label":"GLM-5.3-Flash · CursorBench","modelId":"glm-5-3-flash","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.39, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 56410, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 118, \"tools\": true}","id":1436,"model_id":"glm-5-3-flash","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":36.8,"source_location":"CursorBench 4.0 table: GLM 5.3 Flash Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":180,"type":"result","entityId":"1437","action":"added","label":"GPT-5.6 Luna · CursorBench","modelId":"gpt-5-6-luna","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.03, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 87284, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 208, \"tools\": true}","id":1437,"model_id":"gpt-5-6-luna","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":35.9,"source_location":"CursorBench 4.0 table: GPT-5.6 Luna Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":181,"type":"result","entityId":"1438","action":"added","label":"Claude Sonnet 5.5 · CursorBench","modelId":"claude-sonnet-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.37, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 11668, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 18, \"tools\": true}","id":1438,"model_id":"claude-sonnet-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":35.8,"source_location":"CursorBench 4.0 table: Sonnet 5.5 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":182,"type":"result","entityId":"1439","action":"added","label":"GPT-5.6 Sol · CursorBench","modelId":"gpt-5-6-sol","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 2.85, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 16174, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 41, \"tools\": true}","id":1439,"model_id":"gpt-5-6-sol","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":35.7,"source_location":"CursorBench 4.0 table: GPT-5.6 Sol High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":183,"type":"result","entityId":"1440","action":"added","label":"Claude Sonnet 5 · CursorBench","modelId":"claude-sonnet-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 7.17, \"effort\": \"max\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 149257, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 140, \"tools\": true}","id":1440,"model_id":"claude-sonnet-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":34.1,"source_location":"CursorBench 4.0 table: Sonnet 5 Max","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":184,"type":"result","entityId":"1441","action":"added","label":"GPT-5.6 Terra · CursorBench","modelId":"gpt-5-6-terra","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.81, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 23436, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 43, \"tools\": true}","id":1441,"model_id":"gpt-5-6-terra","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":33.6,"source_location":"CursorBench 4.0 table: GPT-5.6 Terra Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":185,"type":"result","entityId":"1442","action":"added","label":"Muse Spark 1.3 · CursorBench","modelId":"muse-spark-1-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.66, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 30654, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 69, \"tools\": true}","id":1442,"model_id":"muse-spark-1-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":33.4,"source_location":"CursorBench 4.0 table: Muse Spark 1.3 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":186,"type":"result","entityId":"1443","action":"added","label":"GLM-5.3 · CursorBench","modelId":"glm-5-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 2.04, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 31983, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 81, \"tools\": true}","id":1443,"model_id":"glm-5-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":33.3,"source_location":"CursorBench 4.0 table: GLM 5.3 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":187,"type":"result","entityId":"1444","action":"added","label":"Grok 4.7 · CursorBench","modelId":"grok-4-7","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.58, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 15677, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 40, \"tools\": true}","id":1444,"model_id":"grok-4-7","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":33.1,"source_location":"CursorBench 4.0 table: Grok 4.7 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":188,"type":"result","entityId":"1445","action":"added","label":"GPT-5.6 Luna · CursorBench","modelId":"gpt-5-6-luna","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.44, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 40598, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 98, \"tools\": true}","id":1445,"model_id":"gpt-5-6-luna","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":33,"source_location":"CursorBench 4.0 table: GPT-5.6 Luna Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":189,"type":"result","entityId":"1446","action":"added","label":"Muse Spark 1.3 · CursorBench","modelId":"muse-spark-1-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.49, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 27255, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 64, \"tools\": true}","id":1446,"model_id":"muse-spark-1-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":32.6,"source_location":"CursorBench 4.0 table: Muse Spark 1.3 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":190,"type":"result","entityId":"1447","action":"added","label":"Claude Sonnet 5 · CursorBench","modelId":"claude-sonnet-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 4.55, \"effort\": \"xhigh\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 83373, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 102, \"tools\": true}","id":1447,"model_id":"claude-sonnet-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":32,"source_location":"CursorBench 4.0 table: Sonnet 5 Extra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":191,"type":"result","entityId":"1448","action":"added","label":"GPT-5.6 Sol · CursorBench","modelId":"gpt-5-6-sol","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.77, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 10111, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 32, \"tools\": true}","id":1448,"model_id":"gpt-5-6-sol","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":31.1,"source_location":"CursorBench 4.0 table: GPT-5.6 Sol Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":192,"type":"result","entityId":"1449","action":"added","label":"GLM-5.3-Flash · CursorBench","modelId":"glm-5-3-flash","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.25, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 35104, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 84, \"tools\": true}","id":1449,"model_id":"glm-5-3-flash","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":31.1,"source_location":"CursorBench 4.0 table: GLM 5.3 Flash High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":193,"type":"result","entityId":"1450","action":"added","label":"Claude Haiku 5.5 · CursorBench","modelId":"claude-haiku-5-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.08, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 23382, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 25, \"tools\": true}","id":1450,"model_id":"claude-haiku-5-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":30.9,"source_location":"CursorBench 4.0 table: Haiku 5.5 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":194,"type":"result","entityId":"1451","action":"added","label":"Claude Sonnet 5 · CursorBench","modelId":"claude-sonnet-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 3.48, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 61146, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 85, \"tools\": true}","id":1451,"model_id":"claude-sonnet-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":30.8,"source_location":"CursorBench 4.0 table: Sonnet 5 High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":195,"type":"result","entityId":"1452","action":"added","label":"GPT-5.6 Terra · CursorBench","modelId":"gpt-5-6-terra","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.11, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 13162, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 33, \"tools\": true}","id":1452,"model_id":"gpt-5-6-terra","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":30.7,"source_location":"CursorBench 4.0 table: GPT-5.6 Terra High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":196,"type":"result","entityId":"1453","action":"added","label":"GPT-5.6 Luna · CursorBench","modelId":"gpt-5-6-luna","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.25, \"effort\": \"high\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 23368, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 64, \"tools\": true}","id":1453,"model_id":"gpt-5-6-luna","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":29.4,"source_location":"CursorBench 4.0 table: GPT-5.6 Luna High","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":197,"type":"result","entityId":"1454","action":"added","label":"Muse Spark 1.3 · CursorBench","modelId":"muse-spark-1-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.93, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 17483, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 47, \"tools\": true}","id":1454,"model_id":"muse-spark-1-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":29.3,"source_location":"CursorBench 4.0 table: Muse Spark 1.3 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":198,"type":"result","entityId":"1455","action":"added","label":"Claude Sonnet 5 · CursorBench","modelId":"claude-sonnet-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 2.31, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 39114, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 65, \"tools\": true}","id":1455,"model_id":"claude-sonnet-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":28,"source_location":"CursorBench 4.0 table: Sonnet 5 Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":199,"type":"result","entityId":"1456","action":"added","label":"GPT-5.6 Terra · CursorBench","modelId":"gpt-5-6-terra","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.64, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 7307, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 25, \"tools\": true}","id":1456,"model_id":"gpt-5-6-terra","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":27.6,"source_location":"CursorBench 4.0 table: GPT-5.6 Terra Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":200,"type":"result","entityId":"1457","action":"added","label":"GLM-5.3-Flash · CursorBench","modelId":"glm-5-3-flash","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.15, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 17831, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 58, \"tools\": true}","id":1457,"model_id":"glm-5-3-flash","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":26.9,"source_location":"CursorBench 4.0 table: GLM 5.3 Flash Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":201,"type":"result","entityId":"1458","action":"added","label":"GPT-5.6 Terra · CursorBench","modelId":"gpt-5-6-terra","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.52, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 5914, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 23, \"tools\": true}","id":1458,"model_id":"gpt-5-6-terra","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":25.2,"source_location":"CursorBench 4.0 table: GPT-5.6 Terra Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":202,"type":"result","entityId":"1459","action":"added","label":"GPT-5.6 Sol · CursorBench","modelId":"gpt-5-6-sol","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.87, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 4885, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 21, \"tools\": true}","id":1459,"model_id":"gpt-5-6-sol","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":24.6,"source_location":"CursorBench 4.0 table: GPT-5.6 Sol Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":203,"type":"result","entityId":"1460","action":"added","label":"Muse Spark 1.3 · CursorBench","modelId":"muse-spark-1-3","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.56, \"effort\": \"minimal\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 10620, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 34, \"tools\": true}","id":1460,"model_id":"muse-spark-1-3","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":24.3,"source_location":"CursorBench 4.0 table: Muse Spark 1.3 Minimal","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":204,"type":"result","entityId":"1461","action":"added","label":"Claude Sonnet 5 · CursorBench","modelId":"claude-sonnet-5","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 1.39, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 23772, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 46, \"tools\": true}","id":1461,"model_id":"claude-sonnet-5","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":24.1,"source_location":"CursorBench 4.0 table: Sonnet 5 Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":205,"type":"result","entityId":"1462","action":"added","label":"GPT-5.6 Luna · CursorBench","modelId":"gpt-5-6-luna","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.08, \"effort\": \"medium\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 7642, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 32, \"tools\": true}","id":1462,"model_id":"gpt-5-6-luna","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":22.2,"source_location":"CursorBench 4.0 table: GPT-5.6 Luna Medium","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":206,"type":"result","entityId":"1463","action":"added","label":"GPT-5.6 Luna · CursorBench","modelId":"gpt-5-6-luna","benchmarkSlug":"cursorbench","source":"https://cursor.com/cursorbench","previous":null,"current":{"benchmark_id":"cursorbench-4-0","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"cursorbench-4-0\", \"cost_usd_per_task\": 0.03, \"effort\": \"low\", \"harness\": \"Cursor production agent\", \"output_tokens_per_task\": 3288, \"settings\": \"Publisher table observed November 9; row publication date unspecified; current harness may differ from earlier system cards; small score differences may not be significant.\", \"steps_per_task\": 18, \"tools\": true}","id":1463,"model_id":"gpt-5-6-luna","notes":"{\"attribution\": \"Cursor\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":null,"score":16,"source_location":"CursorBench 4.0 table: GPT-5.6 Luna Low","source_publisher":"Cursor","source_title":"CursorBench 4.0 publisher leaderboard","source_type":"benchmark_publisher","source_url":"https://cursor.com/cursorbench","status":"published","supersedes_result_id":null}},{"id":207,"type":"result","entityId":"1464","action":"added","label":"Claude Fable 5.2 · Real-World Finance","modelId":"claude-fable-5-2","benchmarkSlug":"real-world-finance","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"real-world-finance-v2","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"real-world-finance-v2\", \"effort\": \"max\", \"settings\": \"123 tasks; one attempt; three-hour deadline; Opus 5.5 grader; rescaled Elo. Fable missed six tasks; Opus completed all 123. Elo is not completion percentage.\", \"tools\": true}","id":1464,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":1637,"source_location":"Section 8.14.6, PDF pages 241-242","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":208,"type":"result","entityId":"1465","action":"added","label":"Claude Opus 5.5 · Real-World Finance","modelId":"claude-opus-5-5","benchmarkSlug":"real-world-finance","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"real-world-finance-v2","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"real-world-finance-v2\", \"effort\": \"max\", \"settings\": \"123 tasks; one attempt; three-hour deadline; Opus 5.5 grader; rescaled Elo. Fable missed six tasks; Opus completed all 123. Elo is not completion percentage.\", \"tools\": true}","id":1465,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":1571,"source_location":"Section 8.14.6, PDF pages 241-242","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":209,"type":"result","entityId":"1466","action":"added","label":"Claude Fable 5.1 · Real-World Finance","modelId":"claude-fable-5-1","benchmarkSlug":"real-world-finance","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"real-world-finance-v2","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"real-world-finance-v2\", \"effort\": \"max\", \"settings\": \"123 tasks; one attempt; three-hour deadline; Opus 5.5 grader; rescaled Elo. Fable missed six tasks; Opus completed all 123. Elo is not completion percentage.\", \"tools\": true}","id":1466,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":1493,"source_location":"Section 8.14.6, PDF pages 241-242","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":210,"type":"result","entityId":"1467","action":"added","label":"Claude Fable 5.2 · BenchCAD","modelId":"claude-fable-5-2","benchmarkSlug":"benchcad","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"benchcad-2-1-standard","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"benchcad-2-1-standard\", \"effort\": \"max\", \"settings\": \"100 Core cases; five runs; mean partial credit; official scorer. Extra tool-assisted setup has no token/round cap; standard setup has 60 rounds and 500K output tokens. No numeric confidence interval inferred from chart.\", \"tools\": false}","id":1467,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":75.4,"source_location":"Section 8.13.2, PDF pages 232-233, Figure 8.13.2.A","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":211,"type":"result","entityId":"1468","action":"added","label":"Claude Fable 5.2 · BenchCAD","modelId":"claude-fable-5-2","benchmarkSlug":"benchcad","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"benchcad-2-1-tools","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"benchcad-2-1-tools\", \"effort\": \"max\", \"settings\": \"100 Core cases; five runs; mean partial credit; official scorer. Extra tool-assisted setup has no token/round cap; standard setup has 60 rounds and 500K output tokens. No numeric confidence interval inferred from chart.\", \"tools\": true}","id":1468,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":83.5,"source_location":"Section 8.13.2, PDF pages 232-233, Figure 8.13.2.A","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":212,"type":"result","entityId":"1469","action":"added","label":"Claude Opus 5.5 · BenchCAD","modelId":"claude-opus-5-5","benchmarkSlug":"benchcad","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"benchcad-2-1-standard","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"benchcad-2-1-standard\", \"effort\": \"max\", \"settings\": \"100 Core cases; five runs; mean partial credit; official scorer. Extra tool-assisted setup has no token/round cap; standard setup has 60 rounds and 500K output tokens. No numeric confidence interval inferred from chart.\", \"tools\": false}","id":1469,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":49,"source_location":"Section 8.13.2, PDF pages 232-233, Figure 8.13.2.A","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":213,"type":"result","entityId":"1470","action":"added","label":"Claude Opus 5.5 · BenchCAD","modelId":"claude-opus-5-5","benchmarkSlug":"benchcad","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"benchcad-2-1-tools","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"benchcad-2-1-tools\", \"effort\": \"max\", \"settings\": \"100 Core cases; five runs; mean partial credit; official scorer. Extra tool-assisted setup has no token/round cap; standard setup has 60 rounds and 500K output tokens. No numeric confidence interval inferred from chart.\", \"tools\": true}","id":1470,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":65.3,"source_location":"Section 8.13.2, PDF pages 232-233, Figure 8.13.2.A","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":214,"type":"result","entityId":"1471","action":"added","label":"Claude Fable 5.1 · BenchCAD","modelId":"claude-fable-5-1","benchmarkSlug":"benchcad","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"benchcad-2-1-standard","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"benchcad-2-1-standard\", \"effort\": \"max\", \"settings\": \"100 Core cases; five runs; mean partial credit; official scorer. Extra tool-assisted setup has no token/round cap; standard setup has 60 rounds and 500K output tokens. No numeric confidence interval inferred from chart.\", \"tools\": false}","id":1471,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":36.3,"source_location":"Section 8.13.2, PDF pages 232-233, Figure 8.13.2.A","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":215,"type":"result","entityId":"1472","action":"added","label":"Claude Fable 5.1 · BenchCAD","modelId":"claude-fable-5-1","benchmarkSlug":"benchcad","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"benchcad-2-1-tools","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"benchcad-2-1-tools\", \"effort\": \"max\", \"settings\": \"100 Core cases; five runs; mean partial credit; official scorer. Extra tool-assisted setup has no token/round cap; standard setup has 60 rounds and 500K output tokens. No numeric confidence interval inferred from chart.\", \"tools\": true}","id":1472,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":44.4,"source_location":"Section 8.13.2, PDF pages 232-233, Figure 8.13.2.A","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"provider_reported","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":216,"type":"result","entityId":"1473","action":"added","label":"Claude Fable 5.2 · Harvey LAB-AA","modelId":"claude-fable-5-2","benchmarkSlug":"harvey-lab-aa","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"max\", \"harness\": \"Stirrup\", \"settings\": \"120 private tasks; v1.1 three-judge hallucination-gated all-pass rate. Distinct from ungated 10.8% Fable all-pass and earlier v1.0.\", \"tools\": true}","id":1473,"model_id":"claude-fable-5-2","notes":"{\"attribution\": \"Independent results quoted by Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":7.2,"source_location":"Section 8.14.2, PDF page 238","source_publisher":"Artificial Analysis via Anthropic system card","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"independent","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":217,"type":"result","entityId":"1474","action":"added","label":"Claude Fable 5.1 · Harvey LAB-AA","modelId":"claude-fable-5-1","benchmarkSlug":"harvey-lab-aa","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"max\", \"harness\": \"Stirrup\", \"settings\": \"120 private tasks; v1.1 three-judge hallucination-gated all-pass rate. Distinct from ungated 10.8% Fable all-pass and earlier v1.0.\", \"tools\": true}","id":1474,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Independent results quoted by Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":6.4,"source_location":"Section 8.14.2, PDF page 238","source_publisher":"Artificial Analysis via Anthropic system card","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"independent","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":218,"type":"result","entityId":"1475","action":"added","label":"Claude Opus 5.5 · Harvey LAB-AA","modelId":"claude-opus-5-5","benchmarkSlug":"harvey-lab-aa","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"max\", \"harness\": \"Stirrup\", \"settings\": \"120 private tasks; v1.1 three-judge hallucination-gated all-pass rate. Distinct from ungated 10.8% Fable all-pass and earlier v1.0.\", \"tools\": true}","id":1475,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Independent results quoted by Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":4.2,"source_location":"Section 8.14.2, PDF page 238","source_publisher":"Artificial Analysis via Anthropic system card","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"independent","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":219,"type":"result","entityId":"1476","action":"added","label":"GPT-6 Astra · Harvey LAB-AA","modelId":"gpt-6-astra","benchmarkSlug":"harvey-lab-aa","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"max\", \"harness\": \"Stirrup\", \"settings\": \"120 private tasks; v1.1 three-judge hallucination-gated all-pass rate. Distinct from ungated 10.8% Fable all-pass and earlier v1.0.\", \"tools\": true}","id":1476,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Independent results quoted by Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":8.6,"source_location":"Section 8.14.2, PDF page 238","source_publisher":"Artificial Analysis via Anthropic system card","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"independent","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":220,"type":"result","entityId":"1477","action":"added","label":"GPT-6.1 Sol · Harvey LAB-AA","modelId":"gpt-6-1-sol","benchmarkSlug":"harvey-lab-aa","source":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"max\", \"harness\": \"Stirrup\", \"settings\": \"120 private tasks; v1.1 three-judge hallucination-gated all-pass rate. Distinct from ungated 10.8% Fable all-pass and earlier v1.0.\", \"tools\": true}","id":1477,"model_id":"gpt-6-1-sol","notes":"{\"attribution\": \"Independent results quoted by Anthropic\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-11-09","score":6.9,"source_location":"Section 8.14.2, PDF page 238","source_publisher":"Artificial Analysis via Anthropic system card","source_title":"Claude Fable 5.2 and Mythos 5.2 System Card","source_type":"independent","source_url":"https://assets.claude.com/27d60918d218125e7504bc7354c4b4626ec56b11.pdf","status":"published","supersedes_result_id":null}},{"id":221,"type":"result","entityId":"1478","action":"added","label":"Grok 4.7 · Harvey LAB-AA","modelId":"grok-4-7","benchmarkSlug":"harvey-lab-aa","source":"https://artificialanalysis.ai/articles/harvey-lab-aa-v1-1","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"xhigh\", \"harness\": \"Stirrup\", \"settings\": \"October 8 published v1.1 snapshot; hallucination-gated all-pass rate; not an inferred latest leaderboard value.\", \"tools\": true}","id":1478,"model_id":"grok-4-7","notes":"{\"attribution\": \"Artificial Analysis\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-10-08","score":9.4,"source_location":"Score section: October 8 snapshot","source_publisher":"Artificial Analysis","source_title":"Harvey LAB-AA v1.1 launch results","source_type":"independent","source_url":"https://artificialanalysis.ai/articles/harvey-lab-aa-v1-1","status":"published","supersedes_result_id":null}},{"id":222,"type":"result","entityId":"1479","action":"added","label":"Muse Spark 1.3 · Harvey LAB-AA","modelId":"muse-spark-1-3","benchmarkSlug":"harvey-lab-aa","source":"https://artificialanalysis.ai/articles/harvey-lab-aa-v1-1","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"max\", \"harness\": \"Stirrup\", \"settings\": \"October 8 published v1.1 snapshot; hallucination-gated all-pass rate; not an inferred latest leaderboard value.\", \"tools\": true}","id":1479,"model_id":"muse-spark-1-3","notes":"{\"attribution\": \"Artificial Analysis\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-10-08","score":8.9,"source_location":"Score section: October 8 snapshot","source_publisher":"Artificial Analysis","source_title":"Harvey LAB-AA v1.1 launch results","source_type":"independent","source_url":"https://artificialanalysis.ai/articles/harvey-lab-aa-v1-1","status":"published","supersedes_result_id":null}},{"id":223,"type":"result","entityId":"1480","action":"added","label":"Kimi K3 · Harvey LAB-AA","modelId":"kimi-k3","benchmarkSlug":"harvey-lab-aa","source":"https://artificialanalysis.ai/articles/harvey-lab-aa-v1-1","previous":null,"current":{"benchmark_id":"harvey-lab-aa-v1-1","created_at":"2026-11-10T01:17:35.863Z","evaluation_settings":"{\"comparison_group\": \"harvey-lab-aa-v1-1\", \"effort\": \"max\", \"harness\": \"Stirrup\", \"settings\": \"October 8 published v1.1 snapshot; hallucination-gated all-pass rate; not an inferred latest leaderboard value.\", \"tools\": true}","id":1480,"model_id":"kimi-k3","notes":"{\"attribution\": \"Artificial Analysis\", \"checked_at\": \"2026-11-09\", \"source_snapshot\": \"database/benchmarks/additions-2026-11-09\"}","reported_at":"2026-10-08","score":5.3,"source_location":"Score section: October 8 snapshot","source_publisher":"Artificial Analysis","source_title":"Harvey LAB-AA v1.1 launch results","source_type":"independent","source_url":"https://artificialanalysis.ai/articles/harvey-lab-aa-v1-1","status":"published","supersedes_result_id":null}}]},{"id":3,"recordedAt":"2026-11-10T01:04:45.221Z","kind":"update","summary":"43 dataset changes recorded","changeCount":43,"changes":[{"id":2,"type":"model","entityId":"claude-mythos-5-1","action":"added","label":"Claude Mythos 5.1","modelId":"claude-mythos-5-1","benchmarkSlug":null,"source":null,"previous":null,"current":{"classification_notes":"Named explicitly in the official November 9 comparison. Separate identity; no scores or specifications inherited from Mythos 5.","classification_source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","context_window_tokens":null,"created_at":"2026-11-10T01:04:32.953Z","id":"claude-mythos-5-1","input_price_usd_per_million":null,"is_open_source":0,"knowledge_cutoff":null,"latency_metric":"time_to_first_token","latency_ms":null,"latency_source_url":null,"license_name":null,"max_output_checked_at":null,"max_output_notes":null,"max_output_source_url":null,"max_output_status":"not_reviewed","max_output_tokens":null,"name":"Claude Mythos 5.1","notes":"Specialist model configuration named in Anthropic launch table. Release date, pricing and endpoint measurements are not established by this source.","output_price_usd_per_million":null,"performance_as_of":null,"performance_notes":null,"provider_id":"anthropic","release_date":null,"release_date_notes":null,"release_date_source_url":null,"specifications_as_of":null,"specifications_source_url":null,"speed_output_tokens_per_second":null,"speed_source_url":null,"version_identifier":"claude-mythos-5-1","weights_source_url":null}},{"id":3,"type":"benchmark","entityId":"cyscenariobench-openai-cyber-launch","action":"added","label":"CyScenarioBench","modelId":null,"benchmarkSlug":"cyscenariobench","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","category":"Coding agents","created_at":"2026-11-10T01:04:32.954Z","description":"GPT-6 Cyber launch-table score. Each lab uses unique tasks; not directly comparable to the Anthropic 10-challenge subset.","display_name":"CyScenarioBench","higher_is_better":1,"id":"cyscenariobench-openai-cyber-launch","is_primary_track":0,"is_saturated":null,"is_widely_used":null,"maximum_score":100,"metric":"Overall solve rate","minimum_score":0,"name":"CyScenarioBench","official_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","page_slug":"cyscenariobench","summary":"GPT-6 Cyber score on its own task set, kept separate from Anthropic tasks.","tracking_status":"tracked","unit":"%","version":"OpenAI GPT-6 Cyber / unique task set"}},{"id":4,"type":"result","entityId":"1349","action":"added","label":"Claude Fable 5.1 · FrontierCode","modelId":"claude-fable-5-1","benchmarkSlug":"frontiercode","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontiercode-2-0","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontiercode\", \"effort\": \"xhigh\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1349,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":32.4,"source_location":"Benchmark performance table: FrontierCode 2.0","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":5,"type":"result","entityId":"1350","action":"added","label":"Claude Opus 5.5 · FrontierCode","modelId":"claude-opus-5-5","benchmarkSlug":"frontiercode","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontiercode-2-0","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontiercode\", \"effort\": \"max\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1350,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":45.5,"source_location":"Benchmark performance table: FrontierCode 2.0","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":6,"type":"result","entityId":"1351","action":"added","label":"GPT-6 Astra · FrontierCode","modelId":"gpt-6-astra","benchmarkSlug":"frontiercode","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontiercode-2-0","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontiercode\", \"effort\": \"max\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1351,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":42.8,"source_location":"Benchmark performance table: FrontierCode 2.0","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":7,"type":"result","entityId":"1352","action":"added","label":"Gemini 4 Carbon · FrontierCode","modelId":"gemini-4-carbon","benchmarkSlug":"frontiercode","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontiercode-2-0","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontiercode\", \"effort\": \"medium\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1352,"model_id":"gemini-4-carbon","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":41.1,"source_location":"Benchmark performance table: FrontierCode 2.0","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":8,"type":"result","entityId":"1353","action":"added","label":"Claude Fable 5.1 · FrontierSWE","modelId":"claude-fable-5-1","benchmarkSlug":"frontierswe","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontierswe-v2","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontierswe\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1353,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":56.3,"source_location":"Benchmark performance table: FrontierSWE v2","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":9,"type":"result","entityId":"1354","action":"added","label":"Claude Opus 5.5 · FrontierSWE","modelId":"claude-opus-5-5","benchmarkSlug":"frontierswe","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontierswe-v2","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontierswe\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1354,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":62.3,"source_location":"Benchmark performance table: FrontierSWE v2","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":10,"type":"result","entityId":"1355","action":"added","label":"GPT-6 Astra · FrontierSWE","modelId":"gpt-6-astra","benchmarkSlug":"frontierswe","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontierswe-v2","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontierswe\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1355,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":65.5,"source_location":"Benchmark performance table: FrontierSWE v2","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":11,"type":"result","entityId":"1356","action":"added","label":"Gemini 4 Carbon · FrontierSWE","modelId":"gemini-4-carbon","benchmarkSlug":"frontierswe","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"frontierswe-v2","created_at":"2026-11-10T01:04:32.954Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-frontierswe\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1356,"model_id":"gemini-4-carbon","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":61.2,"source_location":"Benchmark performance table: FrontierSWE v2","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":12,"type":"result","entityId":"1357","action":"added","label":"Claude Fable 5.1 · Terminal-Bench-Science","modelId":"claude-fable-5-1","benchmarkSlug":"terminal-bench-science","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"terminal-bench-science-0-1-native","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-science\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1357,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":52.6,"source_location":"Benchmark performance table: Terminal-Bench-Science 0.1","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":13,"type":"result","entityId":"1358","action":"added","label":"Claude Opus 5.5 · Terminal-Bench-Science","modelId":"claude-opus-5-5","benchmarkSlug":"terminal-bench-science","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"terminal-bench-science-0-1-native","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-science\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1358,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":58.7,"source_location":"Benchmark performance table: Terminal-Bench-Science 0.1","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":14,"type":"result","entityId":"1359","action":"added","label":"GPT-6 Astra · Terminal-Bench-Science","modelId":"gpt-6-astra","benchmarkSlug":"terminal-bench-science","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"terminal-bench-science-0-1-native","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-science\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1359,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":68.1,"source_location":"Benchmark performance table: Terminal-Bench-Science 0.1","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":15,"type":"result","entityId":"1360","action":"added","label":"Gemini 4 Carbon · Terminal-Bench-Science","modelId":"gemini-4-carbon","benchmarkSlug":"terminal-bench-science","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"terminal-bench-science-0-1-native","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-science\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1360,"model_id":"gemini-4-carbon","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":64.8,"source_location":"Benchmark performance table: Terminal-Bench-Science 0.1","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":16,"type":"result","entityId":"1361","action":"added","label":"Claude Fable 5.1 · WANDR","modelId":"claude-fable-5-1","benchmarkSlug":"wandr","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"wandr-anthropic-relaxed","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-wandr\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1361,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":68.7,"source_location":"Benchmark performance table: WANDR (relaxed score)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":17,"type":"result","entityId":"1362","action":"added","label":"Claude Opus 5.5 · WANDR","modelId":"claude-opus-5-5","benchmarkSlug":"wandr","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"wandr-anthropic-relaxed","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-wandr\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1362,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":72.3,"source_location":"Benchmark performance table: WANDR (relaxed score)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":18,"type":"result","entityId":"1363","action":"added","label":"Claude Fable 5.1 · OSWorld","modelId":"claude-fable-5-1","benchmarkSlug":"osworld","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"osworld-2-1-offline-partial-anthropic","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-osworld\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1363,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":83.5,"source_location":"Benchmark performance table: OSWorld 2.1 (offline set, partial)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":19,"type":"result","entityId":"1364","action":"added","label":"Claude Opus 5.5 · OSWorld","modelId":"claude-opus-5-5","benchmarkSlug":"osworld","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"osworld-2-1-offline-partial-anthropic","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-osworld\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1364,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":87.2,"source_location":"Benchmark performance table: OSWorld 2.1 (offline set, partial)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":20,"type":"result","entityId":"1365","action":"added","label":"GPT-6 Astra · OSWorld","modelId":"gpt-6-astra","benchmarkSlug":"osworld","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"osworld-2-1-offline-partial-anthropic","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-osworld\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1365,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":79.5,"source_location":"Benchmark performance table: OSWorld 2.1 (offline set, partial)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":21,"type":"result","entityId":"1366","action":"added","label":"Gemini 4 Carbon · OSWorld","modelId":"gemini-4-carbon","benchmarkSlug":"osworld","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"osworld-2-1-offline-partial-anthropic","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-osworld\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1366,"model_id":"gemini-4-carbon","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":81.5,"source_location":"Benchmark performance table: OSWorld 2.1 (offline set, partial)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":22,"type":"result","entityId":"1367","action":"added","label":"Claude Fable 5.1 · RoboDojo Sim","modelId":"claude-fable-5-1","benchmarkSlug":"robodojo-sim","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"robodojo-sim","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-robodojo\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\"}","id":1367,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":27.6,"source_location":"Benchmark performance table: RoboDojo Sim","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":23,"type":"result","entityId":"1368","action":"added","label":"Claude Opus 5.5 · RoboDojo Sim","modelId":"claude-opus-5-5","benchmarkSlug":"robodojo-sim","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"robodojo-sim","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-robodojo\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\"}","id":1368,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":39.4,"source_location":"Benchmark performance table: RoboDojo Sim","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":24,"type":"result","entityId":"1369","action":"added","label":"GPT-6 Astra · RoboDojo Sim","modelId":"gpt-6-astra","benchmarkSlug":"robodojo-sim","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"robodojo-sim","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-robodojo\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\"}","id":1369,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":29.8,"source_location":"Benchmark performance table: RoboDojo Sim","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":25,"type":"result","entityId":"1370","action":"added","label":"Claude Fable 5.1 · AA-Briefcase","modelId":"claude-fable-5-1","benchmarkSlug":"aa-briefcase","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"aa-briefcase-v1-1-rubric","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"aa-briefcase-1.1-stirrup-rubric\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1370,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":61.5,"source_location":"Benchmark performance table: AA-Briefcase v1.1 / rubric","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":26,"type":"result","entityId":"1371","action":"added","label":"Claude Opus 5.5 · AA-Briefcase","modelId":"claude-opus-5-5","benchmarkSlug":"aa-briefcase","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"aa-briefcase-v1-1-rubric","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"aa-briefcase-1.1-stirrup-rubric\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1371,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":61.2,"source_location":"Benchmark performance table: AA-Briefcase v1.1 / rubric","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":27,"type":"result","entityId":"1372","action":"added","label":"GPT-6 Astra · AA-Briefcase","modelId":"gpt-6-astra","benchmarkSlug":"aa-briefcase","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"aa-briefcase-v1-1-rubric","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"aa-briefcase-1.1-stirrup-rubric\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1372,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":52,"source_location":"Benchmark performance table: AA-Briefcase v1.1 / rubric","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":28,"type":"result","entityId":"1373","action":"added","label":"Gemini 4 Carbon · AA-Briefcase","modelId":"gemini-4-carbon","benchmarkSlug":"aa-briefcase","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"aa-briefcase-v1-1-rubric","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"aa-briefcase-1.1-stirrup-rubric\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1373,"model_id":"gemini-4-carbon","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":61.8,"source_location":"Benchmark performance table: AA-Briefcase v1.1 / rubric","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":29,"type":"result","entityId":"1374","action":"added","label":"Claude Fable 5.1 · Remote Labor Index","modelId":"claude-fable-5-1","benchmarkSlug":"remote-labor-index","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"remote-labor-index","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-rli\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1374,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":17.9,"source_location":"Benchmark performance table: Remote Labor Index (RLI)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":30,"type":"result","entityId":"1375","action":"added","label":"Claude Opus 5.5 · Remote Labor Index","modelId":"claude-opus-5-5","benchmarkSlug":"remote-labor-index","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"remote-labor-index","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-rli\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1375,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":21.3,"source_location":"Benchmark performance table: Remote Labor Index (RLI)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":31,"type":"result","entityId":"1376","action":"added","label":"GPT-6 Astra · Remote Labor Index","modelId":"gpt-6-astra","benchmarkSlug":"remote-labor-index","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"remote-labor-index","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-rli\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1376,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":20.8,"source_location":"Benchmark performance table: Remote Labor Index (RLI)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":32,"type":"result","entityId":"1377","action":"added","label":"Claude Fable 5.1 · Chartography","modelId":"claude-fable-5-1","benchmarkSlug":"chartography","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"chartography","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-chartography\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": false}","id":1377,"model_id":"claude-fable-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":44.8,"source_location":"Benchmark performance table: Chartography (no tools)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":33,"type":"result","entityId":"1378","action":"added","label":"Claude Opus 5.5 · Chartography","modelId":"claude-opus-5-5","benchmarkSlug":"chartography","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"chartography","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-chartography\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": false}","id":1378,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":64.4,"source_location":"Benchmark performance table: Chartography (no tools)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":34,"type":"result","entityId":"1379","action":"added","label":"GPT-6 Astra · Chartography","modelId":"gpt-6-astra","benchmarkSlug":"chartography","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"chartography","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-chartography\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": false}","id":1379,"model_id":"gpt-6-astra","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":71,"source_location":"Benchmark performance table: Chartography (no tools)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":35,"type":"result","entityId":"1380","action":"added","label":"Gemini 4 Carbon · Chartography","modelId":"gemini-4-carbon","benchmarkSlug":"chartography","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"chartography","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-chartography\", \"effort\": \"highest available\", \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": false}","id":1380,"model_id":"gemini-4-carbon","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":72.5,"source_location":"Benchmark performance table: Chartography (no tools)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":36,"type":"result","entityId":"1381","action":"added","label":"Claude Mythos 5.2 · Terminal-Bench-Science","modelId":"claude-mythos-5-2","benchmarkSlug":"terminal-bench-science","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"terminal-bench-science-0-1-native","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-science\", \"effort\": \"max\", \"safeguards\": false, \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1381,"model_id":"claude-mythos-5-2","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":75.3,"source_location":"Benchmark performance table: Terminal-Bench-Science 0.1 / Mythos 5.2","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":37,"type":"result","entityId":"1382","action":"added","label":"Claude Mythos 5.1 · CyScenarioBench","modelId":"claude-mythos-5-1","benchmarkSlug":"cyscenariobench","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"cyscenariobench-anthropic","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-cyscenario\", \"effort\": \"highest available\", \"safeguards\": false, \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1382,"model_id":"claude-mythos-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":61.7,"source_location":"Benchmark performance table: CyScenarioBench / Anthropic 10-challenge subset","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":38,"type":"result","entityId":"1383","action":"added","label":"Claude Opus 5.5 · CyScenarioBench","modelId":"claude-opus-5-5","benchmarkSlug":"cyscenariobench","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"cyscenariobench-anthropic","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-cyscenario\", \"effort\": \"highest available\", \"safeguards\": false, \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1383,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":67.6,"source_location":"Benchmark performance table: CyScenarioBench / Anthropic 10-challenge subset","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":39,"type":"result","entityId":"1384","action":"added","label":"Claude Mythos 5.1 · CVE Rediscovery","modelId":"claude-mythos-5-1","benchmarkSlug":"cve-rediscovery","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"cve-rediscovery-2026-10","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-cve\", \"effort\": \"highest available\", \"safeguards\": false, \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1384,"model_id":"claude-mythos-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":78.3,"source_location":"Benchmark performance table: CVE Rediscovery","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":40,"type":"result","entityId":"1385","action":"added","label":"Claude Opus 5.5 · CVE Rediscovery","modelId":"claude-opus-5-5","benchmarkSlug":"cve-rediscovery","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"cve-rediscovery-2026-10","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-cve\", \"effort\": \"highest available\", \"safeguards\": false, \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1385,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":81.3,"source_location":"Benchmark performance table: CVE Rediscovery","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":41,"type":"result","entityId":"1386","action":"added","label":"Claude Mythos 5.1 · De novo binder design","modelId":"claude-mythos-5-1","benchmarkSlug":"de-novo-binder-design-24h","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"de-novo-binder-design-24h","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-binder\", \"effort\": \"highest available\", \"safeguards\": false, \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1386,"model_id":"claude-mythos-5-1","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":79.1,"source_location":"Benchmark performance table: De novo binder design (24 hr)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":42,"type":"result","entityId":"1387","action":"added","label":"Claude Opus 5.5 · De novo binder design","modelId":"claude-opus-5-5","benchmarkSlug":"de-novo-binder-design-24h","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"de-novo-binder-design-24h","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"anthropic-fable-mythos-5.2-2026-11-09-binder\", \"effort\": \"highest available\", \"safeguards\": false, \"settings\": \"November 9 launch-table comparison; highest available effort unless explicitly noted. Preserve the benchmark track and model configuration; competitor figures are as reported by Anthropic.\", \"tools\": true}","id":1387,"model_id":"claude-opus-5-5","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":82.6,"source_location":"Benchmark performance table: De novo binder design (24 hr)","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":43,"type":"result","entityId":"1388","action":"added","label":"GPT-6 Cyber · CyScenarioBench","modelId":"gpt-6-cyber","benchmarkSlug":"cyscenariobench","source":"https://www.anthropic.com/claude-fable-and-mythos-5-2","previous":null,"current":{"benchmark_id":"cyscenariobench-openai-cyber-launch","created_at":"2026-11-10T01:04:32.955Z","evaluation_settings":"{\"comparison_group\": \"openai-cyber-cyscenario-unique-tasks\", \"effort\": \"highest available\", \"settings\": \"GPT-6 Cyber: 61%. Labs use unique tasks; not a shared task set with the Anthropic 10-challenge subset.\", \"tools\": true}","id":1388,"model_id":"gpt-6-cyber","notes":"{\"attribution\": \"Anthropic launch-table report; not a new independent measurement\", \"checked_at\": \"2026-11-09\", \"evidence\": \"database/models/reviewed-2026-11-09/anthropic-launch.md\"}","reported_at":"2026-11-09","score":61,"source_location":"Benchmark performance table: CyScenarioBench / GPT-6 Cyber","source_publisher":"Anthropic","source_title":"Claude Fable 5.2 and Mythos 5.2 launch benchmark comparison","source_type":"provider_reported","source_url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","status":"published","supersedes_result_id":null}},{"id":44,"type":"index","entityId":"claude-mythos-5-1","action":"added","label":"Claude Mythos 5.1 · Agent Index","modelId":"claude-mythos-5-1","benchmarkSlug":"agent-index","source":null,"previous":null,"current":{"coverage":0,"model_id":"claude-mythos-5-1","rank":null,"score":null,"status":"insufficient_coverage","version_id":"agent-index-v1-3-preview"}}]},{"id":2,"recordedAt":"2026-11-10T00:06:07.886Z","kind":"update","summary":"1 dataset change recorded","changeCount":1,"changes":[{"id":1,"type":"policy","entityId":"reported-results","action":"updated","label":"Latest reported results","modelId":null,"benchmarkSlug":null,"source":null,"previous":{"policy":"{\n  \"version\": \"2026-11-09\",\n  \"reportSelection\": \"Within each model, benchmark version and evaluation setup, select the newest publication date, then the best reported effort on that date, then report ID. Use the recorded date only when the publication date is unavailable.\",\n  \"releaseSelection\": \"Family summaries prefer the newest explicitly dated benchmark release with comparable results. Undated or incompatible harnesses retain the designated comparison track. Historical versions remain available on benchmark pages.\",\n  \"indexSelection\": \"Agent Index uses its own immutable, versioned methodology. New benchmark releases are adopted explicitly, never silently substituted.\",\n  \"discovery\": \"Latest means latest verified report in this dataset. External source discovery and verification are not continuous or automatic.\",\n  \"history\": \"Changes are recorded during data refresh before a build. The initial snapshot establishes a baseline; it does not reconstruct earlier edits. Recorded timestamps describe dataset changes, not source publication dates.\"\n}\n"},"current":{"changes":"Charts now select the latest dated report before ranking. Heatmap summaries use the newest dated benchmark release. Earlier reports and index versions remain stored.","discovery":"Latest means latest verified report in this dataset. External source discovery and verification are not continuous or automatic.","history":"Changes are recorded during data refresh before a build. The initial snapshot establishes a baseline; it does not reconstruct earlier edits. Recorded timestamps describe dataset changes, not source publication dates.","indexSelection":"Agent Index uses its own immutable, versioned methodology. New benchmark releases are adopted explicitly, never silently substituted.","releaseSelection":"Family summaries prefer the newest explicitly dated benchmark release with comparable results. Undated or incompatible harnesses retain the designated comparison track. Historical versions remain available on benchmark pages.","reportSelection":"Within each model, benchmark version and evaluation setup, select the newest publication date, then the best reported effort on that date, then report ID. Use the recorded date only when the publication date is unavailable.","version":"2026-11-09"}}]},{"id":1,"recordedAt":"2026-11-10T00:04:05.315Z","kind":"baseline","summary":"Change tracking started for the existing dataset","changeCount":0,"changes":[]}],"nextBefore":null}