{ "cases": [ { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/118", "summary": "ADE-bench PR adds a run_id discriminator to container names so concurrent runs of the same tasks do not collide in Docker.", "task_scope": "benchmark-level", "title": "Add run-level discriminator to container names", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/issues/55", "summary": "ADE-bench issue argues tasks should not require agents to edit dbt_packages because those files are overwritten/gitignored, making such solutions invalid in real workflows.", "task_scope": "ade-bench-quickbooks001", "title": "Agents should not be expected to edit the contents of the `dbt_packages` directory to solve a problem", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/issues/57", "summary": "API 호출 오류가 있었던 실행을 통과나 실패로 세면 모델 성능 신호가 오염될 수 있어 indeterminate로 분리해야 한다는 신고", "task_scope": "benchmark-level", "title": "API 응답 오류를 실패/통과로 처리하는 판정", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/85", "summary": "Maintainer PR cleaned inconsistent dbt_utils/package dependency handling that made logs noisy and auto-generated equality tests depend on inconsistently installed packages.", "task_scope": "benchmark-level", "title": "Cleanup deps cycle", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/150", "summary": "ADE-bench disables dbt-fusion+duckdb variants on six tasks because an upstream duckdb-driver bug made always-pass CI fail.", "task_scope": "ade-bench-ana-eng001, ade-bench-ana-eng002, ade-bench-ana-eng002-medium, ade-bench-ana-eng004, ade-bench-ana-eng005, ade-bench-ana-eng006, ade-bench-ana-eng007, ade-bench-ana-eng007-medium", "title": "Disable dbt-fusion + duckdb variant on 6 analytics_engineering tasks", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/issues/76", "summary": "Running ADE-bench sibling task variants simultaneously causes IO/file-not-found errors due generated test files colliding; maintainers discuss sequential workaround.", "task_scope": "benchmark-level", "title": "Don't get errors when running two variants of a task simultaneously", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/144", "summary": "Fixes ADE-bench harness paths where setup/migration/solution patch failures could be silently ignored and surface as downstream errors instead of setup failures.", "task_scope": "benchmark-level", "title": "feat: detect and fail loudly on patch failures in setup/migration/solution scripts", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/36", "summary": "ADE-Bench analytics_engineering008 task was fixed to explicitly say where the new dbt project should be created.", "task_scope": "ade-bench-ana-eng008", "title": "Fix analytics_engineering008: be explicit about where the new dbt project should be created", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/69", "summary": "ADE-bench PR fixes differing counts across harness/results writer/parser outputs, indicating benchmark result aggregation inconsistency.", "task_scope": "benchmark-level", "title": "fix issue where counts are different", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/102", "summary": "ADE-bench PR fixes asana003 task.yaml task_id incorrectly set to asana002.", "task_scope": "ade-bench-asana003", "title": "fix(asana003): correct task_id typo", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/120", "summary": "Fixes asana003 structural check that rejected valid solutions using multiple source() calls; 34/80 runs had all data tests pass but failed only this check.", "task_scope": "ade-bench-asana003", "title": "fix(asana003): relax check_model_sources to accept multiple source() calls", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/104", "summary": "Live ADE-bench PR adds schema: main to f1001 source definition because dbt could not locate source tables without it.", "task_scope": "ade-bench-f1001", "title": "fix(f1001): add schema: main to source definition in setup.sh", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/132", "summary": "Live PR fixes ADE-Bench test generator relation lookup and disables static analysis/partial parsing that interfered with evaluation, but author is a repo collaborator.", "task_scope": "benchmark-level", "title": "fix(test-harness): fusion compatibility with load_relation and static analysis off", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/147", "summary": "ADE-bench non-editable installs omitted shared, docker, and tasks directories from the wheel, breaking handler path resolution, compose build contexts, and task enumeration.", "task_scope": "benchmark-level", "title": "Include shared/, docker/, and tasks/ in the built wheel", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/issues/11", "summary": "Known transient failures where concurrently running tasks read generated tests while other tasks are creating/updating them, causing file-not-found errors that pass on rerun.", "task_scope": "benchmark-level", "title": "Known transient failures", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/issues/70", "summary": "dbt-fusion panic can mark a correct agent execution failed; maintainer adds handling while root cause remains upstream.", "task_scope": "benchmark-level", "title": "Panic! at the dbt", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/88", "summary": "Live ADE-bench PR fixes race conditions where two concurrent runners could delete each other's generated AUTO test SQL files.", "task_scope": "benchmark-level", "title": "Resolve race conditions when two runners try to delete files at once", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/34", "summary": "ADE-Bench quickbooks tasks needed dbt deps in setup so dependencies are installed before runs.", "task_scope": "ade-bench-quickbooks001, ade-bench-quickbooks002, ade-bench-quickbooks003, ade-bench-quickbooks004", "title": "Run `dbt deps` in setup for quickbooks tasks.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/issues/96", "summary": "ADE-bench issue shows multiple attempts at the same task share the same HTML results path, so the viewer opens the wrong run output.", "task_scope": "benchmark-level", "title": "The HTML results directory does not store results for multiple attempts at the same task", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "ade-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/dbt-labs/ade-bench/pull/95", "summary": "ADE-bench PR updates QuickBooks task setup because the old task asked agents to modify a source table directly, an unrealistic task design.", "task_scope": "ade-bench-quickbooks001", "title": "Update quickbooks with realistic data type error", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "aider-polyglot", "evidence_label": "external_source_reported", "source_url": "https://github.com/Aider-AI/polyglot-benchmark/pull/10", "summary": "Live PR states the JavaScript twelve-days task instructions omit necessary details, making one-shot solution impossible and ambiguity/data-contamination likely.", "task_scope": "polyglot_javascript_twelve-days", "title": "Fixed impossible-to-one-shot instructions for twelve-days JS problem", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "aider-polyglot", "evidence_label": "external_source_reported", "source_url": "https://github.com/Aider-AI/polyglot-benchmark/issues/11", "summary": "Aider polyglot benchmark mocked open() in a way that penalized reasonable os.path.exists implementations not forbidden by the prompt.", "task_scope": "polyglot_python_grep", "title": "tests implmentation affects pass@1 in polyglot-benchmark", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "aider-polyglot", "evidence_label": "external_source_reported", "source_url": "https://github.com/Aider-AI/polyglot-benchmark/issues/14", "summary": "복원되는 테스트 파일이 패키지 선언과 주석뿐이라 go test가 아무것도 검증하지 않고 성공할 수 있다는 신고", "task_scope": "polyglot_go_counter", "title": "실제 테스트가 0개인데 통과하는 Go 과제", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "algotune", "evidence_label": "external_source_reported", "source_url": "https://github.com/oripress/AlgoTune/pull/13", "summary": "AlgoTune PR adds a more complete self-removal function to fix boundary cases reported in issue #14.", "task_scope": "benchmark-level", "title": "[Add] adding more perfect self-removing function", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/33", "summary": "ARC-AGI-2 PR fixes evaluation task 67e490f4 where a shape in the test input is one pixel too short for the intended rule.", "task_scope": "67e490f4_0", "title": "67e490f4 test input - fixed length of shape", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/31", "summary": "PR fixes ARC-AGI-2 task 6e4f6532 by correcting an extra pixel in example 1 and a shape mismatch in test 1.", "task_scope": "6e4f6532_0, 6e4f6532_1", "title": "6e4f6532 example 1 extra pixel + test 1 shape mismatch (supercedes pull 30)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/27", "summary": "ARC-AGI-2 PR fixes evaluation task 898e7135 where two squares in example 2 are offset upward by one pixel.", "task_scope": "898e7135_0", "title": "898e7135 example 2 squares offset by 1px", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/34", "summary": "ARC-AGI-2 PR corrects evaluation task 8e5c0c38 test output where the least-pixel-removal symmetry rule was applied incorrectly.", "task_scope": "8e5c0c38_0, 8e5c0c38_1", "title": "8e5c0c38 test 1 - correct application of rule", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/29", "summary": "ARC-AGI-2 PR fixes evaluation task a25697e4 where example 1’s mirrored shape has a one-pixel offset error.", "task_scope": "a25697e4_0, a25697e4_1", "title": "a25697e4 example 1 shape has offset error by 1 px", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/26", "summary": "ARC-AGI-2 PR fixes task cb2d8a2c test 1 output by replacing an inconsistent blue shape with red.", "task_scope": "cb2d8a2c_0, cb2d8a2c_1", "title": "cb2d8a2c test 1 output - replace blue with red", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/22", "summary": "ARC-AGI-2 issue reports puzzle 247ef758 accepts a recolored-border solution despite no example suggesting border recoloring; maintainer says the task was updated.", "task_scope": "247ef758_0, 247ef758_1", "title": "Error in puzzle 247ef758", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/35", "summary": "ARC-AGI-2 issue reports evaluation task 271d71e2 example 2 has the middle rectangle moved one unit too far right; maintainer says it was fixed on GitHub.", "task_scope": "271d71e2_0", "title": "evaluation/271d71e2 example 2 output seems to have a mistake", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/36", "summary": "ARC-AGI-2 PR fixes evaluation task 271d71e2 by shifting the middle rectangle in training example 2 so the boundary/bar relationship matches the rule.", "task_scope": "271d71e2_0", "title": "evaluation/271d71e2: in training example 2, shift middle rectangle.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/23", "summary": "Live ARC PR fixes f560132c test input where a grey shape was missing two pixels compared with the expected output pattern.", "task_scope": "f560132c_0", "title": "f560132c test 2 - grey input shape missing 2 pixels", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/25", "summary": "ARC-AGI-2 PR fixes evaluation task f931b4a8 example 3, which had a random color switch and missing tiling relative to the rule used by examples and tests.", "task_scope": "f931b4a8_0, f931b4a8_1", "title": "f931b4a8 example 3 incorrect color switch and missing tiling", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/64", "summary": "ARC-AGI-2 PR fixes evaluation item a6f40cea by changing train output frame-border cells from 8 to 9 and 9 to 8 at specified positions.", "task_scope": "a6f40cea_0", "title": "fix a6f40cea frame border", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/11", "summary": "ARC-AGI-2 issue reports puzzle d8e07eb2 changes the highlighted shape into a different shape; maintainer says it was fixed and then updated again.", "task_scope": "d8e07eb2_0, d8e07eb2_1", "title": "Mistake in puzzle d8e07eb2 [Not fixed on wesbsite]", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/73", "summary": "Public evaluation task a6f40cea has an off-by-one (striped-frame phase) inconsistency in train pair 2's output grid. The maintainer acknowledged it as an unintentional error, not prioritized for fix because it sits in a train example.", "task_scope": "a6f40cea_0", "title": "Off-by-one striped-frame inconsistency in public eval task", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/1", "summary": "Reporter argues ARC challenge 16de56c4 test case expected output is inconsistent with the training rule; maintainer caught the tweet and updated the task.", "task_scope": "16de56c4_0, 16de56c4_1", "title": "Possible Mistake in Test Case 0 for Challenge 16de56c4", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/45", "summary": "ARC-AGI-2 issue argues puzzle 291dc1e1 has several plausible transformations from the single asymmetric example, making the intended answer ambiguous.", "task_scope": "291dc1e1_0", "title": "Puzzle 291dc1e1 has too many possible solutions", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/14", "summary": "ARC-AGI-2 issue reports puzzle dfadab01 requires information from a second test case; maintainer agrees information gain should not sit in test cases and converts/fixes the task.", "task_scope": "dfadab01_0", "title": "Puzzle breaking the rules! dfadab01", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/issues/39", "summary": "ARC-AGI-2 puzzle f560132c test 1 has rotational symmetry and non-unique solutions; comments say two solutions are acceptable and point to a changelog resolution.", "task_scope": "f560132c_0", "title": "Puzzle f560132c (test 1) does not have a unique solution", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "arc_agi_2", "evidence_label": "external_source_reported", "source_url": "https://github.com/arcprize/ARC-AGI-2/pull/75", "summary": "ARC-AGI-2 task d931c21c includes an unsupported extra outer border in a training output; open PR #75 edits data/training/d931c21c.json to remove it.", "task_scope": "d931c21c_0", "title": "Remove extra outer border in d931c21c", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/737", "summary": "BFCL v3 live dataset fix series removes ground-truth mismatches and ambiguous user intents from simple/parallel categories.", "task_scope": "benchmark-level", "title": "[BFCL Dataset Revamp 2/n] Live Dataset Fix (Simple, Parallel, Parallel Multiple)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/739", "summary": "BFCL v3 live dataset cleanup explicitly aims to eliminate ground-truth mismatches and ambiguous prompts across multiple files.", "task_scope": "benchmark-level", "title": "[BFCL Dataset Revamp 3/n] Live Dataset Fix (Multiple)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/763", "summary": "BFCL live irrelevance dataset and possible-answer fixes as part of V3 bug-fix sweep.", "task_scope": "benchmark-level", "title": "[BFCL Dataset Revamp 4/n] Live Irrelevance", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/789", "summary": "BFCL live relevance data contained entries that actually belonged in multiple or irrelevance categories because parameter availability was misclassified.", "task_scope": "benchmark-level", "title": "[BFCL Dataset Revamp 6/n] Live Relevance Data Fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/804", "summary": "BFCL PR fixes multi-turn dataset issues including ground-truth label mismatches, plausible alternatives, and ambiguous phrasing.", "task_scope": "benchmark-level", "title": "[BFCL Dataset Revamp 7/n] Augmented Multi-turn Dataset Fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/846", "summary": "PR fixes several BFCL dataset ground truth errors across exec_parallel, java, live_multiple, and parallel categories.", "task_scope": "bfcl-simple-java-94, bfcl-live-multiple-969-210-1", "title": "[BFCL Dataset] Ground Truth Error Fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/709", "summary": "BFCL Java/JavaScript prompts were rephrased because questions could be read as asking for explanations rather than function-call actions.", "task_scope": "benchmark-level", "title": "[BFCL v1] Rephrase Question for Better Clarity for Java & JavaScript Categories", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/708", "summary": "BFCL v1 REST executable ground truth had to be updated because an API changed its response structure and sanity checks failed.", "task_scope": "benchmark-level", "title": "[BFCL v1] Update Executable Ground Truth for REST Category", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/661", "summary": "BFCL v2/v3 live datasets and possible answers required broad fixes across hundreds of entries, including ambiguous ground truths noted in review.", "task_scope": "bfcl-irrelevance-196, bfcl-live-multiple-59-22-6", "title": "[BFCL-v2] Dataset and Possible Answer Fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1206", "summary": "Merged BFCL PR fixed multiple V4 multi-turn dataset prompts, possible answers, and backend behavior where ambiguous instructions or impossible/incorrect gold trajectories affected scoring.", "task_scope": "benchmark-level", "title": "[BFCL] - Additional Dataset Fixes, Builds off Issues 1133 PR", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1215", "summary": "BFCL format_sensitivity accuracy computation used the wrong branch and produced significantly changed scores after correction.", "task_scope": "benchmark-level", "title": "[BFCL] [Bug] Evaluation of format_sensitivity is incorrect", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1244", "summary": "BFCL generation could corrupt result files under high thread counts because concurrent writers missed entries; merged PR #1244 serializes writes with per-file locks.", "task_scope": "benchmark-level", "title": "[BFCL] Add file locking to utils.py to serialize concurrent writes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1197", "summary": "BFCL partial_eval CLI flag did not reach evaluation_main, so bfcl evaluate --partial-eval had no effect.", "task_scope": "benchmark-level", "title": "[BFCL] Add partial_eval parameter to evaluate function in __main__.py", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/777", "summary": "BFCL PR adds a unit test for illegal Python parameter names and updates affected possible-answer data so AST parsing does not error.", "task_scope": "benchmark-level", "title": "[BFCL] Add Unit Test to Check for Illegal Python Parameter Name", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/893", "summary": "BFCL score files used one-indexed IDs while result JSON files used testcase IDs, making evaluator output inconsistent across files; PR #893 aligns score IDs to result IDs.", "task_scope": "benchmark-level", "title": "[BFCL] align score file IDs with result testcase IDs", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/620", "summary": "BFCL Java/JavaScript prompts asked how to initiate actions, leading models to explain rather than emit function calls; maintainer said they would address it.", "task_scope": "benchmark-level", "title": "[BFCL] Ambigious Instructions in Java/Javascript Test Cases", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/536", "summary": "PR applies earlier BFCL model-handler fixes to a newly added Granite handler that missed them.", "task_scope": "benchmark-level", "title": "[BFCL] Apply Fix to Newly Introduced Model Handler Missed in Previous PR Merge", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/805", "summary": "BFCL Claude prompt-caching handler constructed execution-result messages in the wrong content format.", "task_scope": "benchmark-level", "title": "[BFCL] Bug Fix for Execution_Result_Message Construction for Prompt Caching Feature in Claude Handler", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/656", "summary": "BFCL PR fixed single-turn dataset format issues where question was wrapped in an unnecessary extra list.", "task_scope": "benchmark-level", "title": "[BFCL] Bug Fix inference_single_turn_FC function for base_handler", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/660", "summary": "BFCL PR fixed parse_nested_value so dictionary arguments are not misread as nested function calls.", "task_scope": "benchmark-level", "title": "[BFCL] Bug Fix parse_nested_value function for model_handler utils", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1252", "summary": "BFCL Grok handler called the prompting reasoning-content helper instead of the function-calling helper, dramatically lowering multi-turn results.", "task_scope": "benchmark-level", "title": "[BFCL] Bug report", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1167", "summary": "BFCL leaderboard generation crashed with ZeroDivisionError when weighted accuracy was computed for categories with zero evaluated results.", "task_scope": "benchmark-level", "title": "[BFCL] Bug: `ZeroDivisionError` in `calculate_weighted_accuracy` when categories lack evaluation results", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1168", "summary": "BFCL_v4_simple_javascript possible_answer IDs used double underscores while source data used single underscores; maintainer confirmed it was fixed in PR 1196.", "task_scope": "benchmark-level", "title": "[BFCL] Bug: ID format mismatch in `possible_answer/BFCL_v4_simple_javascript.json`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/652", "summary": "BFCL issue reports executable REST predictions with dict arguments failing decode because parse_nested_value assumed nested values were function calls.", "task_scope": "benchmark-level", "title": "[BFCL] Bugs in decode executable predictions", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/753", "summary": "BFCL generation crashed because _multi_threaded_inference passed include_state_log to inference_single_turn_prompting with the wrong signature.", "task_scope": "benchmark-level", "title": "[BFCL] bugs in function def _multi_threaded_inference(self, test_case, include_input_log: bool, include_state_log: bool):", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/675", "summary": "BFCL PR fixed hard-coded relative paths that made scripts impossible to run from different directories.", "task_scope": "benchmark-level", "title": "[BFCL] Chore: Refactor File Path Handling and Automate apply_function_credential_config.py", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1352", "summary": "BFCL V4 Agentic Web Search documentation leaves synthetic request failures ambiguous while the released leaderboard checkpoint has the failure injection code disabled, obscuring how default and robustness runs are reproduced.", "task_scope": "benchmark-level", "title": "[BFCL] Clarify default and ablation behavior of synthetic request failures in BFCL V4 Web Search", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/600", "summary": "BFCL PR fixed 24 dataset and possible-answer entries across simple, multiple, parallel, and parallel-multiple categories.", "task_scope": "bfcl-simple-python-277, bfcl-simple-python-322, bfcl-multiple-64, bfcl-multiple-160, bfcl-parallel-45, bfcl-parallel-49", "title": "[BFCL] Dataset and Possible Answer Fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/778", "summary": "BFCL fixes 55 dataset entries where illegal Python parameter name class appeared in function docs/possible answers.", "task_scope": "benchmark-level", "title": "[BFCL] Dataset and Possible Answer Fix (Live Categories) for Illegal Python Parameter Name", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1365", "summary": "BFCL eagerly imports the Qwen Agent handler during CLI startup, so qwen_agent transitively requires soundfile before bfcl --help can run for non-Qwen users.", "task_scope": "benchmark-level", "title": "[BFCL] Defer qwen_agent import until handler initialization", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/575", "summary": "BFCL issue/plan corrected precision settings for local models that should use bfloat16 instead of float16, affecting runnable hardware and scores.", "task_scope": "benchmark-level", "title": "[BFCL] Evaluation with Correct Precision Settings for Locally-Hosted Models", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/870", "summary": "BFCL ThinQ_Connect tasks expected airConJobMode in ground truth, but the function spec parameter is currentJobMode; PR fixes possible_answer entries.", "task_scope": "bfcl-live-simple-45-18-1", "title": "[BFCL] Fix `live_simple_44-18-0` and `live_simple_45-18-1`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/959", "summary": "BFCL PR fixes clean-repo API key handling bugs where missing keys still allowed REST test collection/execution and could also hit an undefined existing_ids path.", "task_scope": "benchmark-level", "title": "[BFCL] Fix API Keys Handling", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/924", "summary": "PR fixes bfcl evaluate --score-dir handling: the code accidentally reassigned result_dir instead of resolving score_dir, so custom score output paths were mishandled.", "task_scope": "benchmark-level", "title": "[BFCL] fix bug in how score_dir is handled for bfcl evaluate", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/557", "summary": "BFCL PR fixed many dataset/possible-answer issues across simple, multiple, parallel, executable, JavaScript categories.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Dataset and Possible Answer Issue", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/545", "summary": "PR fixes BFCL Java/JavaScript preprocessing so all model handlers get the same function docs and nested types are handled correctly.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Dataset Pre-Processing for Java and JavaScript Test Category, Part 2", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/623", "summary": "BFCL PR fixed Nvidia handler decoding whitespace that caused many model responses to fail decoding and affected leaderboard scores.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Decoding Issue in Nvidia Handler", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/516", "summary": "PR fixes BFCL double-casting in Java/JavaScript model handlers that gave models a second chance and introduced false positives.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Double-Casting Issue in model_handler for Java and JS category.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/663", "summary": "BFCL PR fixed OSS model inference hanging indefinitely on some multi-turn entries such as multi_turn_36.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Hanging Inference for OSS Models on GPU Platforms", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/796", "summary": "BFCL DeepSeek-Coder handler treated non-function-call messages as decoded outputs, harming irrelevance-category scoring.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Irrelevance Category Performance for DeepSeek Coder Handler", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1082", "summary": "Merged BFCL PR fixes is_fc_model propagation from model_config into handlers/evaluator.", "task_scope": "benchmark-level", "title": "[BFCL] Fix is_fc_model config propagation", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/549", "summary": "Live BFCL PR fixes JS type conversion for dictionaries with array values and notes the change will affect leaderboard scores.", "task_scope": "benchmark-level", "title": "[BFCL] Fix JS type converter to handle dictionaries with array values", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/538", "summary": "PR fixes BFCL Java/JavaScript language-specific preprocessing that an indentation bug left incorrectly implemented.", "task_scope": "benchmark-level", "title": "[BFCL] Fix language_specific_pre_processing for Java and JavaScript Test Category", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/871", "summary": "Live BFCL PR updates possible answers so a non-required schema parameter with default expected value also accepts an empty string.", "task_scope": "bfcl-live-simple-165-98-0", "title": "[BFCL] Fix live_simple_165-98-0", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/626", "summary": "BFCL PR fixed Llama handler prompt formatting missing newlines after end_header_id tags, affecting leaderboard scores.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Llama Handler", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/881", "summary": "BFCL PR fixes Nova evaluation failures where live_irrelevance entries contained consecutive user prompts, violating AWS Bedrock Converse alternating-role requirements.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Nova Handler for Consecutive User Prompt Issue", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/503", "summary": "HuanzhiMao fixed two BFCL possible-answer entries in AST parallel and parallel-multiple categories and noted the leaderboard would need score updates.", "task_scope": "benchmark-level", "title": "[BFCL] Fix Possible Answer for AST Parallel and Parallel_Multiple Category", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1068", "summary": "Merged BFCL PR fixes Qwen prompt concatenation where <|im_end|> was not appended in-place, significantly affecting Qwen3 leaderboard performance.", "task_scope": "benchmark-level", "title": "[BFCL] Fix prompt concatenation bug in Qwen template", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1175", "summary": "BFCL live_simple_205-116-13 asked for upcoming Saturday from Tuesday April 25 2023, but possible answer still expected April 27 instead of April 29, marking correct model outputs invalid.", "task_scope": "bfcl-live-simple-205-116-13", "title": "[BFCL] Fix wrong date answer in live_simple_205-116-13", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1347", "summary": "BFCL memory kv backend raised ZeroDivisionError when key_search ran against an empty memory store, surfacing a backend error to models instead of an empty result; open PR #1347 guards the empty corpus path.", "task_scope": "benchmark-level", "title": "[BFCL] Fix ZeroDivisionError in memory kv backend when searching an empty memory", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1314", "summary": "BFCL live categories reject valid enum/default values across nine entries, including units, weather units, Spotify search format, and date defaults.", "task_scope": "bfcl-live-simple-241-125-4", "title": "[BFCL] Ground truth rejects valid enum values in live_parallel and live_simple", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/636", "summary": "BFCL PR removed extra NoAPIKeyError parameters for executable categories where all API keys are required for reliable results.", "task_scope": "benchmark-level", "title": "[BFCL] Hot Fix to Remove Extra Parameters for NoAPIKeyError", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/505", "summary": "CharlieJCJ simplified BFCL tree-sitter Java/JavaScript installation and tested end-to-end generation after the prior setup path was fragile.", "task_scope": "benchmark-level", "title": "[BFCL] Improved tree-sitter java, javascript installation", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/581", "summary": "BFCL issue reports temporal/current-date models making valid answers impossible under current possible answers for several tasks.", "task_scope": "benchmark-level", "title": "[BFCL] Issue with temporal models/function calling", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/903", "summary": "BFCL issue reports live_simple_205-116-13 says Thursday April 25th 2023, but that date was a Tuesday, making the date-dependent expected tool call wrong.", "task_scope": "bfcl-live-simple-205-116-13", "title": "[BFCL] live_simple_205-116-13 wrong date", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/614", "summary": "BFCL issue reports ModuleNotFoundError after restructuring because imports were not updated to the new bfcl package paths.", "task_scope": "benchmark-level", "title": "[BFCL] ModuleNotFoundError after Restructuring in Commit 70d6722: Incorrect Import Paths", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/683", "summary": "BFCL multi-turn data and possible answers had ambiguity in action intention and function parameters, affecting leaderboard score.", "task_scope": "benchmark-level", "title": "[BFCL] Multi Turn Dataset and Possible Answer Fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/719", "summary": "BFCL PR fixes multi-turn base configs and ground-truth calls that caused execution errors even when following ground truth.", "task_scope": "benchmark-level", "title": "[BFCL] Multi Turn Dataset and Possible Answer Fix (Base Category)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/723", "summary": "Live PR fixes BFCL ambiguous prompts and wrong ground truth, but the author is a project collaborator/internal actor.", "task_scope": "benchmark-level", "title": "[BFCL] Multi Turn Dataset Fix (Base Category)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/722", "summary": "Maintainer PR fixed multi-turn function doc extraction/compilation bugs that produced wrong or missing function documentation and was part of a broader ground-truth bugfix effort.", "task_scope": "benchmark-level", "title": "[BFCL] Multi Turn Dataset Fix (Function Doc)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/724", "summary": "BFCL multi-turn evaluator needed guards against unsafe function calls like kill/exit/remove that could break the evaluation pipeline.", "task_scope": "benchmark-level", "title": "[BFCL] Multi Turn Pipeline Robustness Patch", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1064", "summary": "Merged BFCL PR prevents reasoning content from leaking into subsequent chat turns for function-calling models, matching earlier prompt-mode cleanup.", "task_scope": "benchmark-level", "title": "[BFCL] Omit Reasoning Content from Chat History for Function-Calling Models", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1159", "summary": "BFCL evaluate compared a subset model result against full prompt/possible-answer files, causing a length-mismatch AssertionError; PR #1185 adds partial-eval and ID-based subsetting.", "task_scope": "benchmark-level", "title": "[BFCL] partial evaluation length-mismatch AssertionError", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/730", "summary": "BFCL multi-turn evaluator stripped the object prefix from function calls, producing variable-not-found errors during eval(func_call).", "task_scope": "benchmark-level", "title": "[BFCL] Patch #724", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/421", "summary": "HuanzhiMao patched the BFCL Gemini handler to make generation more robust after parallel-call handling changes.", "task_scope": "benchmark-level", "title": "[BFCL] Patch Gemini Handler", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1004", "summary": "BFCL issue shows bfcl evaluate splits underscores in a local model path into slashes, causing KeyError for valid model paths.", "task_scope": "benchmark-level", "title": "[BFCL] Path Splitting Error in bfcl evaluate Command", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/462", "summary": "HuanzhiMao made the BFCL checker robust when less-capable models emit non-float-compatible values for float parameters instead of crashing the pipeline.", "task_scope": "benchmark-level", "title": "[BFCL] PR#407 Evaluation Pipeline Robustness Patch", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/651", "summary": "BFCL issue reports proprietary model handler failing with missing test_entry argument in inference_single_turn_FC.", "task_scope": "benchmark-level", "title": "[BFCL] proprietary model handle", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/733", "summary": "Live BFCL PR refines multi-turn evaluation by removing flag_task_unachievable behavior and adding response checking after identifying over-penalization and scoring issues.", "task_scope": "benchmark-level", "title": "[BFCL] Refine Evaluation Metric for Multi Turn Categories", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/593", "summary": "BFCL PR moved formatting/function docs to the system prompt and also fixed six named dataset issues.", "task_scope": "bfcl-irrelevance-49, bfcl-live-parallel-0-0-0", "title": "[BFCL] Relocate Formatting Instructions and Function Documentation to System Prompt", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1246", "summary": "BFCL removed update_market_status from the trading-bot backend because models should not be able to mutate stock-market state; affected test cases were updated.", "task_scope": "benchmark-level", "title": "[BFCL] Remove update_market_status from trading-bot backend", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/811", "summary": "BFCL dataset used class as a parameter name, causing function-calling formatting errors until renamed to _class.", "task_scope": "benchmark-level", "title": "[BFCL] Replace 'class' with '_class' to Avoid Function Calling Formatting Error", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1057", "summary": "Merged BFCL PR changes Java/JavaScript parser failures from generic Exception to SyntaxError so model syntax errors are handled separately from runtime/evaluator failures.", "task_scope": "benchmark-level", "title": "[BFCL] Replace Exception with SyntaxError for Java and JavaScript Parsers", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/809", "summary": "BFCL Gemini handler crashed with IndexError when Gemini returned no candidates, especially in irrelevance prompt mode.", "task_scope": "benchmark-level", "title": "[BFCL] Resolve Issue in Gemini Model When No Model Output", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/943", "summary": "BFCL retired executable categories because external APIs caused random sanity-check failures, bandwidth issues, and inconsistent evaluation.", "task_scope": "benchmark-level", "title": "[BFCL] Retire Executable Categories from Leaderboard", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/754", "summary": "BFCL _multi_threaded_inference passed include_state_log into a function that did not accept it, causing generation runtime errors.", "task_scope": "benchmark-level", "title": "[BFCL] Robustness Patch for `_multi_threaded_inference`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/496", "summary": "ShishirPatil made BFCL API sanity checks optional so one down external API would not block unrelated AST evaluations.", "task_scope": "benchmark-level", "title": "[BFCL] Sanity check is now optional", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/486", "summary": "ShishirPatil reported that BFCL sanity checks should not halt unrelated offline evaluations when a REST API is down.", "task_scope": "benchmark-level", "title": "[BFCL] Sanity check should be optional and by default off", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1367", "summary": "BFCL v4 multi_turn_base contains current tasks whose gold calls require strings or file effects not derivable from the user turns, initial state, or tool outputs.", "task_scope": "bfcl-multi-turn-base-4, bfcl-multi-turn-base-8, bfcl-multi-turn-base-10, bfcl-multi-turn-base-18, bfcl-multi-turn-base-35, bfcl-multi-turn-base-178, bfcl-multi-turn-base-180, bfcl-multi-turn-base-187", "title": "[BFCL] Several multi_turn_base items have ground truth that cannot be produced from the environment", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/848", "summary": "BFCL executable categories errored when required API keys were absent; PR changed the pipeline to skip affected categories and clarify scoring.", "task_scope": "benchmark-level", "title": "[BFCL] Skip Executable Categories When API Keys Missing", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/820", "summary": "BFCL issue reports live task tool schemas with integer parameters whose enum values are strings; maintainer confirms this is an OpenAPI-format oversight and invites a PR to fix all similar cases.", "task_scope": "bfcl-live-simple-178-103-1", "title": "[BFCL] some anomalies about the dataset", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/786", "summary": "BFCL possible answers contained scalar values where lists were required, producing invalid-value errors for otherwise valid outputs.", "task_scope": "bfcl-simple-javascript-5, bfcl-live-multiple-189-83-0", "title": "[BFCL] some tiny fix in possible_answer", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/439", "summary": "HuanzhiMao standardized BFCL model naming between handler_map and eval_runner_helper to remove confusing mismatches when running evaluations.", "task_scope": "benchmark-level", "title": "[BFCL] Standardize Model Name Among handler_map and eval_runner_helper", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/637", "summary": "BFCL issue identifies incorrect GLM prompt formatting: missing tool descriptions and no multiple-role support, causing abnormal leaderboard behavior.", "task_scope": "benchmark-level", "title": "[BFCL] The handling of prompt formatting in GLM is incorrect.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1216", "summary": "BFCL language-specific hint logic incorrectly applied Python-style hints to Java and JavaScript data.", "task_scope": "benchmark-level", "title": "[BFCL] The logic for adding language-specific hints in the generation framework is incorrect.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/725", "summary": "BFCL multi-turn miss_func/miss_param metric was too strict for reasonable exploratory calls before recognizing a task is unachievable.", "task_scope": "benchmark-level", "title": "[BFCL] Update Eval Metric for Multi Turn Irrelevance Scenarios", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/433", "summary": "justinwangx fixed BFCL tree-sitter dependency breakage after newer tree-sitter removed build_library and blocked data compilation.", "task_scope": "benchmark-level", "title": "[BFCL] update tree_sitter version in requirements.txt", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1247", "summary": "BFCL edit_ticket documentation/schema was corrected so title is an update property and update fields are optional as the backend expects.", "task_scope": "benchmark-level", "title": "[BFCL] Updated edit_ticket documentation", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/849", "summary": "BFCL score reports represented unevaluated categories as 0, conflating skipped categories with true zero scores until N/A reporting was added.", "task_scope": "benchmark-level", "title": "[BFCL] Use `N/A` in Score Report for Unevaluated Categories", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1171", "summary": "BFCL issue reports wrong ground-truth data for live_multiple_59-22-6 and proposes exact corrected sizes list.", "task_scope": "bfcl-live-multiple-59-22-6", "title": "[BFCL] wrong data", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/622", "summary": "BFCL live possible answers had format inconsistencies, including translated-only parameter values that rejected valid Chinese-location outputs.", "task_scope": "benchmark-level", "title": "[BFCL] Wrong Format in the Possible Answers of Live Parallel Multiple", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1261", "summary": "BFCL issue reports verbose_xml output format mapping is malformed because the params tag for func_name2 is missing; maintainer says it was fixed in #1266.", "task_scope": "benchmark-level", "title": "[BFCL] Wrong verbose_xml output format in default_prompts.py", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/364", "summary": "JasonZhu1313 fixed BFCL parsing for Hermes-2-Pro-Mistral-7B by adding the model to UNDERSCORE_TO_DOT so API names are converted correctly.", "task_scope": "benchmark-level", "title": "[Bug fix] Add Hermes-2-Pro-Mistral-7B model to UNDERSCORE_TO_DOT to parse API properly", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/252", "summary": "Fanjia-Yan fixed BFCL parallel-function ground-truth answers that only contained single function calls at listed problematic indices.", "task_scope": "benchmark-level", "title": "[Bug Fix] Fix Error in Parallel Function Possible Answer", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/251", "summary": "Fanjia-Yan fixed a BFCL evaluation-by-execution bug where exact match was not performed when multiple function calls were invoked.", "task_scope": "benchmark-level", "title": "[Bug Fix] Fix Executable Exact Match Condition Did not Meet", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/256", "summary": "Fanjia-Yan restricted BFCL AST checker boolean possible answers so string literals like \"True\" and \"False\" were no longer accepted as booleans.", "task_scope": "benchmark-level", "title": "[Bug Fix] Restrict AST checker on Boolean Variable", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1269", "summary": "Live PR fixes BFCL nanbeige3.5-pro-thinking by setting underscore_to_dot=True and notes that re-evaluation is needed.", "task_scope": "benchmark-level", "title": "[Fix] Correct underscore_to_dot parameter for nanbeige3.5-pro-thinking", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1311", "summary": "BFCL benchmark requests returned garbage against Brotli-capable servers because brotli dependency was missing.", "task_scope": "benchmark-level", "title": "[fix] Update pyproject.toml to fix requests issue", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/245", "summary": "Fanjia-Yan added a Gemini-1.0-Pro result checker and validated that AST and executable BFCL statistics matched expected leaderboard results.", "task_scope": "benchmark-level", "title": "[Update Gemini-1.0-Pro result checker]", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1257", "summary": "BFCL PR adds Nanbeige models and explicitly fixes two small data inconsistencies in possible_answer JSON files, improving benchmark data correctness.", "task_scope": "bfcl-simple-python-363", "title": "Add model Nanbeige and Fix some data bugs", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/642", "summary": "BFCL PR adding Qwen also reports and fixes an anomaly in mean latency calculation for OSS models.", "task_scope": "benchmark-level", "title": "Add Qwen handler and fix mean_latency calculation error for OSS models", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/532", "summary": "BFCL xLAM PR discussion exposed Java/JavaScript evaluation false negatives from prompt/function-document processing, later fixed in #538/#545.", "task_scope": "benchmark-level", "title": "Add Salesforce xLAM handler and fix minor issues", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1173", "summary": "BFCL num_threads handling was adjusted so omitted --num-threads is represented and handled explicitly instead of surprising generation behavior.", "task_scope": "benchmark-level", "title": "Adjust `num_threads` handling in `bfcl generate`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/407", "summary": "vandyxiaowei and danieljannai21 pushed BFCL to loosen overly strict float comparison so valid integer values for float parameters were not incorrectly marked wrong.", "task_scope": "benchmark-level", "title": "better handle float value comparison", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/377", "summary": "HuanzhiMao released BFCL executable-category dataset and pipeline fixes, including prompt/function-doc updates and checker changes.", "task_scope": "benchmark-level", "title": "BFCL April 19th Release (Dataset & Pipeline)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/390", "summary": "HuanzhiMao fixed BFCL cost and latency calculation inconsistencies for open-source model leaderboard metrics.", "task_scope": "benchmark-level", "title": "BFCL April 27th Release (Bug Fix in Cost/Latency Calculation)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/330", "summary": "HuanzhiMao fixed BFCL float-parameter handling and model-output parsing issues that changed leaderboard scores.", "task_scope": "benchmark-level", "title": "BFCL April 8th Release", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/338", "summary": "HuanzhiMao fixed BFCL evaluation dataset prompts, function docs, possible answers, and REST evaluation responses in the April 9 release.", "task_scope": "benchmark-level", "title": "BFCL April 9th Release (Dataset Bug Fix)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/412", "summary": "HuanzhiMao shipped a BFCL release fixing evaluation dataset prompts, function docs, and possible answers after external issue reports.", "task_scope": "benchmark-level", "title": "BFCL May 6th Release (Dataset Bug Fix)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1019", "summary": "BFCL V4 release fixed benchmark-level scoring/data issues, including excluding relevance/irrelevance from Live and Non-Live accuracy calculations and resolving live-relevance ground-truth issue #1094.", "task_scope": "benchmark-level", "title": "BFCL V4 Release", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1006", "summary": "Live BFCL PR bumps writer-sdk to fix a runtime TypeError from incompatible client/proxies behavior.", "task_scope": "benchmark-level", "title": "Bump `writer-sdk` Dependency Version", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/838", "summary": "BFCL possible answers used parameter values with types inconsistent with function descriptions, causing valid model outputs to be marked invalid.", "task_scope": "bfcl-live-multiple-857-180-5", "title": "Check and fix some parameter type errors in possible answers", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/301", "summary": "MayankAgarwal reported detailed BFCL AST dataset issues with problematic indices, which maintainers validated and folded into later fixes.", "task_scope": "benchmark-level", "title": "Data issues identified in Gorilla leaderboard test dataset during data sanity checks", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/452", "summary": "eitanturok fixed BFCL Claude function-calling evaluation after an Anthropic SDK change made the eval impossible to run.", "task_scope": "benchmark-level", "title": "Fix breaking changes due to updated Anthropic SDK", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/356", "summary": "HuanzhiMao fixed a circular import introduced into the BFCL evaluation pipeline by earlier OSS-evaluation changes.", "task_scope": "benchmark-level", "title": "Fix Circular Import Issue for BFCL evluation pipeline", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/770", "summary": "BFCL Gemini handler sent an empty Tool object for examples with no function definitions, causing a 400 invalid GenerateContentRequest.", "task_scope": "benchmark-level", "title": "Fix handling of examples with no tools in Gemini", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/615", "summary": "BFCL PR fixed #614 by updating oss_handler.py and gorilla_handler.py import paths after restructuring.", "task_scope": "benchmark-level", "title": "Fix issue #614: [BFCL] ModuleNotFoundError after commit 70d6722", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/531", "summary": "PR fixes BFCL credential config output being squashed onto the second line because len(data) counted JSON object size, not source lines.", "task_scope": "benchmark-level", "title": "fix line return", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/829", "summary": "BFCL live_relevance changes from #826 were not properly incorporated after a merge conflict, requiring a data-file correction.", "task_scope": "benchmark-level", "title": "Fix Merge Conflict From #826", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1126", "summary": "BFCL Qwen handlers omitted reasoning content and dropped assistant tool-call messages, requiring recalculation of qwen3 scores.", "task_scope": "benchmark-level", "title": "Fix qwen3 inference data and qwenfc handler chat template", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/608", "summary": "BFCL PR fixes many prompt and possible-answer bugs: wrong ground truths, missing accepted answers, ambiguity, and parameter schema mismatch.", "task_scope": "benchmark-level", "title": "Fix some bugs in test case prompts/ground truths", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/423", "summary": "vandyxiaowei corrected BFCL parallel-function possible answers where optional/default parameters should still allow valid model outputs.", "task_scope": "benchmark-level", "title": "fix some data issues in parallel/parallel multiple answers", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/826", "summary": "BFCL live datasets contained enum values whose types did not match their parameter schemas; PR fixed those dataset errors.", "task_scope": "benchmark-level", "title": "fix some enum type errors in datasets", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1176", "summary": "BFCL TravelAPI.book_flight return type annotation was wrong compared with the actual booking_history dict return shape.", "task_scope": "benchmark-level", "title": "Fix type hint for TravelAPI.book_flight return type", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/335", "summary": "zuxin666 found BFCL dataset parameter-name errors in parallel_multiple_function while also patching evaluation-script logic.", "task_scope": "benchmark-level", "title": "Fix Typos in Evaluation Script and System Prompt. Identify Errors in a Dataset", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1364", "summary": "BFCL aggregate scoring treated deliberately never-evaluated categories as zero instead of excluding them from Overall Acc and related aggregates.", "task_scope": "benchmark-level", "title": "fix(bfcl): exclude never-evaluated categories from aggregate scores", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1290", "summary": "BFCL PR fixes evaluation failure when an LLM response contains Unicode that cannot be encoded to UTF-8.", "task_scope": "benchmark-level", "title": "fix: :bug: fix issue with unicode that can't be encoded to utf-8", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1291", "summary": "Live PR fixes BFCL local inference when model config is available locally instead of downloaded from HuggingFace.", "task_scope": "benchmark-level", "title": "fix: :bug: model path error for evaluating with locally deployed vllm", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/638", "summary": "BFCL PR fixed GLM prompt formatting that omitted tool descriptions/multiple-role support and produced abnormal scores.", "task_scope": "benchmark-level", "title": "fix: bug for glm prompt format", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1303", "summary": "Live PR fixes BFCL multiturn execution where colons in generated variable names made invalid Python identifiers and eval raised SyntaxError.", "task_scope": "benchmark-level", "title": "fix: substitute colons with underscores when executing multiturn function calls", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/879", "summary": "BFCL function documentation check found 1202 schema/default/enum formatting errors across function docs, with data-file fixes in progress.", "task_scope": "benchmark-level", "title": "Function documentation format fix", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/406", "summary": "vandyxiaowei fixed BFCL Gemini handling for parallel function calls in the model handler.", "task_scope": "benchmark-level", "title": "handle parallel function calls from gemini", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1362", "summary": "BFCL v4 irrelevance scoring treats decode failures as no function call, so malformed tool-call outputs can be credited as correct abstentions in published irrelevance results.", "task_scope": "benchmark-level", "title": "Irrelevance detection credits a tool call that failed to parse as a correct abstention", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1351", "summary": "The BFCL OpenAI-compatible function-calling parser treats message.tool_calls=[] as an empty tool-call response instead of falling back to message.content, so valid final textual responses can be discarded and memory subset scores collapse.", "task_scope": "benchmark-level", "title": "OpenAI-compatible BFCL handler drops final text when tool_calls is empty", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/768", "summary": "BFCL Gemini multi-turn FC mode could submit empty content and trigger a 400 invalid-role API error; PR skips empty turns.", "task_scope": "benchmark-level", "title": "Skip adding empty content from gemini", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/405", "summary": "aastroza found BFCL simple-category model answers that were being marked wrong despite appearing correct and submitted possible-answer corrections.", "task_scope": "benchmark-level", "title": "Small corrections to possible_answers for simple test category", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/617", "summary": "BFCL PR fixed Gemini handler decoding so natural language parts are ignored when function-call parts exist.", "task_scope": "benchmark-level", "title": "Update gemini_handler.py to better handle NL+FC model output", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/677", "summary": "Gorilla evaluation for Gemini failed to write many results because google-cloud-aiplatform returned non-JSON-serializable RepeatedComposite objects; dependency update fixed it.", "task_scope": "benchmark-level", "title": "Update google-cloud-aiplatform dependency", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/pull/1085", "summary": "Merged BFCL PR changes irrelevance_232 from a hotel request to a yoga-retreat request, fixing question wording in an irrelevance task.", "task_scope": "bfcl-irrelevance-232", "title": "Update irrelevance_232 question", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1217", "summary": "BFCL issue notes Mistral local prompt handler appends EOS to the prompt, which could stop generation; maintainer calls it an oversight and says it was fixed in #1266.", "task_scope": "benchmark-level", "title": "Why EOS token added at the end?", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bfcl", "evidence_label": "external_source_reported", "source_url": "https://github.com/ShishirPatil/gorilla/issues/1091", "summary": "underscore_to_dot 변환의 이름과 적용 방식이 어긋나 함수 이름 매칭 충돌을 만들 수 있다는 신고", "task_scope": "benchmark-level", "title": "함수 이름의 밑줄/점 변환이 불완전한 문제", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/42", "summary": "BigCodeBench issue reports lingering processes after evaluation completion for sanitized outputs.", "task_scope": "benchmark-level", "title": "[bug] Lingering processes after eval run completion", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/78", "summary": "BigCodeBench issue reports E2B evaluation crashing with tuple pass_k parsing and later empty Gradio eval output; maintainer updates the E2B image and clarifies expected output.", "task_scope": "benchmark-level", "title": "Evaluation broken in e2b", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/70", "summary": "BigCodeBench issue reports both Gradio and E2B evaluation paths broken; maintainer confirms Gradio service trouble and fixes E2B afterward.", "task_scope": "benchmark-level", "title": "Evaluation Pipelines broken: Gradio, E2B", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/79", "summary": "BigCodeBench maintainer PR fixes E2B execution/debug behavior related to reported evaluation pipeline failures.", "task_scope": "benchmark-level", "title": "fix E2b execution debug", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/112", "summary": "BigCodeBench evaluation broke with e2b SDK 2.0 interface changes; PR pins e2b 1.x so eval works again.", "task_scope": "benchmark-level", "title": "fix e2b sdk version", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/116", "summary": "BigCodeBench timeout env var was a string and needed float casting before math functions.", "task_scope": "benchmark-level", "title": "fix env variable type", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/75", "summary": "BigCodeBench PR fixes make_raw_chat_prompt when prefill is disabled, repairing task prompting for DeepSeek distilled/vLLM runs.", "task_scope": "benchmark-level", "title": "fix make_raw_chat_prompt when prefill is disabled", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/49", "summary": "BigCodeBench PR fixes tasks 211 and 215 by adding missing 200 OK mocks, allowing response.json(), fixing a description typo, and noting task 211 still has underspecified save-location behavior.", "task_scope": "bigcodebench_211", "title": "Fixes for tasks 211 and 215", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/119", "summary": "BigCodeBench PR handles None input in sanitize so generation/evaluation code does not error when LLMs fail to produce answers for some tasks.", "task_scope": "benchmark-level", "title": "Handle None input in sanitize function", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/76", "summary": "Live BigCodeBench issue reports frequent Matplotlib lock errors while evaluating ground truths locally in the official Docker image; maintainer asks reporter to submit a PR.", "task_scope": "benchmark-level", "title": "Matplotlib Lock error", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/103", "summary": "evaluate가 정수로 파싱된 pass_k를 반복 가능한 값처럼 다뤄 TypeError를 낸다는 신고", "task_scope": "benchmark-level", "title": "pass_k 정수 입력에서 발생하는 평가기 오류", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/77", "summary": "BigCodeBench PR sets a unique cache directory for each code execution to avoid libraries such as Matplotlib competing for the same files and causing evaluation errors.", "task_scope": "benchmark-level", "title": "Specify a unique cache directory before each code execution", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/pull/25", "summary": "BigCodeBench v0.1.8 update includes evaluation time-limit changes and generation/evaluate fixes after timeout/setup issues.", "task_scope": "benchmark-level", "title": "Update on v0.1.8", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/108", "summary": "Reporter asks where BigCodeBench/227 audio.wav is, indicating a task references a missing required asset.", "task_scope": "bigcodebench_227", "title": "where is the file: audio.wav?", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/111", "summary": "BigCodeBench task 120 expected date-series length 9 for Jan 1 through Jan 10, which the reporter flags as an unexpected/incorrect test case.", "task_scope": "bigcodebench_120", "title": "🐛 [TaskRemoval/TaskRepair] - <120> ", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bigcodebench-hard-complete", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/bigcodebench/issues/35", "summary": "BigCodeBench maintainer issue repairs task BigCodeBench/760 imports, changing datetime import behavior.", "task_scope": "bigcodebench_760", "title": "🐛 [TaskRepair] - 760 Imports", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bird-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/AlibabaResearch/DAMO-ConvAI/issues/128", "summary": "BIRD evaluation script referenced bird/prestore_dev_result.py/dev_result.bin, but reporter could not find the file and evaluation failed.", "task_scope": "benchmark-level", "title": "Could not Find Key File prestore_dev_result.py", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "bixbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/BixBench/pull/33", "summary": "Merged BixBench PR makes grading/evaluation tolerate missing evaluation_mode by using a default verifier mode.", "task_scope": "benchmark-level", "title": "Avoid eval mode if not present", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bixbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/BixBench/pull/44", "summary": "Live PR fixes BixBench range_verifier where out-of-range predictions raised UnboundLocalError instead of receiving incorrect grade.", "task_scope": "benchmark-level", "title": "Fix range verifier grading for out-of-range predictions", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bixbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/BixBench/pull/46", "summary": "Open PR fixes BixBench open-ended LLM graders so refused is parsed as a refusal instead of incorrect, restoring refusal metrics for both LLM grading paths.", "task_scope": "benchmark-level", "title": "fix(graders): record refused verdicts from the LLM graders", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "bixbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/BixBench/pull/32", "summary": "Merged PR fixes BixBench grading when openanswer rows omit evaluation_mode by defaulting to llm_verifier instead of raising during grade_outputs.", "task_scope": "benchmark-level", "title": "Make `llm-verifier` default for openanswer setting", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bixbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/BixBench/pull/23", "summary": "PR makes BixBench trajectory zip extraction recurse for Data/Notebook folders and fixes the config argument, preventing render/trajectory generation failures on different archive layouts.", "task_scope": "benchmark-level", "title": "Render zip extraction more resilient", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "bixbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/BixBench/pull/13", "summary": "PR fixes BixBench MCQ grading so refusal is detected as predicted == unsure and precision uses the non-refusal count; previous code inverted refusal handling.", "task_scope": "benchmark-level", "title": "Unit tests for grading", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "code-contests", "evidence_label": "external_source_reported", "source_url": "https://github.com/google-deepmind/code_contests/issues/31", "summary": "Python solutions in code_contests failed compilation under the sandbox because arch_prctl was blocked; commenters discussed allowing the syscall or using Python 3.10.", "task_scope": "benchmark-level", "title": "Executing and evaluating solutions 3x Compilation failure with many python versions", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/79", "summary": "CooperBench coop agents could receive false-success message delivery and stale peer-branch signals after a peer exited, causing coordination failures and merge-conflict double failures in evaluated pairs.", "task_scope": "benchmark-level", "title": "agent: coop agents could not tell a finished peer from a silent one", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/80", "summary": "CooperBench mini_swe_agent_v2 coop prompts told agents the shared remote was read-only and omitted push/PR workflow, preventing agents from sharing code during evaluated cooperative runs.", "task_scope": "benchmark-level", "title": "agent: coop prompt told agents the shared git remote was read-only", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/51", "summary": "CooperBench CLI adapters stripped patch.txt output before evaluation, removing trailing newline/context required by git apply and causing valid generated patches to be rejected.", "task_scope": "benchmark-level", "title": "agents/codex: add Codex adapter; lift shared coop bits into _coop", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/83", "summary": "CooperBench pair grading could lose an entire pair when one agent died because sandbox/runner timeouts and error handling treated a missing side as no gradeable pair.", "task_scope": "benchmark-level", "title": "coop: stop a pair dying with one of its agents", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/82", "summary": "Merged CooperBench dataset audit fixes 24 ungradeable features and preserves prebuilt image outputs, changing benchmark gradeability across the released dataset.", "task_scope": "benchmark-level", "title": "dataset: make all 199 features gradeable, and stop wiping prebuilt images", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/85", "summary": "CooperBench lacked the combined.patch invariant used by coop evaluation; a sweep found 19 features across five tasks whose tests failed against the merged gold tree, making affected pairs unwinnable.", "task_scope": "benchmark-level", "title": "dataset: verify every feature against combined.patch, on both architectures", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/38", "summary": "Live PR fixes CooperBench go_chi task56 combined.patch bugs that made oracle score 0 for feature pairs involving feature 1.", "task_scope": "cb-chi-t56-f1-5, cb-chi-t56-f1-3, cb-chi-t56-f1-2, cb-chi-t56-f1-4", "title": "Fix chi task56 combined.patch: Allow header duplication bug", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/43", "summary": "CooperBench datasets task 7309 pinned pyarrow 20, which changed GeoParquet string dtype behavior and broke all feature pairs on a pre-existing test; PR pins pyarrow below 20.", "task_scope": "cb-datasets-t7309-f1-2", "title": "Fix datasets task 7309: pin pyarrow < 20.0 for geoparquet compatibility", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/40", "summary": "Fixes CooperBench dspy task 8635 where combined.patch default min_instr_chars=30 made a pre-existing tests1 assertion fail, while tests2.patch already corrected it.", "task_scope": "cb-dspy-t8635-f1-3, cb-dspy-t8635-f1-2, cb-dspy-t8635-f1-4, cb-dspy-t8635-f1-6, cb-dspy-t8635-f1-5", "title": "Fix dspy task 8635: change min_instr_chars default from 30 to 0", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/41", "summary": "Live PR fixes CooperBench react-hook-form task153 where combined.patch API shape disagreed with tests and all onFinally tests failed.", "task_scope": "cb-react-hook-form-t153-f2-3, cb-react-hook-form-t153-f2-6, cb-react-hook-form-t153-f2-5, cb-react-hook-form-t153-f1-2, cb-react-hook-form-t153-f2-4", "title": "Fix react-hook-form task 153: handleSubmit 3rd arg type mismatch with test", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/10", "summary": "Live CooperBench PR fixes task Dockerfiles that did not run properly on Modal by adding Python and related compatibility changes.", "task_scope": "benchmark-level", "title": "Fix skip errored tasks", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/3", "summary": "CooperBench PR fixes the default agent name mismatch: runner defaults used mini_swe while the adapter registered mini_swe_agent, causing Unknown agent failures with defaults.", "task_scope": "benchmark-level", "title": "fix: use correct default agent name in runner.py", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/9", "summary": "CooperBench PR changes rerun logic so tasks with prior Error status are rerun instead of skipped, suggesting previous harness behavior could preserve false error outcomes.", "task_scope": "benchmark-level", "title": "Re-run tasks with Error status instead of skipping", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/66", "summary": "CooperBench Claude Code auto-proxy streamed malformed vLLM tool-call events through LiteLLM, making agent loops abort with Content block not found during evaluated runs.", "task_scope": "benchmark-level", "title": "release: v0.0.17", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/67", "summary": "CooperBench Claude Code proxy routing through LiteLLM could rewrite thinking requests to /v1/responses or ignore stream disabling, producing validation errors and adapter failures against vLLM endpoints.", "task_scope": "benchmark-level", "title": "release: v0.0.18", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/68", "summary": "CooperBench Claude Code trajectory parsing dropped every assistant turn when recent session JSONL stored the role only in the top-level event.type field.", "task_scope": "benchmark-level", "title": "release: v0.0.19", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/49", "summary": "CooperBench coop rollout crashed sorting mixed int/string timestamps before writing structured trajectories, causing downstream evals to see missing assistant turns.", "task_scope": "benchmark-level", "title": "runner/coop: coerce message timestamps to float before sorting", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/23", "summary": "CooperBench PR fixes evaluation console behavior including a pass-rate calculation that excluded Error runs from the denominator, creating misleading benchmark metrics.", "task_scope": "benchmark-level", "title": "SOme console output fixes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/53", "summary": "CooperBench team-mode prompts let members write diffs only to the scratchpad, so evaluation saw no submitted patch and scored otherwise completed team runs as zero.", "task_scope": "benchmark-level", "title": "team mode: fs mirror, typed protocol, MCP server, in-loop refresh", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "cooperbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/cooperbench/CooperBench/pull/55", "summary": "CooperBench team-mode and adapter plumbing had multiple benchmark-running blockers, including Modal stdin hangs, missing adapter dependencies, wrong backend dispatch, patch normalization, and misleading merge fallback policy.", "task_scope": "benchmark-level", "title": "team mode: wire team prompt + env into the three Python-loop adapters", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "crustbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/anirudhkhatry/CRUST-bench/issues/8", "summary": "CRUST-bench stats scripts failed because tree-sitter-c/tree-sitter-rust resources were absent or not submodules at required versions.", "task_scope": "benchmark-level", "title": "[get_stats] FileNotFoundError for `tree-sitter-c` & `tree-sitter-rust`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "crustbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/anirudhkhatry/CRUST-bench/issues/43", "summary": "Rust 테스트가 양방향 절댓값 오차를 사용해 C 테스트가 허용하는 낮은 false-positive 비율까지 거부한다는 신고", "task_scope": "crustbench-rhbloom", "title": "C와 Rust 오라클의 허용 범위 불일치", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "crustbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/anirudhkhatry/CRUST-bench/issues/27", "summary": "CRUST-bench unsafe Rust code finder exists but is never invoked; recursive wrapper returns nothing in the reported code path.", "task_scope": "benchmark-level", "title": "Function to find unsafe code is never invoked", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "crustbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/anirudhkhatry/CRUST-bench/pull/35", "summary": "CRUST-bench PR fixes prompt language for correctness and consistency across repair/transpilation prompts.", "task_scope": "benchmark-level", "title": "Minor fixes to prompt language for correctness and consistency", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "crustbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/anirudhkhatry/CRUST-bench/issues/42", "summary": "CRUST-bench report shows razz_simulation C tests seed srand48 but Rust tests do not, while expecting identical random outputs.", "task_scope": "crustbench-razz-simulation", "title": "razz_simulation: C and Rust tests are not equivalent", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/issues/30", "summary": "Across the 113-task DeepSWE set, 8 tasks' official solution.patch fails grading: 6 from unpinned dependency drift in the task Dockerfile (newer deps break unrelated base tests under filterwarnings=error; e.g. polars 1.40 in narwhals-rolling-window-suite, confirmed fixed by downgrading to 1.39.3) and 2 where the official solution deterministically fails the hidden new tests (mnamer-daemon-watch-lifecycle, helm-unified-manifest-stream).", "task_scope": "narwhals-rolling-window-suite, skrub-duration-encoding, dateutil-rfc5545-timezone-interop, langchain-request-coalescing, fd-deterministic-multi-key-sorting, igel-persist-feature-schema, mnamer-daemon-watch-lifecycle, helm-unified-manifest-stream", "title": "8 tasks whose official solution.patch fails grading", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/issues/53", "summary": "Three DeepSWE v1.1 tasks have incorrect language metadata in task.toml: Koota is labeled python, Prometheus is labeled typescript, and HTTPX is labeled typescript despite their upstream repositories being TypeScript, Go, and Python respectively.", "task_scope": "koota-entity-snapshot-rollback, prometheus-transactional-reload-status, httpx-deterministic-cookie-store", "title": "[Bug] 3 tasks with wrong language tags", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/issues/51", "summary": "DeepSWE task drizzle-orm-window-function-builders is unbuildable because its Dockerfile runs pnpm install --frozen-lockfile at a pinned drizzle-orm base commit whose lockfile requires unpublished drizzle-kit@0.25.0-b1faa33.", "task_scope": "drizzle-orm-window-function-builders", "title": "[Bug] `drizzle-orm-window-function-builders` is unbuildable: pnpm lockfile pins `drizzle-kit@0.25.0-b1faa33`, which returns 404 on the public npm registry", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/issues/45", "summary": "The DeepSWE task prometheus-transactional-reload-status whitelists 25 unrelated TestQueryLog P2P node ids in config.json. The task patches target transactional reload behavior, so these unrelated flaky query-log tests can make a correct solution score 0.", "task_scope": "prometheus-transactional-reload-status", "title": "[Bug] prometheus-transactional-reload-status: p2p_node_ids 包含不相关的 flaky TestQueryLog，导致误判 score=0", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/pull/60", "summary": "DeepSWE task claude-code-by-agents-recursive-delegation can hang the verifier when a submitted patch recurses synchronously; open PR #60 wraps Vitest suites in an OS-level timeout.", "task_scope": "claude-code-by-agents-recursive-delegation", "title": "claude-code-by-agents-recursive-delegation: OS-level watchdog so a hung patch fails instead of stalling the verifier", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/pull/44", "summary": "DeepSWE task configs used the older Harbor allow_internet schema; merged PR #44 migrates the suite to schema_version 1.3 with explicit agent and verifier network_mode settings.", "task_scope": "benchmark-level", "title": "feat: use harbor's network_mode", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/pull/78", "summary": "DeepSWE task quill-shared-toolbar-focus runs Vitest browser tests headed unless CI=true is set, so display-less substrates time out before running tests and can score the reference solution 0.", "task_scope": "quill-shared-toolbar-focus", "title": "fix(quill-shared-toolbar-focus): run Vitest browser tests headless (CI=true) so display-less substrates don't time out", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/pull/24", "summary": "The quill-shared-toolbar-focus grader asserts getAttribute('aria-disabled') is exactly the string 'false' after re-enabling the editor, rejecting the standards-compliant approach of removing the attribute (which yields null). A correct, accessibility-friendly solution fails the restore test.", "task_scope": "quill-shared-toolbar-focus", "title": "Grader rejects standards-compliant aria-disabled removal", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/issues/31", "summary": "DeepSWE rewards base exit 0 AND new exit 0, but 69/113 tasks collect base tests via directory wildcards (pytest tests/, go test ./..., cargo test) and the reset step restores only test.patch-touched files. A model that solves the task can still score 0 if it leaves any extra/broken test the wildcard collects -- a false negative. Proposal: ship FAIL_TO_PASS/PASS_TO_PASS lists and grade via log parsing (SWE-bench style).", "task_scope": "benchmark-level", "title": "Return-code grading causes false negatives", "upstream_status": "found" }, { "audit_label": "L1", "benchmark": "deep-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/datacurve-ai/deep-swe/issues/75", "summary": "과제에 쓰이지 않은 규칙 표시명을 정확히 맞춰야 해 실행된 테스트가 있어도 0점이 될 수 있다는 신고", "task_scope": "obsidian-linter-auto-table-of-contents", "title": "명시되지 않은 테스트 표시명에 결합된 점수", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "deveval", "evidence_label": "external_source_reported", "source_url": "https://github.com/seketeam/DevEval/issues/3", "summary": "DevEval setup failed with AttributeError: Distribution has no tests_require, and commenters asked for a Docker environment, suggesting reproducibility/dependency setup fragility.", "task_scope": "benchmark-level", "title": "AttributeError: 'Distribution' object has no attribute 'tests_require'. Did you mean: 'extras_require'?", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "ds-1000", "evidence_label": "external_source_reported", "source_url": "https://github.com/xlang-ai/DS-1000/issues/21", "summary": "DS-1000 check_correctness fails under Python 3.12 because unsafe_execute is a local function that cannot be pickled.", "task_scope": "benchmark-level", "title": "AttributeError: Can't pickle local object 'check_correctness..unsafe_execute'", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "evoeval", "evidence_label": "external_source_reported", "source_url": "https://github.com/evo-eval/evoeval/issues/2", "summary": "EvoEval issue #2 identifies incorrect expected outputs, ambiguous specs, and incorrect reference logic; maintainer confirmed five counted task defects.", "task_scope": "22, 23, 3, 32, 73", "title": "Potential test case bugs in the EvoEval difficult subset", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "featurebench-lite", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiberCoders/FeatureBench/issues/5", "summary": "FeatureBench lite interface descriptions omit or contradict APIs and concrete values that the gold tests require, leaving affected tasks underspecified relative to their tests.", "task_scope": "pydata__xarray.97f3a746.test_treenode.aa8ba777.lv2, mlflow__mlflow.93dab383.test_judge_tool_search_traces.cad8fc4b.lv1, huggingface__transformers.e2e8dbed.test_serve.4e7860c7.lv1", "title": "Interface Description defects in lite split — gold tests exercise APIs absent from the Problem Statement (L1 + L2)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "financeagent", "evidence_label": "external_source_reported", "source_url": "https://github.com/vals-ai/finance-agent/pull/47", "summary": "retrieve_information interpolates prompts with str.format(), so ordinary braces in prompts (e.g. JSON output schemas or dictionary examples) are treated as format fields and raise KeyError before the LLM call. The fix replaces this with direct {{key}} substitution.", "task_scope": "benchmark-level", "title": "str.format() mangles braces in retrieval prompts", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/pull/80", "summary": "FrontierSWE frogsgame-rl verifier failed scoring when Step 0 checkpoint audit-preservation download failed, even though scoring uses Tinker sampling from the checkpoint path.", "task_scope": "frogsgame-rl", "title": "add grok cli harness and minor fix on frogsgamerl task", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/issues/69", "summary": "Task revideo-perf-opt gives contradictory rules: the workspace starts at revideo v0.4.2 but the verifier compares output against v0.4.4 while forbidding /baseline access. Correct, untouched v0.4.2 output still scores 0 (3/8 correctness), and the correctness oracle is unknowable during the agent phase.", "task_scope": "revideo-perf-opt", "title": "Contradictory baseline-access rules and impossible correctness target", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/issues/82", "summary": "Frontier-SWE task dart-style-haskell tells agents to test against /app/reference/test/, but the shipped reference directory contains no test subdirectory.", "task_scope": "dart-style-haskell", "title": "dart-style-haskell: instruction.md references `/app/reference/test/` but this directory is not shipped in the environment", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/pull/81", "summary": "Frontier-SWE Grok adapter omitted token usage because headless Grok JSON lacks counts; the merged PR aggregates usage from logs/unified.jsonl into result and ATIF metrics.", "task_scope": "benchmark-level", "title": "fix: use logs/unified.jsonl for token usage", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/pull/72", "summary": "The postgres task's TAP tests could hang on deadlocks, blocking partial scores. Per-suite and per-test timeouts were added so hangs are killed and partial scores are recorded.", "task_scope": "postgres-sqlite-wire-adapter", "title": "Postgres-TAP tests could hang on deadlocks", "upstream_status": "fixed" }, { "audit_label": "L1", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/issues/65", "summary": "Frontier-SWE tasks expose verifier-only material in the agent sandbox, and the maintainer acknowledged this as a Harbor 0.4.0 limitation handled by post-rollout QA rather than benchmark isolation.", "task_scope": "benchmark-level", "title": "Reward-hacking surfaces: hidden verifier material is readable in agent sandbox", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/issues/73", "summary": "Task notebook-compression cannot be set up: instruction.md expects the data at $DATA_ROOT/visible, but that directory does not exist in the task's Docker environment.", "task_scope": "notebook-compression", "title": "Task missing setup data (broken Docker environment)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "frontier-swe", "evidence_label": "external_source_reported", "source_url": "https://github.com/Proximal-Labs/frontier-swe/pull/84", "summary": "Frontier-SWE Grok CLI config generation wrote vendor model options to the wrong TOML block in some model-id cases, so eval runs could miss the knobs actually used by the selected model.", "task_scope": "benchmark-level", "title": "update: fix grok cli list if else bug", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gpqa-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/idavidrein/gpqa/issues/28", "summary": "GPQA 5-shot evaluation appears not to filter few-shot examples out of evaluation examples, causing shot/test collisions.", "task_scope": "benchmark-level", "title": "Few shot examples are not filtered out of evaluation test examples", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/8", "summary": "GSO patch application could reverse or block intended diffs when build commands left uncommitted changes, so optimizations were not detected.", "task_scope": "benchmark-level", "title": "[BUG] `patch` command: any uncommitted build changes in docker can block patch application", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/10", "summary": "GSO load_gso_dataset dropped keys containing gt_, breaking dataset loading after gt_diff and gt_commit_message were added.", "task_scope": "benchmark-level", "title": "[BUG] load_gso_dataset drops keys w/ `gt_`; breaks after #2", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/22", "summary": "Setuptools 82 removed pkg_resources, breaking evaluation for old pandas instances during fresh reinstall and causing elicitation/evaluation issues.", "task_scope": "benchmark-level", "title": "[BUG] new setuptools version breaks evaluation for old pandas instances", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/5", "summary": "PyArrow 21 removed PyExtensionType, causing datasets-based GSO tasks such as huggingface__datasets-5994036 to error during evaluation.", "task_scope": "gso-huggingface--datasets-5994036", "title": "[BUG] PyArrow dependency failure for datasets tasks", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/30", "summary": "GSO issue reports agents commonly hit ModuleNotFoundError from importing packages from source roots, with a Docent trace example.", "task_scope": "benchmark-level", "title": "Agents commonly run into ModuleNotFoundError", "upstream_status": "found" }, { "audit_label": "L1", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/27", "summary": "Task gso-huggingface--tokenizers-fc76ad4 used a test fixture that wrote sentence entries containing internal newlines to mixed.txt, then read them back line-by-line, causing BASE and PATCH tokenizer training inputs to diverge and no-op/reference runs to fail equivalence.", "task_scope": "gso-huggingface--tokenizers-fc76ad4", "title": "BUG: huggingface__tokenizers-fc76ad4 Test Script Bug", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/pull/16", "summary": "PR fixes GSO grading failures from HuggingFace API rate limits by adding caching/retry changes in the harness.", "task_scope": "benchmark-level", "title": "enh: fixing rate limit issues using caching for HuggingFace API", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/31", "summary": "Task gso-uploadcare--pillow-simd-0514e20 cannot be graded: gso_test_11.py downloads an external image whose URL no longer returns a valid image (PIL.UnidentifiedImageError), so even the dataset reference patch fails before a score is produced.", "task_scope": "gso-uploadcare--pillow-simd-0514e20", "title": "External image URL no longer returns a valid image", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/pull/7", "summary": "GSO PR fixes missing pydantic dependency and HF token variable naming in harness environment code.", "task_scope": "benchmark-level", "title": "fix: missing pydantic dep + rename var for hf_token", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/pull/23", "summary": "GSO evaluation failed on recent systems because setuptools 82 removed pkg_resources and HuggingFace Hub Xet downloads errored; PR pins setuptools below 82 and disables Xet, recovering 20 instances.", "task_scope": "benchmark-level", "title": "fix: setuptools 82+ and HF Hub Xet compatibility", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/33", "summary": "Task gso-abetlen--llama-cpp-python-218d361: the dataset reference patch applies and installs, but fails the GSO functional-equivalence check with a deterministic completion mismatch on prompt 0 for an LLM-generation workload, so the oracle cannot pass its own equivalence gate.", "task_scope": "gso-abetlen--llama-cpp-python-218d361", "title": "Reference patch fails the functional-equivalence oracle", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/18", "summary": "User asked whether agents can see all GSO test scripts in official docker images; maintainer clarified only prob_script should be accessible and recommended flushing hidden tests if exposed.", "task_scope": "benchmark-level", "title": "Use of Test Scripts in the GSO Benchmark", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "gso", "evidence_label": "external_source_reported", "source_url": "https://github.com/gso-bench/gso/issues/32", "summary": "Task gso-uploadcare--pillow-simd-2818b90 cannot be graded: gso_test_10.py requests a Wikimedia thumbnail URL that now returns HTTP 400, so the dataset reference patch cannot be evaluated reproducibly.", "task_scope": "gso-uploadcare--pillow-simd-2818b90", "title": "Wikimedia thumbnail URL now returns HTTP 400", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "hil-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/hilbenchauthors/hil-bench/issues/4", "summary": "In create_mixed_mode_instances() (hil_bench/scripts/swe.py#L735) instances are duplicated per (model, mode) with a shallow instance.copy(); a nested extra_fields dict is shared by reference across expansions. The later in-place ps_extra.update(extra) / top_extra.update(extra) (#L777-L787) mutates that shared dict, so every expansion's _agent_config is overwritten on each model_configs iteration. ask_human mode silently loses its tool bundle, corrupting mixed-mode evaluation.", "task_scope": "benchmark-level", "title": "Shallow-copy aliasing shares _agent_config across (model, mode) expansions", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "humanevalfix", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/octopack/issues/31", "summary": "HumanEvalPack python/64 appears to include a leftover FIX comment in both prompt and declaration; maintainer invited a PR to correct it.", "task_scope": "python-64", "title": "Leftover comment in HumanEvalPack python/64", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "humanevalfix", "evidence_label": "external_source_reported", "source_url": "https://github.com/bigcode-project/octopack/pull/27", "summary": "PR removes a spurious double quotation mark from HumanEvalPack Python data/test suite, a concrete benchmark data/test defect.", "task_scope": "python-139", "title": "Remove spurious double quotation mark in the Python test suite", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "labbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/LAB-Bench/pull/2", "summary": "LAB-Bench PR corrected FigQA split sets and public data/image references.", "task_scope": "benchmark-level", "title": "FigQA Fix - JL", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "labbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/Future-House/LAB-Bench/pull/3", "summary": "LAB-Bench PR mirrored tag fixes from a private repo across public benchmark JSONL files.", "task_scope": "benchmark-level", "title": "Mirroring tag fixes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/153", "summary": "LiveCodeBench date-window and release-version CLI flags are honored only on the default code_generation_lite path, while not_fast, selfrepair, testoutputprediction, and codeexecution silently ignore them.", "task_scope": "benchmark-level", "title": "--not_fast silently ignores --start_date/--end_date and --release_version; other scenarios ignore the date flags too", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/48", "summary": "Code Execution dataset loading failed with ExpectedMoreSplitsError because the dataset expected a train split; maintainer said README was updated to fix the issue.", "task_scope": "benchmark-level", "title": "[Bug] Code Execution dataset loading error", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/127", "summary": "PR changes LiveCodeBench code extraction to accept correct model code even when no markdown backticks are present, avoiding false failures from formatting style.", "task_scope": "benchmark-level", "title": "accept output if no backticks as well", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/118", "summary": "LiveCodeBench extraction took the last pair of backticks, so a markdown summary after code could be extracted instead of the Python solution.", "task_scope": "benchmark-level", "title": "Added precise python backtick check", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/108", "summary": "Open issue reports LiveCodeBench code_generation loading breaks with datasets 4.0; linked PR #110 changes the dataset loading path and remains open.", "task_scope": "benchmark-level", "title": "datasets 4.0 trust_remote_code removal breaks LiveCodeBench loading", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/117", "summary": "LiveCodeBench default stop token `###` truncated model outputs such as Qwen solution code, harming result reproduction for default vLLM runs.", "task_scope": "benchmark-level", "title": "Default stop flag to None", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/146", "summary": "format_prompt_generation builds llama3 prompts via apply_chat_template(tokenize=False), which already emits <|begin_of_text|>; vLLM then tokenizes with add_special_tokens=True and prepends a second BOS, corrupting llama3 prompts during evaluation.", "task_scope": "benchmark-level", "title": "Double BOS token in llama3 prompts under vLLM", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/39", "summary": "PR fixes JSON/prompt issues in LiveCodeBench code-execution COT prompts for dracarys-llama.", "task_scope": "benchmark-level", "title": "fix for prompt in dracarys-llama", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/3", "summary": "PR fixes LiveCodeBench vLLM runner when not using cache: RequestOutput objects were not converted to text, so the code extractor could not parse outputs.", "task_scope": "benchmark-level", "title": "fix output in vllm_runner w/o use_cache", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/46", "summary": "PR fixes LiveCodeBench --use_cache handling in the runner, suggesting a harness/runtime behavior bug rather than model or dataset addition.", "task_scope": "benchmark-level", "title": "fix: `--use_cache` usage", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/pull/103", "summary": "Live LiveCodeBench PR reports code-generation evaluation can leave zombie processes and grow memory until a 1TB server idles, proposing PID kill and memory guard changes.", "task_scope": "benchmark-level", "title": "Fix: memory explosion issue", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/133", "summary": "LiveCodeBench exact Decimal equality for heavy-precision outputs causes many false negatives; reporter suggests tolerance-based comparison in grade_stdio/testing_util.", "task_scope": "benchmark-level", "title": "Heavy precision problem", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/106", "summary": "Reporter shows an AtCoder solution accepted by the official judge but marked wrong by LiveCodeBench because the problem allows multiple valid answers; maintainer added it to ERRATA.", "task_scope": "abc343_a", "title": "Incorrect Evaluation for Problems with Multiple Acceptable Answers", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/53", "summary": "Live issue shows LiveCodeBench installation fails on Python 3.11 because pyext uses removed inspect.getargspec; maintainer says a cleaner autograder release handles it.", "task_scope": "benchmark-level", "title": "Installation error with python 3.11", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/60", "summary": "LiveCodeBench Poetry install failed because package metadata was wrong; maintainer says pyproject.toml fixed it.", "task_scope": "benchmark-level", "title": "LiveCodeBench Installation Not Working", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/134", "summary": "LiveCodeBench release_v6 fails through datasets with a missing version_tag config key, linking to the active Hugging Face discussion.", "task_scope": "benchmark-level", "title": "Using livecodebench(release_v6) with the datasets lib causes an error! \"ValueError: BuilderConfig JsonConfigdoesn't have a 'version_tag' key.\"", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/131", "summary": "LiveCodeBench evaluation cannot find the requested cached code_generation_lite config when following README-style release_v5 setup.", "task_scope": "benchmark-level", "title": "ValueError: Couldn't find cache for livecodebench/code_generation_lite for config 'default-version_tag=release_latest' Available configs in the cache: ['release_latest-version_tag=release_v5']", "upstream_status": "found" }, { "audit_label": "L3", "benchmark": "livecodebench", "evidence_label": "external_source_reported", "source_url": "https://github.com/LiveCodeBench/LiveCodeBench/issues/155", "summary": "pyproject.toml의 패키지 탐색이 최상위만 포함해 일반 wheel 설치 뒤 하위 모듈 import가 깨진다는 신고", "task_scope": "benchmark-level", "title": "비 editable 설치에서 빠지는 하위 패키지", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "medagentbench", "evidence_label": "external_source_reported", "source_url": "https://github.com/stanfordmlgroup/MedAgentBench/issues/6", "summary": "MedAgentBench external evaluation reports 100 percent by prefetch-and-execute architecture versus much lower raw HTTP scores, suggesting a benchmark shortcut surface.", "task_scope": "benchmark-level", "title": "External evaluation: 300/300 (100%) achieved with prefetch-and-execute architecture", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "mlgym-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/facebookresearch/MLGym/pull/15", "summary": "MLGym PR fixes validation failing when no baseline is provided because the environment still tries to inspect a missing baseline score.", "task_scope": "benchmark-level", "title": "Fix list out of index error for cases without baseline", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "mlgym-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/facebookresearch/MLGym/issues/24", "summary": "MLGym-Bench tasks declare local dataset paths for 3SATTime and imageCaptioningCOCO, but the repository tree does not include those data directories.", "task_scope": "3SATTime, imageCaptioningCOCO", "title": "Local data missing for some tasks", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "mlgym-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/facebookresearch/MLGym/pull/12", "summary": "MLGym PR changes language-modeling evaluation to compute cross-entropy directly instead of trusting the model forward loss, preventing agents from changing evaluation code to cheat the score.", "task_scope": "benchmark-level", "title": "Use cross-entropy loss instead of model loss for evaluation", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "mmau", "evidence_label": "external_source_reported", "source_url": "https://github.com/Sakshi113/MMAU/issues/31", "summary": "MMAU issue identifies 349 test samples with duplicate answer choices plus questions that say select multiple despite being single-choice-style entries.", "task_scope": "benchmark-level", "title": "Duplicate choices and \"select multiple\" in question text", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "mmau", "evidence_label": "external_source_reported", "source_url": "https://github.com/Sakshi113/MMAU/pull/22", "summary": "Live PR fixes inconsistent category field types in MMAU test JSON that cause HuggingFace datasets Arrow conversion failures.", "task_scope": "benchmark-level", "title": "fix category attribute", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "openthoughts-tblite", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thoughts/OpenThoughts-TBLite/issues/2", "summary": "git-repo-forensics task appears to ask agents to uncover git artifacts, but Dockerfile/data folder do not include git metadata in /workdir/data.", "task_scope": "git-repo-forensics", "title": "Bug with git-repo-forensics", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "otel-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/QuesmaOrg/otel-bench/pull/12", "summary": "otel-bench PR adds missing packages to the rust-microservices Dockerfile, consistent with an environment dependency defect in the benchmark task.", "task_scope": "rust-microservices", "title": "Adding missing packages to Dockerfile", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "otel-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/QuesmaOrg/otel-bench/pull/8", "summary": "Merged PR changes java-advanced instruction and tests for a current Harbor task.", "task_scope": "java-advanced", "title": "Adding solution and minor fixes to java-advanced", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "otel-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/QuesmaOrg/otel-bench/pull/7", "summary": "Merged PR corrects the java-simple instruction from misleading custom-tracing wording to OpenTelemetry SDK wording.", "task_scope": "java-simple", "title": "Adding solution and minor fixes to java-simple", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "otel-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/QuesmaOrg/otel-bench/pull/13", "summary": "otel-bench PR adds a solution and disables test_db_logs for the erlang-microservices task, suggesting a task test/oracle was failing or unstable.", "task_scope": "erlang-microservices", "title": "Adding solution to erlang-microservices, disable test_db_logs for now", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "otel-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/QuesmaOrg/otel-bench/pull/16", "summary": "Merged PR fixes go-workflow-tracing test output expectations for a current Harbor task.", "task_scope": "go-workflow-tracing", "title": "Adding solution to go-workflow-tracing and minor fixes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "otel-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/QuesmaOrg/otel-bench/pull/17", "summary": "Merged PR fixes swift-microservices Docker/task setup for a current Harbor task.", "task_scope": "swift-microservices", "title": "Adding solution to swift-microservices and minor Dockerfile fixes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/pull/24", "summary": "tothemoon96 fixed PIXIU multi-turn evaluation request reformulation so prior answers are inserted without dropping request arguments.", "task_scope": "benchmark-level", "title": "bug fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/pull/17", "summary": "tothemoon96 fixed PIXIU automated task assessment setup by adding BARTScore, docker, and evaluation dependency changes.", "task_scope": "benchmark-level", "title": "fix bug", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/pull/18", "summary": "Jimin Huang fixed PIXIU F1 evaluation code that passed predictions and labels in the wrong order.", "task_scope": "benchmark-level", "title": "fix bugs", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/pull/88", "summary": "Internal PR fixes FLARE dataset paths from chancefocus to TheFinAI and resolves BARTScore checkpoint path handling, addressing evaluation errors.", "task_scope": "benchmark-level", "title": "fix errors", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/pull/21", "summary": "Jimin Huang fixed PIXIU NER evaluation by marking the task to evaluate the last turn.", "task_scope": "benchmark-level", "title": "fix NER bugs", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/pull/9", "summary": "Jimin Huang fixed PIXIU requirements and submodule wiring, including a transformers package typo and financial-evaluation path issue.", "task_scope": "benchmark-level", "title": "fix: modify readme and submodules", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/pull/35", "summary": "Jimin Huang fixed a PIXIU vLLM headlines evaluation bug where unstripped outputs were misclassified.", "task_scope": "benchmark-level", "title": "fix: vllm headlines evaluation bug", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/issues/87", "summary": "PIXIU/FLARE eval fails while loading flare_fpb with Hugging Face datasets due KeyError length in sequence feature metadata.", "task_scope": "benchmark-level", "title": "KeyError: 'length'", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "pixiu", "evidence_label": "external_source_reported", "source_url": "https://github.com/The-FinAI/PIXIU/issues/86", "summary": "PIXIU evaluation imports bart_score but the dependency is missing, causing ModuleNotFoundError for multiple users.", "task_scope": "benchmark-level", "title": "No module named 'bart_score'", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/15", "summary": "martinezmatias fixed QuixBugs correct_java_programs/SHUNTING_YARD because the correct version used the wrong queue method.", "task_scope": "quixbugs-java-shunting_yard", "title": "Bugfix on correct version of SHUNTING_YARD", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/41", "summary": "furunkel fixed the QuixBugs Java minimum_spanning_tree implementation after finding it mishandled node-set identity semantics.", "task_scope": "quixbugs-java-minimum_spanning_tree", "title": "Fix bug in Java version of minimum_spanning_tree", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/42", "summary": "furunkel fixed the QuixBugs Java shortest_path_lengths implementation to avoid treating INF as an ordinary weight.", "task_scope": "quixbugs-java-shortest_path_lengths", "title": "Fix bug in Java version of shortest_path_lengths", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/38", "summary": "h4iku fixed a QuixBugs Python detect_cycle test that could not run because of mixed tabs and spaces.", "task_scope": "quixbugs-python-detect_cycle", "title": "Fix TabError in detect_cycle_test", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/16", "summary": "SophieHYe fixed the QuixBugs buggy Java SQRT return type from float to double to match expected precision.", "task_scope": "quixbugs-java-sqrt", "title": "fix:modify return type of SQRT", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/4", "summary": "SophieHYe fixed QuixBugs Java compilation errors and added a JUnit generator.", "task_scope": "benchmark-level", "title": "Fixed java programs compilation errors and added Junit generator", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/13", "summary": "martinezmatias fixed an inconsistent QuixBugs pascal JSON testcase whose first expected output had the wrong list shape.", "task_scope": "quixbugs-java-pascal, quixbugs-python-pascal", "title": "Inconsistent output", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/issues/37", "summary": "QuixBugs tester.py hangs indefinitely on bitcount.py because the buggy implementation loops forever; maintainers later agreed silent failure should be handled with a timeout.", "task_scope": "quixbugs-python-bitcount", "title": "Infinite loop on bitcount not handled by tester.py", "upstream_status": "found" }, { "audit_label": "L3", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/11", "summary": "SophieHYe flagged a QuixBugs knapsack JSON expected-output error where correct Python and Java produced 1300 instead of 1652.", "task_scope": "quixbugs-java-knapsack, quixbugs-python-knapsack", "title": "Json test case error of knapsack", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "quixbugs", "evidence_label": "external_source_reported", "source_url": "https://github.com/jkoppel/QuixBugs/pull/14", "summary": "martinezmatias replaced a QuixBugs correct Java POWERSET implementation that failed on some inputs.", "task_scope": "quixbugs-java-powerset", "title": "new correct version of Powerset", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/485", "summary": "Merged PR fixes reasoning-gym basic_arithmetic division-by-zero task generation and adds tests.", "task_scope": "reasoning-gym-arithmetic-basic-arithmetic-hard0292e21f-seed45-0, reasoning-gym-arithmetic-basic-arithmetic-hard0292e21f-seed45-1, reasoning-gym-arithmetic-basic-arithmetic-hard0292e21f-seed45-2", "title": "[fix #484] resolve basic_arithmetic fails when size is large", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/128", "summary": "Reasoning Gym PR fixes evaluators and ambiguous questions across multiple tasks.", "task_scope": "benchmark-level", "title": "[Ongoing] Eval Template and Score Fixes", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/332", "summary": "CodeI/O PR explicitly fixes a numeric answer scoring bug while adding manually reviewed samples.", "task_scope": "reasoning-gym-code-codeio-hard0292e21f-seed45-0, reasoning-gym-code-codeio-hard0292e21f-seed45-1, reasoning-gym-code-codeio-hard0292e21f-seed45-2", "title": "Add a few new CodeI/O samples, resolve numeric answer scoring bug", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/80", "summary": "Reasoning-gym PR updates Tsumego with hints, legend, disallowed numeric answer, and expected string storage to make the task answer format scorable.", "task_scope": "reasoning-gym-games-tsumego-hard0292e21f-seed45-0, reasoning-gym-games-tsumego-hard0292e21f-seed45-1, reasoning-gym-games-tsumego-hard0292e21f-seed45-2", "title": "Add GO hints, legend, disallow numeric answer, store expected string …", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/212", "summary": "Fix graph_colour score_answer handling.", "task_scope": "reasoning-gym-algorithmic-graph-color-hard0292e21f-seed45-0, reasoning-gym-algorithmic-graph-color-hard0292e21f-seed45-1, reasoning-gym-algorithmic-graph-color-hard0292e21f-seed45-2", "title": "Add llama-3.3-70b-instruct algebra, algorithmic eval configs", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/106", "summary": "Reasoning-gym PR improves PolynomialMultiplication score_answer parsing and reviewer discussion flags a loophole where models could copy the question expression.", "task_scope": "benchmark-level", "title": "Better support for multivariate polynomials in PolynomialMultiplicationDataset", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/479", "summary": "Merged PR fixes both countdown scoring bugs from #478 and adds unit tests.", "task_scope": "reasoning-gym-games-countdown-hard0292e21f-seed45-0, reasoning-gym-games-countdown-hard0292e21f-seed45-1, reasoning-gym-games-countdown-hard0292e21f-seed45-2", "title": "corrected countdown issue", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/155", "summary": "Reasoning-gym PR adds explicit answer-format prompts and score_answer logic for Cryptarithm so partial/structured mappings can be evaluated instead of relying on brittle exact matching.", "task_scope": "benchmark-level", "title": "Cryptarithm add score_answer function", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/406", "summary": "Reasoning-gym PR fixes gsm_symbolic generator #27 floating-point rounding failures; basic_arithmetic links are excluded because this PR head still fails large-generation with a later ZeroDivisionError fixed separately by PR #485.", "task_scope": "reasoning-gym-arithmetic-gsm-symbolic-hard0292e21f-seed45-0, reasoning-gym-arithmetic-gsm-symbolic-hard0292e21f-seed45-1, reasoning-gym-arithmetic-gsm-symbolic-hard0292e21f-seed45-2", "title": "Data collisions notebooks and data", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/526", "summary": "Live PR adds a scoring cascade to Reasoning Gym to recover correct answers rejected by formatting, numeric, or symbolic-equivalence differences.", "task_scope": "benchmark-level", "title": "feat: add scoring cascade for reducing false negatives", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/50", "summary": "Reasoning-gym PR implements Tower of Hanoi answer solving/scoring, fixing the prior inability to score list-of-move answers and non-optimal valid solutions.", "task_scope": "reasoning-gym-games-tower-of-hanoi-hard0292e21f-seed45-0, reasoning-gym-games-tower-of-hanoi-hard0292e21f-seed45-1, reasoning-gym-games-tower-of-hanoi-hard0292e21f-seed45-2", "title": "feat: toh scoring", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/232", "summary": "Tsumego score_answer was simplified to compare against the oracle answer string instead of metadata solution tuple parsing, fixing scoring behavior.", "task_scope": "reasoning-gym-games-tsumego-hard0292e21f-seed45-0, reasoning-gym-games-tsumego-hard0292e21f-seed45-1, reasoning-gym-games-tsumego-hard0292e21f-seed45-2", "title": "Fix & simplify score_answer() of TsumegoDataset", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/444", "summary": "PR fixed prime_factorization normalize_answer so empty answers no longer crash/parse incorrectly during scoring.", "task_scope": "benchmark-level", "title": "Fix bug in normalize_answer method", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/464", "summary": "Merged PR fixes color_cubes answer strings by returning enum .value instead of the enum object.", "task_scope": "reasoning-gym-cognition-color-cube-rotation-hard0292e21f-seed45-0, reasoning-gym-cognition-color-cube-rotation-hard0292e21f-seed45-1, reasoning-gym-cognition-color-cube-rotation-hard0292e21f-seed45-2", "title": "fix color_cubes answer strings, update gallery with latest envs", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/219", "summary": "Color Cube Rotation scoring was broken according to o3 evals; PR fixes comparison to entry answer and adds scoring tests.", "task_scope": "reasoning-gym-cognition-color-cube-rotation-hard0292e21f-seed45-0, reasoning-gym-cognition-color-cube-rotation-hard0292e21f-seed45-1, reasoning-gym-cognition-color-cube-rotation-hard0292e21f-seed45-2", "title": "Fix Cube Rotation Scoring", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/330", "summary": "Dice answer checking was broken because removing an unused variable changed tuple destructuring into the wrong object; PR restores probability handling and adds metadata.", "task_scope": "reasoning-gym-arithmetic-dice-hard0292e21f-seed45-0, reasoning-gym-arithmetic-dice-hard0292e21f-seed45-1, reasoning-gym-arithmetic-dice-hard0292e21f-seed45-2", "title": "Fix Dice", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/383", "summary": "PR fixes figlet-font curriculum imports and interval metadata after the earlier version was not mergeable/working.", "task_scope": "reasoning-gym-cognition-figlet-font-hard0292e21f-seed45-0, reasoning-gym-cognition-figlet-font-hard0292e21f-seed45-1, reasoning-gym-cognition-figlet-font-hard0292e21f-seed45-2", "title": "fix figlet font curr imports", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/317", "summary": "Game-of-Life-Halting dataset lacked an __init__, breaking deterministic generation by seed; PR adds initialization and determinism tests.", "task_scope": "benchmark-level", "title": "Fix GoL-Halt Determinism", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/237", "summary": "Graph Color prompt example used invalid/unhelpful JSON-style keys; PR fixes the example template to quote vertex keys.", "task_scope": "reasoning-gym-algorithmic-graph-color-hard0292e21f-seed45-0, reasoning-gym-algorithmic-graph-color-hard0292e21f-seed45-1, reasoning-gym-algorithmic-graph-color-hard0292e21f-seed45-2", "title": "Fix graph color example template", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/524", "summary": "Fix knight_swap impossible_ratio generation.", "task_scope": "reasoning-gym-games-knight-swap-hard0292e21f-seed45-0, reasoning-gym-games-knight-swap-hard0292e21f-seed45-1, reasoning-gym-games-knight-swap-hard0292e21f-seed45-2", "title": "Fix impossible_ratio not being respected in knight_swap", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/523", "summary": "Fix shortest_path misleading instruction.", "task_scope": "reasoning-gym-graphs-shortest-path-hard0292e21f-seed45-0, reasoning-gym-graphs-shortest-path-hard0292e21f-seed45-1, reasoning-gym-graphs-shortest-path-hard0292e21f-seed45-2", "title": "Fix misleading instruction in shortest_path prompt", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/215", "summary": "Reasoning-gym PR fixes PoolMatrixConfigs.score_answer and adds unit tests, directly repairing a benchmark scorer.", "task_scope": "reasoning-gym-algorithmic-pool-matrix-hard0292e21f-seed45-1, reasoning-gym-algorithmic-pool-matrix-hard0292e21f-seed45-0, reasoning-gym-algorithmic-pool-matrix-hard0292e21f-seed45-2", "title": "Fix PoolMatrixConfigs::score_answer(), add unit tests", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/386", "summary": "PR removes a stale sat_utils qualifier from zebra clue generation, allowing the generator to use the full available clue set instead of failing on the old reference.", "task_scope": "reasoning-gym-logic-zebra-puzzles-hard0292e21f-seed45-0, reasoning-gym-logic-zebra-puzzles-hard0292e21f-seed45-1, reasoning-gym-logic-zebra-puzzles-hard0292e21f-seed45-2", "title": "Fix small bit of old code", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/161", "summary": "Reasoning-gym PR fixes 9x9 Sudoku generation to ensure unique puzzles and adds scoring; mini_sudoku is not attributed to this PR because it remains non-unique at this PR head and is covered by PR #159.", "task_scope": "reasoning-gym-games-sudoku-hard0292e21f-seed45-0, reasoning-gym-games-sudoku-hard0292e21f-seed45-1, reasoning-gym-games-sudoku-hard0292e21f-seed45-2", "title": "Fix Sudoku generator for uniqueness, implement scoring", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/135", "summary": "Reasoning-gym PR fixes Power Function scoring exceptions when model answers were not perfectly formatted.", "task_scope": "benchmark-level", "title": "fix(env): Power Function", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/94", "summary": "Live PR fixes Reasoning Gym prime factorization scoring behavior, but author is a project member.", "task_scope": "reasoning-gym-arithmetic-prime-factorization-hard0292e21f-seed45-0, reasoning-gym-arithmetic-prime-factorization-hard0292e21f-seed45-1, reasoning-gym-arithmetic-prime-factorization-hard0292e21f-seed45-2", "title": "fix(env): Prime Factorization scoring", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/514", "summary": "Reasoning Gym PR fixes countdown validation/generation failing for 10+ input numbers because SymPy symbol replacement created phantom numbers.", "task_scope": "reasoning-gym-games-countdown-hard0292e21f-seed45-0, reasoning-gym-games-countdown-hard0292e21f-seed45-1, reasoning-gym-games-countdown-hard0292e21f-seed45-2", "title": "Fix(reasoning_gym/games/countdown): Resolve SymPy parsing conflict for 10+ input numbers", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/517", "summary": "Live PR fixes cryptarithm scoring so any mapping satisfying puzzle constraints receives full credit instead of requiring the stored solution exactly.", "task_scope": "benchmark-level", "title": "Fix/cryptarithm multiple solutions", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/260", "summary": "Family_relationships was using the base score_answer, causing inaccurate scoring; PR implements a task-specific scorer.", "task_scope": "reasoning-gym-graphs-family-relationships-hard0292e21f-seed45-0, reasoning-gym-graphs-family-relationships-hard0292e21f-seed45-1, reasoning-gym-graphs-family-relationships-hard0292e21f-seed45-2", "title": "Fix/family relationship - minor", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/145", "summary": "Reasoning-gym PR fixes Tower of Hanoi, Countdown, and Maze question templates where unclear output instructions caused very low eval scores.", "task_scope": "benchmark-level", "title": "Fix/games", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/179", "summary": "Reasoning-gym PR says propositional_logic was broken and refactors prompt, parsing, trivial-solution checks, and scoring to accept all valid logic solutions.", "task_scope": "reasoning-gym-logic-propositional-logic-hard0292e21f-seed45-0, reasoning-gym-logic-propositional-logic-hard0292e21f-seed45-1, reasoning-gym-logic-propositional-logic-hard0292e21f-seed45-2", "title": "Fix/prop logix", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/380", "summary": "Adds score_answer() for number_sorting and tests, addressing missing scoring logic needed to evaluate dataset answers.", "task_scope": "reasoning-gym-algorithmic-number-sorting-hard0292e21f-seed45-0, reasoning-gym-algorithmic-number-sorting-hard0292e21f-seed45-1, reasoning-gym-algorithmic-number-sorting-hard0292e21f-seed45-2", "title": "fix: add score_answer() to number_sorting", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/537", "summary": "Reasoning Gym leg_counting generates ungrammatical animal plurals and its curriculum cannot select all 37 available animals.", "task_scope": "reasoning-gym-arithmetic-leg-counting-hard0292e21f-seed45-0, reasoning-gym-arithmetic-leg-counting-hard0292e21f-seed45-1, reasoning-gym-arithmetic-leg-counting-hard0292e21f-seed45-2", "title": "fix: minor leg counting bugs", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/254", "summary": "PR removes worked examples and explanatory solution steps from many task prompts, reducing prompt contamination and unintended hints.", "task_scope": "benchmark-level", "title": "fix: Unify Prompts", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/533", "summary": "reasoning-gym kakurasu and survo items used metadata.source_idx while the rest of the registry uses metadata.source_index, breaking source-index metadata consistency for the hard task rows.", "task_scope": "reasoning-gym-games-kakurasu-hard0292e21f-seed45-0, reasoning-gym-games-kakurasu-hard0292e21f-seed45-1, reasoning-gym-games-kakurasu-hard0292e21f-seed45-2, reasoning-gym-games-survo-hard0292e21f-seed45-0, reasoning-gym-games-survo-hard0292e21f-seed45-1, reasoning-gym-games-survo-hard0292e21f-seed45-2", "title": "fix: use source_index metadata key in kakurasu and survo", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/265", "summary": "Countdown score_answer accepted expressions using numbers not specified in the question; PR requires all listed numbers and validates number usage.", "task_scope": "benchmark-level", "title": "Fixed `countdown` `score_answer`", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/251", "summary": "PR fixes #245 by adding depth_constraint support for knights_knaves, removing problematic variables, and adding unit tests.", "task_scope": "reasoning-gym-logic-knights-knaves-hard0292e21f-seed45-1, reasoning-gym-logic-knights-knaves-hard0292e21f-seed45-0, reasoning-gym-logic-knights-knaves-hard0292e21f-seed45-2", "title": "fixed problems in knights_knaves", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/258", "summary": "Game of Life prompt failed to specify Moore neighborhood and wrapping topology, forcing models to guess; PR clarifies rules and adds partial grid scoring.", "task_scope": "benchmark-level", "title": "Game of Life partial scoring and rule-clarification", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/339", "summary": "Advanced geometry scoring compared rational orthocenter coordinates as strings; PR converted parsed answer and expected coordinates to floats.", "task_scope": "reasoning-gym-geometry-advanced-geometry-hard0292e21f-seed45-0, reasoning-gym-geometry-advanced-geometry-hard0292e21f-seed45-1, reasoning-gym-geometry-advanced-geometry-hard0292e21f-seed45-2", "title": "Minor fix to advanced geometry", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/261", "summary": "PR applies broad question-template and score_answer improvements across many datasets, including parser tolerance for complex arithmetic and integration formatting instructions.", "task_scope": "benchmark-level", "title": "Minor question template & score_answer improvements", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/250", "summary": "PR adjusts core scoring/extraction so numeric answers tolerate signs, leading zeros, trailing decimals, and commas instead of false mismatches.", "task_scope": "benchmark-level", "title": "Remove strip from ProceduralDataset::core score_answer()", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/159", "summary": "Reasoning-gym PR resolves mini Sudoku generation of non-unique puzzles, choosing unique generation over accepting multiple possible solutions in scoring.", "task_scope": "reasoning-gym-games-mini-sudoku-hard0292e21f-seed45-0, reasoning-gym-games-mini-sudoku-hard0292e21f-seed45-1, reasoning-gym-games-mini-sudoku-hard0292e21f-seed45-2", "title": "Resolve mini sudoku generator outputting non-unique puzzles", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/252", "summary": "Word-sequence-reversal prompt omitted the required comma-separated format, causing an objectively correct answer without commas to score 0.01; PR adds explicit format instructions.", "task_scope": "reasoning-gym-algorithmic-word-sequence-reversal-hard0292e21f-seed45-0, reasoning-gym-algorithmic-word-sequence-reversal-hard0292e21f-seed45-1, reasoning-gym-algorithmic-word-sequence-reversal-hard0292e21f-seed45-2", "title": "small change to word sequence reversal prompt", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/pull/269", "summary": "PR fixes multiple eval-sheet issues: pool_matrix lacked precision instructions, gsm_symbolic answers sometimes included units, and power_function scoring was too lenient for tiny values and too harsh for large ones.", "task_scope": "benchmark-level", "title": "updated algorithmics dataset", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "reasoning-gym-hard", "evidence_label": "external_source_reported", "source_url": "https://github.com/open-thought/reasoning-gym/issues/536", "summary": "Reasoning Gym syllogism generation marks undistributed-middle forms as valid, causing hard syllogism task rows to reward invalid conclusions as correct.", "task_scope": "reasoning-gym-logic-syllogism-hard0292e21f-seed45-0, reasoning-gym-logic-syllogism-hard0292e21f-seed45-1, reasoning-gym-logic-syllogism-hard0292e21f-seed45-2", "title": "Wrong answer in Syllogism", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-qna", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/issues/10", "summary": "SWE-Atlas QnA task task-6905333b74f22949d97baa22 marked two desirable rubric behaviors as negative, reversing judge polarity until merged PR #12 rewrote them as actual negative behaviors.", "task_scope": "task-6905333b74f22949d97baa22", "title": "[swe-atlas-qna] Inverted negative rubric polarity in task-6905333b74f22949d97baa22", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-qna", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/7", "summary": "SWE-Atlas QnA Modal run configs lacked the keepalive environment kwarg needed for containers to stay alive under the latest Harbor runtime; merged PR #7 adds it to all three QnA run scripts.", "task_scope": "benchmark-level", "title": "Add QnA Modal keepalive run flag", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-qna", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/6", "summary": "Eight SWE-Atlas Q&A gold answer files failed their rubric verifiers until merged PR #6 added the missing facts, runtime evidence, or corrected names needed for each answer to pass.", "task_scope": "task-6905333b74f22949d97ba9d8, task-6905333b74f22949d97ba9f1, task-6905333b74f22949d97ba9f4, task-6905333b74f22949d97ba9f5, task-6905333b74f22949d97baa0b, task-6905333b74f22949d97baa15, task-6905333b74f22949d97baa17, task-6905333b74f22949d97baa19", "title": "data/qa: fix 8 gold answers so each passes its own verifier", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-qna", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/24", "summary": "SWE-Atlas QnA judge evaluators used brittle JSON extraction and strict parsing, so fenced replies or literal control characters could make valid judge ratings fail to parse until open PR #24 replaces the parser.", "task_scope": "benchmark-level", "title": "qa/tw: make the judge JSON parsers tolerant of fences, control characters, and trailing text", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-qna", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/13", "summary": "SWE-Atlas QnA oracle run config defaulted HARBOR_BIN to a maintainer-specific absolute path, making the oracle script non-portable until merged PR #13 switched it to PATH-resolved harbor.", "task_scope": "benchmark-level", "title": "Remove hardcoded Harbor paths from oracle scripts", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-tw", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/8", "summary": "SWE-Atlas TW Modal run configs lacked the keepalive environment kwarg needed for containers to stay alive under the latest Harbor runtime; merged PR #8 adds it to all three TW run scripts.", "task_scope": "benchmark-level", "title": "Add TW Modal keepalive run flag", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-tw", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/14", "summary": "Thirty-eight SWE-Atlas TW Go tasks needed their pinned Go module graphs prewarmed at image build time so closed-runtime agent/verifier execution could resolve dependencies offline instead of failing under GOPROXY=off.", "task_scope": "task-6902ef3ab97fe23e2ad271f9, task-6902ef3ab97fe23e2ad27202, task-6902ef3ab97fe23e2ad27207, task-6902ef3ab97fe23e2ad27209, task-6902ef3ab97fe23e2ad27211, task-6902ef3ab97fe23e2ad27214, task-6902ef3ab97fe23e2ad27223, task-6902ef3ab97fe23e2ad2722e, task-6902ef3ab97fe23e2ad2723f, task-6902ef3ab97fe23e2ad27245, task-6902ef3ab97fe23e2ad27247, task-6902ef3ab97fe23e2ad27248, task-6902ef3ab97fe23e2ad2724b, task-6902ef3ab97fe23e2ad2724e, task-6902ef3ab97fe23e2ad27256, task-6902ef3ab97fe23e2ad2725c, task-6902ef3ab97fe23e2ad2725e, task-6902ef3ab97fe23e2ad27260, task-6902ef3ab97fe23e2ad27262, task-6902ef3ab97fe23e2ad27264, task-6902ef3ab97fe23e2ad27265, task-6902ef3ab97fe23e2ad27266, task-6902ef3ab97fe23e2ad27267, task-6902ef3ab97fe23e2ad27268, task-6902ef3ab97fe23e2ad2726a, task-6902ef3ab97fe23e2ad2726d, task-6902ef3ab97fe23e2ad27273, task-6902ef3ab97fe23e2ad27274, task-6902ef3ab97fe23e2ad27275, task-6902ef3ab97fe23e2ad27276, task-6902ef3ab97fe23e2ad27277, task-6902ef3ab97fe23e2ad27279, task-6902ef3ab97fe23e2ad2727a, task-6902ef3ab97fe23e2ad2727c, task-6902ef3ab97fe23e2ad2727d, task-6902ef3ab97fe23e2ad27280, task-6902ef3ab97fe23e2ad27281, task-6902ef3ab97fe23e2ad27283", "title": "Prewarm Go dependencies for TW tasks", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-tw", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/11", "summary": "SWE-Atlas Test Writing tasks lacked agent-phase network allowlists to prevent solution lookup; merged PR #11 adds network_mode allowlist settings to TW task files and passes the allowed agent host in TW run configs.", "task_scope": "benchmark-level", "title": "Restrict TW agent network access", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-tw", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/issues/4", "summary": "SWE-Atlas TW post-hoc manifests include non-test files, wrong paths/test names, mutation-surviving tests, or vague descriptions, causing verifier reward 0 on affected oracle tasks.", "task_scope": "task-6902ef3ab97fe23e2ad27209, task-6902ef3ab97fe23e2ad2720c, task-6902ef3ab97fe23e2ad27215, task-6902ef3ab97fe23e2ad27224, task-6902ef3ab97fe23e2ad2723c, task-6902ef3ab97fe23e2ad27242, task-6902ef3ab97fe23e2ad27243, task-6902ef3ab97fe23e2ad27275, task-6902ef3ab97fe23e2ad2721c, task-6902ef3ab97fe23e2ad2727e, task-6902ef3ab97fe23e2ad27229", "title": "SWE-Atlas data/image defects (oracle solutions fail their own verifier", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "scale-ai__swe-atlas-tw", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-Atlas/pull/20", "summary": "Ten SWE-Atlas TW simple-login task images lacked uv and Python 3.11 even though their verifier resolves Python from the uv tool directory, preventing offline verification.", "task_scope": "benchmark-level", "title": "TW: provision uv and Python 3.11 in the simple-login task images", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "seta-env", "evidence_label": "external_source_reported", "source_url": "https://github.com/camel-ai/seta-env/issues/10", "summary": "SETA task 999 Dockerfiles copy mock_dmesg.log and mock_journalctl.log fixtures that are absent from the task package, so the task image cannot build.", "task_scope": "999", "title": "Task 999 is missing mock_dmesg.log and mock_journalctl.log required by its Dockerfiles", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "simpleqa", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/simple-evals/pull/66", "summary": "simple-evals PR fixes a Python 3.11+ import collision where a local types.py shadows the stdlib types module and prevents the eval code from running.", "task_scope": "benchmark-level", "title": "fix: import collision for types", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "simpleqa", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/simple-evals/pull/118", "summary": "SimpleQA grader parsing scans for any A/B/C character, so named labels such as INCORRECT, CORRECT, and NOT_ATTEMPTED can be mapped to the wrong grade.", "task_scope": "benchmark-level", "title": "Make the grader parser robust", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "simpleqa", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/simple-evals/issues/51", "summary": "Live issue reports a simple-evals grader prompt inconsistency where predicted 1.75 should not match gold 1.73 to the required significant figure.", "task_scope": "benchmark-level", "title": "Mistake in grader prompt", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "simpleqa", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/simple-evals/issues/58", "summary": "Live issue reports two incorrect answers in OpenAI simple-evals with full metadata/problem payloads.", "task_scope": "simpleqa-1244", "title": "Two incorrect answers", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "simpleqa", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/simple-evals/pull/26", "summary": "simple-evals SimpleQA data loading used blobfile/Azure access; PR switches to a direct public CSV URL to avoid auth friction.", "task_scope": "benchmark-level", "title": "Update SimpleQAEval to use direct CSV URL instead of blobfile", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "simpleqa", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/simple-evals/issues/25", "summary": "Live issue reports the az:// SimpleQA path produces ResourceNotFound/access failure and provides a working public HTTPS URL instead.", "task_scope": "benchmark-level", "title": "What is the recommended way to access the simpleQA dataset?", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "spider2-dbt", "evidence_label": "external_source_reported", "source_url": "https://github.com/xlang-ai/Spider2/issues/156", "summary": "Spider2 still has missing gold truth for dbt questions, plus an orphan gold folder and a broken conditional_tabs case.", "task_scope": "airbnb002", "title": "Missing gold truth for dbt questions", "upstream_status": "found" }, { "audit_label": "L3", "benchmark": "spider2-dbt", "evidence_label": "external_source_reported", "source_url": "https://github.com/xlang-ai/Spider2/issues/200", "summary": "Spider2-DBT duckdb_match accepts row-scrambled tables when ignore_order=true because it compares sorted columns independently rather than row tuples.", "task_scope": "benchmark-level", "title": "spider2-dbt: duckdb_match with ignore_order=true accepts row-scrambled tables (cross-column alignment never checked)", "upstream_status": "found" }, { "audit_label": "L3", "benchmark": "spider2-dbt", "evidence_label": "external_source_reported", "source_url": "https://github.com/xlang-ai/Spider2/issues/201", "summary": "Spider2-DBT evaluate.py averages only over submitted instance ids, so partial submissions omit missing tasks from the denominator.", "task_scope": "benchmark-level", "title": "spider2-dbt: evaluate.py averages over submitted instances only; published scores mix /64 and /68 denominators", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/33", "summary": "SpreadsheetBench evaluator splits answer_position on commas, which breaks legal Excel sheet names containing commas.", "task_scope": "130-9", "title": "Bug: answer_position.split(',') breaks evaluation for sheet names containing commas", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/31", "summary": "SpreadsheetBench evaluator reads cached Excel formula values, so correct formulas written by openpyxl/pandas can be marked wrong when workbooks have not been recalculated.", "task_scope": "benchmark-level", "title": "Evaluation bug: formulas written by openpyxl/pandas aren’t calculated, causing false negatives", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/34", "summary": "SpreadsheetBench task 118-50 scores C2:D5000 even though the golden answer starts at C1:D1, dropping the first result pair from evaluation.", "task_scope": "118-50", "title": "Incorrect answer position for 118-50 (drops the first result row)", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/24", "summary": "Live issue reports SpreadsheetBench item 263-1 answer position is incorrect.", "task_scope": "263-1", "title": "Incorrect answer position for 263-1", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/35", "summary": "SpreadsheetBench task 12864 asks for dates and Column C data, but the workbook/golden only produce Sheet2 column B deal-name values and Sheet2 has no Column C.", "task_scope": "12864", "title": "Incorrect question/answer for 12864 (prompt asks for dates + column C, golden has neither)", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/6", "summary": "SpreadsheetBench requirements pinned numpy too old to be compatible with the pinned pandas version.", "task_scope": "benchmark-level", "title": "Issues in `requirements.txt`", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/36", "summary": "SpreadsheetBench Verified tasks 60-7, 283-32, and 41-47 contain malformed answer_sheet or answer_position metadata that makes the scorer target invalid sheet/range strings.", "task_scope": "60-7, 283-32, 41-47", "title": "Malformed answer_position / answer_sheet in tasks 41-47, 60-7, 283-32", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/17", "summary": "SpreadsheetBench 912 dataset contains xlsx files that Excel/open_spreadsheet cannot open without recovery.", "task_scope": "31746", "title": "more broken files in 912 dataset", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/16", "summary": "SpreadsheetBench 912 dataset has additional filename mistakes such as spaces before .xlsx and wrong separators.", "task_scope": "59129", "title": "more incorrect filenames in 912 dataset", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/37", "summary": "SpreadsheetBench task 13284 golden fills missing Streets End values before lookup, while the init leaves them blank and the instruction says to only use populated Start/End rows.", "task_scope": "13284", "title": "Task 13284: init/golden mismatch (Streets End filled with 9999) + instruction contradicts golden", "upstream_status": "found" }, { "audit_label": "L3", "benchmark": "spreadsheetbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/RUCKBReasoning/SpreadsheetBench/issues/38", "summary": "평가가 생성된 workbook 대신 원본 입력을 가리키고 inference와 evaluation의 JSONL 경로도 달라 결과가 평가되지 않을 수 있다는 신고", "task_scope": "benchmark-level", "title": "생성 결과를 읽지 않는 공식 평가 파이프라인", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "strongreject", "evidence_label": "external_source_reported", "source_url": "https://github.com/dsbowen/strong_reject/issues/19", "summary": "StrongREJECT rubric fails when all models fail because missing judge-output columns are not filled before dataset creation.", "task_scope": "benchmark-level", "title": "StrongReject Rubric fails when all models fail", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/96", "summary": "Frontier Evals PR fixes stricter NetworkMode.NONE fallout: SWELancer must run UNPROXIED due to side images and Alcatraz should fail loudly when unsupported.", "task_scope": "benchmark-level", "title": "[alctz/swelancer] Minor fixes following up the stricter alctz NetworkMode.NONE", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/58", "summary": "Frontier Evals PR fixes SWELancer task 42153-manager-0 where a CSV row was broken over two lines.", "task_scope": "42153-manager-0", "title": "[swel] fix 42153-manager-0 being broken over two lines", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/68", "summary": "SWEL IC_SWE on the monolith showed near-zero performance despite valid rollouts because environment variables in the image broke Expensify flows; PR removed them.", "task_scope": "benchmark-level", "title": "[SWEL] minor fix necessary for monolith image", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/69", "summary": "Frontier Evals PR removes a RunnerArgs instance because recommended args were not being picked up by chz, reducing confusion and fixing documented run arguments.", "task_scope": "benchmark-level", "title": "[swel] Remove RunnerArgs instance and update readme/tests", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/issues/131", "summary": "SWE-Lancer computer-task rollouts could use one fewer action than configured because the shared LimitsHelper raised when the action count equaled max_actions.", "task_scope": "benchmark-level", "title": "Computer-task action limit rejects the final allowed action", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/175", "summary": "Frontier Evals Alcatraz returns host-port leases before container removal finishes, allowing another container to reuse a still-bound port.", "task_scope": "benchmark-level", "title": "Fix Alcatraz host port release order during container cleanup", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/170", "summary": "SWE-Lancer default run output path was built from the module name string, producing a relative swelancer.utils/runs path instead of the project runs directory.", "task_scope": "benchmark-level", "title": "Fix SWE-Lancer default runs directory", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/57", "summary": "Frontier Evals PR fixes SWELancer manager tasks by requiring the monolith image and correcting two task issues.", "task_scope": "benchmark-level", "title": "manager tasks fixes and cleanup", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/pull/62", "summary": "PR changes internet-disabling logic so per-container network blocks do not conflict across containers.", "task_scope": "benchmark-level", "title": "modified disable internet function, allows for container specific blockers", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/issues/100", "summary": "Frontier Evals issue shows SWELancer sample 16912_4 instructions allow either consistent “format” or “e.g.” wording, but tests only accept changing bank account page to “e.g.”.", "task_scope": "16912-manager-0", "title": "Overly-strict test in swelancer sample 16912_4", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/issues/151", "summary": "pytest 종료 코드가 모두 없을 때 시스템/rollout 오류를 일반 모델 실패로 점수화한다는 신고", "task_scope": "benchmark-level", "title": "pytest 종료 코드 누락을 모델 실패로 처리하는 문제", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/issues/139", "summary": "SWE-Lancer aggregate token totals start at one in each category, adding a phantom input, output, and reasoning token to every run summary.", "task_scope": "benchmark-level", "title": "SWE-Lancer token totals include one phantom token per category", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swe-lancer-diamond", "evidence_label": "external_source_reported", "source_url": "https://github.com/openai/frontier-evals/issues/99", "summary": "SWELancer user_tool.py files call context.tracing.export(), a Playwright API absent in pinned Playwright 1.47.0; correct call is tracing.stop(path=...).", "task_scope": "benchmark-level", "title": "swelancer `user_tool` files call non-existent playwright function", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/274", "summary": "The excel-clone CUA persistence criterion could PARTIAL valid oracle runs when the judge used same-origin URL navigation instead of an explicit hard reload or omitted a blank pre-write screenshot.", "task_scope": "excel-clone", "title": "Accept URL reload for excel-clone persistence CUA", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/271", "summary": "The excel-clone wide-fanout performance gate used a 3 second budget that could fail valid oracle runs on Modal single-vCPU timing even when UX and other correctness checks passed.", "task_scope": "excel-clone", "title": "Bump excel-clone fanout performance budget", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/246", "summary": "Merged SWE-Marathon PR converted non-CUA long-horizon tasks from open internet to closed or dependency-allowlisted execution, removing a fairness leak where agents could use public internet during task runs.", "task_scope": "biofabric-rust-rewrite, embedding-eval, find-network-alignments, jax-pytorch-rewrite, nextjs-vite-rewrite, ruby-rust-port, rust-c-compiler, rust-java-lsp, vliw-kernel-optimization, wasm-simd, zstd-decoder", "title": "Convert long-horizon tasks to closed internet (keep 4 CUA open)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/269", "summary": "The excel-clone oracle could fail the CUA realism rubric because reference grid headers were only about 4 percentage points darker than cells while the rubric required at least 10.", "task_scope": "excel-clone", "title": "Darken excel-clone oracle headers to satisfy CUA realism threshold", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/281", "summary": "SWE-Marathon public v1.0/v1.1 GitHub tags were applied to commits that did not match the official leaderboard versions, making official score reproduction ambiguous until the short tags were deleted and v1.0-official/v1.1-official pins were published.", "task_scope": "benchmark-level", "title": "Document leaderboard version pins and the swapped GitHub tags", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/264", "summary": "Rust-java-lsp image builds failed after Docker interpreted an inline COPY comment as extra source paths; merged PR #264 stages the corpus and copies it without invalid Docker syntax.", "task_scope": "rust-java-lsp", "title": "Fix Java LSP golden corpus Docker copy", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/272", "summary": "The excel-clone oracle could fail a CUA realism clause because active and inactive sheet tabs were visually too similar in the reference implementation.", "task_scope": "excel-clone", "title": "Harden excel-clone active sheet-tab styling", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/275", "summary": "The excel-clone CUA sheet_tabs criterion could PARTIAL valid oracle runs unless the judge used one narrow distinct-content proof for independent per-sheet storage.", "task_scope": "excel-clone", "title": "Harden excel-clone sheet-tabs CUA storage proof", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/194", "summary": "The mastodon-clone task had a soft-gated CUA verifier plus oracle and contract gaps, including venv PATH failures, invalid CSS comments, Alpine bindings blocked by strict CSP, missing Mastodon UX routes, and undocumented vendored asset paths that made valid runs fail before meaningful CUA grading.", "task_scope": "mastodon-clone", "title": "Harden mastodon-clone CUA verifier and oracle contract", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/241", "summary": "Four SWE-Marathon CUA clone tasks depended on fork-only multi-stage type=cua verifier plumbing; merged PR #241 inlines an upstream Harbor-compatible shell verifier using Computer1.", "task_scope": "slack-clone, excel-clone, s3-clone, mastodon-clone", "title": "Inline, upstream-compatible CUA verifier for the 4 clone tasks", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/256", "summary": "Four SWE-Marathon CUA clone wrappers wrote fractional blended rewards into reward.json; merged PR #256 makes trial rewards binary and moves the blend to metrics.json.partial_score.", "task_scope": "slack-clone, excel-clone, s3-clone, mastodon-clone", "title": "Make CUA clone rewards binary", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/250", "summary": "Task rust-java-lsp still used unrestricted internet; open PR #250 documents restricted networking and replaces allow_internet=true with a minimal crate/GitHub allowlist.", "task_scope": "rust-java-lsp", "title": "Make rust-java-lsp closed internet", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/issues/248", "summary": "Task post-train-ifeval still depends on Tinker-hosted meta-llama/Llama-3.2-1B; the maintainer acknowledged the report and plans to rewrite the task to use a single H100 instead of Tinker.", "task_scope": "post-train-ifeval", "title": "post-train-ifeval depends on decommissioned Llama-3.2-1B Tinker model", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/261", "summary": "Ruby-rust-port oracle verification could flake from in-process SQLite writer contention in the Ruby reference, causing valid oracle runs to lose a gate until merged PR #261 shared DB pools and added busy handling.", "task_scope": "ruby-rust-port", "title": "Prevent Ruby port oracle SQLite lock flakes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/issues/266", "summary": "Published SWE-Marathon trials for four task families include agent self-termination failures where broad pkill patterns matched the Claude Code process argv, making valid or partially valid runs appear as model failures.", "task_scope": "nextjs-vite-rewrite, find-network-alignments, parameter-golf, embedding-eval", "title": "Published results include multiple agent self-termination failures", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/270", "summary": "The excel-clone CUA dependent_recompute criterion could mark valid oracle runs PARTIAL when the judge proved live recompute but skipped the hard-reload/bash portion.", "task_scope": "excel-clone", "title": "Relax excel-clone dependent recompute CUA reload proof", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/268", "summary": "The excel-clone CUA polish rubric required column-header highlighting or drag-range selection even when the oracle implemented the feature but the CUA probe skipped it, causing a false oracle failure.", "task_scope": "excel-clone", "title": "Relax excel-clone polish CUA criterion (drop flaky affordance probe)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/262", "summary": "Ruby-rust-port needed phase-scoped networking so installed agents could set up under public environment access while trajectory-producing agent and verifier phases stayed restricted to dependency hosts.", "task_scope": "ruby-rust-port", "title": "Scope Ruby task networking by execution phase", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/273", "summary": "The mastodon-clone oracle hid the notifications badge at zero unread, causing the CUA notifications criterion to PARTIAL even though the template rendered a muted zero badge.", "task_scope": "mastodon-clone", "title": "Show mastodon-clone zero notification badge", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swe-marathon", "evidence_label": "external_source_reported", "source_url": "https://github.com/abundant-ai/swe-marathon/pull/249", "summary": "과제가 열린 인터넷에서 검증기 자산을 다시 받도록 되어 있어 평가 환경이 고정되지 않는다는 신고와 오프라인 패키징 수정", "task_scope": "rust-c-compiler", "title": "검증기가 실행 중 인터넷과 재다운로드에 의존하는 문제", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/280", "summary": "Live issue argues SWE-bench only runs changed test files, allowing generated patches to pass official tests while failing unchanged developer tests.", "task_scope": "benchmark-level", "title": "\"Successful\" patches fail on unexecuted developer-written tests", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/485", "summary": "SWE-bench PR adds retries to PSF Requests tests to prevent false negatives from transient httpbin 503/timeout errors.", "task_scope": "psf__requests-2317", "title": "[Bug Fix] Add retry mechanism in PSF Requests test to prevent false negatives", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/584", "summary": "SWE-bench Verified astropy__astropy-8872 gold patch was marked unresolved because pytest collection aborted before required tests ran; maintainer now says current main gold passes this task.", "task_scope": "astropy__astropy-8872", "title": "[BUG] Gold patch for `astropy__astropy-8872` does not resolve — pytest collection error from setuptools-vendored `distutils`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/502", "summary": "Issue shows django__django-7530 is missing its real F2P test, so no-op code changes can pass the instance.", "task_scope": "django__django-7530", "title": "`django__django-7530` is missing test case", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/160", "summary": "SWE-bench exec_run_with_timeout raises TimeoutError but leaves the worker thread/command alive because Python threads cannot be killed, risking lingering runs.", "task_scope": "benchmark-level", "title": "`exec_run_with_timeout` does not actually kill long-running thread", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/215", "summary": "Live issue reports UnicodeDecodeError while streaming docker logs for a gold patch run.", "task_scope": "django__django-14011", "title": "`UnicodeDecodeError` when running gold patch for `django__django-14011` in the dockerized harness", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/634", "summary": "SWE-bench remove_containers matched exact container names even though eval containers include the run_id suffix, so stale containers were not removed and 409 conflicts could block reruns.", "task_scope": "benchmark-level", "title": "Add a swebench CLI with grouped commands and worked examples", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/456", "summary": "SWE-bench Modal report generation crashed on empty/invalid JSON report files; PR adds validation and external commenter reproduced same issue.", "task_scope": "benchmark-level", "title": "Add extra validation for `make_run_report` to prevent Modal crash", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/279", "summary": "Live PR shows parse_log_pytest_v2 truncates testcase names containing spaces, affecting current HuggingFace dataset test names, and demonstrates the corrected parse result.", "task_scope": "benchmark-level", "title": "Avoid breaking apart testcase name when they contain spaces", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/206", "summary": "Live PR avoids raising when cleanup tries to remove a non-existent Docker image, preventing an annoying interruption during execution.", "task_scope": "benchmark-level", "title": "avoid raising error when removing non-existent image", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/150", "summary": "Loading SWE-bench from a local JSON file failed because PASS_TO_PASS/FAIL_TO_PASS could already be Python lists but make_test_spec always json.loads strings.", "task_scope": "benchmark-level", "title": "Cannot load dataset from JSON file", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/366", "summary": "PR broadens Docker pull exception handling so non-NotFound Docker errors are not silently mishandled.", "task_scope": "benchmark-level", "title": "catch-all exception for docker pull", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/353", "summary": "Live PR fixes repo_setup scripts leaving dirty working trees, which makes generated patches fail to apply cleanly in evaluation containers.", "task_scope": "benchmark-level", "title": "Clean diff after setup", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/142", "summary": "SWE-bench PR containerized evaluation to address multiple environment failures including conda and testbed issues.", "task_scope": "benchmark-level", "title": "Containerize SWE-bench evaluation", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/157", "summary": "Live issue reports SWE-bench docker evaluation hanging; maintainers discuss timeout/cleanup fixes and later say related PRs likely addressed it.", "task_scope": "benchmark-level", "title": "docker evaluation gets stuck", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/111", "summary": "SWE-bench TaskEnvContextManager clobbered process env vars such as HTTP_PROXY during pip install, causing dependency installs to time out behind proxies; maintainer merged the fix.", "task_scope": "benchmark-level", "title": "environment is lost when running pip install", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/414", "summary": "Issue reports Modal sandbox creation failures can abort an entire SWE-bench evaluation and prevent final reports.", "task_scope": "pylint-dev__pylint-4661", "title": "Error creating sandbox using Modal", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/659", "summary": "SWE-bench v5 Verified cannot consistently evaluate non-amd64 image families because architecture defaults and runtime image selection stay x86_64-centric.", "task_scope": "benchmark-level", "title": "Evaluate the host architecture's images, and report absent ones as their own class", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/357", "summary": "Gold evaluation for astropy__astropy-12907 parsed the wrong output region when markers and pytest output landed on different streams; maintainer says commit 2c9df79 fixed fallback parsing.", "task_scope": "astropy__astropy-12907", "title": "Failure in parsing test output for`astropy__astropy-12907`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/342", "summary": "SWE-bench astropy__astropy-8707 is marked unresolved because tests error during setup rather than evaluating patches; later v5 Dockerfiles pin pytest and setuptools to compatible versions.", "task_scope": "astropy__astropy-8707", "title": "Failure In Setting Up the Test Cases In Task `astropy__astropy-8707`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/615", "summary": "SWE-bench Django directive generation passed every changed Python file to runtests.py, so non-test support modules such as tests/file_storage/models.py could become invalid dotted test labels and cause loader failures instead of selecting the intended tests.", "task_scope": "benchmark-level", "title": "Filter Django test directives to test files", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/202", "summary": "SWE-bench PR fixed image removal code using a nonexistent image_id attribute on a string dependency id.", "task_scope": "benchmark-level", "title": "fix bug when remove images", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/314", "summary": "SWE-bench PR fixes the Django log parser so it can handle multiline test cases; discussion says parser defects caused annotation errors in the existing benchmark.", "task_scope": "benchmark-level", "title": "fix django-parser's defect that it cannot handle multiline test cases.", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/475", "summary": "Three SWE-bench Verified instances have test patches that add only new files; the Python eval script resets the whole worktree when get_modified_files(test_patch) is empty, undoing setup changes and making gold evaluation fail.", "task_scope": "django__django-16877, sphinx-doc__sphinx-8595, sphinx-doc__sphinx-9711", "title": "Fix django__django-16877, sphinx-doc__sphinx-8595, sphinx-doc__sphinx-9711", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/83", "summary": "Live SWE-bench PR removes inaccessible conda env names from env_list parsing.", "task_scope": "benchmark-level", "title": "Fix env_list parsing", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/162", "summary": "Live PR fixes SWE-bench evaluation hanging when run_instance raises during patch application and improves patch apply fallback handling.", "task_scope": "benchmark-level", "title": "Fix evaluation hanging issue and improve patch apply", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/471", "summary": "Live PR fixes git-log leakage in SWE-bench environment images, but the author is a project member/internal actor.", "task_scope": "benchmark-level", "title": "Fix git log leakage in environment images", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/611", "summary": "SWE-bench harness parse_log_seaborn can raise IndexError on bare PASSED/FAILED log lines, crashing grading for seaborn instances.", "task_scope": "benchmark-level", "title": "Fix IndexError in parse_log_seaborn on bare status lines", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/390", "summary": "SWE-bench PR fixes JSONL loading by parsing .jsonl line-by-line instead of treating it like JSON, closing a harness data-loading bug.", "task_scope": "benchmark-level", "title": "Fix loading of jsonl data", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/570", "summary": "PR fixes Modal eval marker ordering where non-Django gold evaluations were reported failed despite tests passing.", "task_scope": "benchmark-level", "title": "Fix Modal eval marker ordering for non-django repos (#447)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/166", "summary": "SWE-bench PR fixed Django log parser for multi-line test outputs that broke testcase parsing.", "task_scope": "benchmark-level", "title": "Fix newline outputs for django's log parser", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/164", "summary": "Live SWE-bench PR fixes prepare_images crashing when all environment images already exist because build_env_images returned None instead of empty lists.", "task_scope": "benchmark-level", "title": "Fix so it doesn't crash when no env imgs to build", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/635", "summary": "SWE-bench images build defaulted --tag to None, causing make_image_spec to assert and making the image prebuild command crash.", "task_scope": "benchmark-level", "title": "Fix three harness bugs that silently drop instances", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/183", "summary": "PR fixes SWE-bench timeout handling by killing timed-out test runs in the container and improving logging/exception handling.", "task_scope": "benchmark-level", "title": "Fix timeout", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/533", "summary": "SWE-bench v5 PR fixed timezone-dependent git tag cleanup where string comparisons could leave/remove wrong future references; verified on 500 images.", "task_scope": "benchmark-level", "title": "Fix timezone bug in git tag cleanup", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/482", "summary": "SWE-bench Python test spec compared formatted commit timestamps across timezones; PR switches to Unix timestamps for robust comparisons.", "task_scope": "benchmark-level", "title": "Fix timezone bug in Python test spec by using Unix timestamps", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/423", "summary": "Live issue reports --namespace none with force_rebuild throws an unintended ValueError due to type mismatch.", "task_scope": "benchmark-level", "title": "Fix type mismatch in `run_instances` function for `namespace` parameter", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/417", "summary": "Merged SWE-bench PR fixes Python base image requirement replacement so package versions are retained.", "task_scope": "benchmark-level", "title": "fix(build): fix python base images requirement types-setuptools incor…", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/41", "summary": "Live SWE-bench PR handles conda env list lines that contain only a path, inferring the env name from the path.", "task_scope": "benchmark-level", "title": "fix(get_conda_env_names): handle lines with only path in `conda env l…", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/608", "summary": "SWE-bench grading reads agent-controlled test stdout between the first START/END markers and current main has no exit-code marker guard, so forged stdout can be parsed as passing output before a real failing command.", "task_scope": "benchmark-level", "title": "fix(harness): harden eval grading against stdout forging", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/619", "summary": "The common SWE-bench eval-script builder resets only files returned by get_modified_files(test_patch), so new-file-only gold tests in non-Python paths are not removed before applying the official test patch and can be pre-planted by a submission.", "task_scope": "benchmark-level", "title": "fix(harness): reset new test files in the common eval-script builder (follow-up to #539)", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/498", "summary": "SWE-bench report_dir bug wrote the final report outside the requested directory; maintainer says it was resolved by commit 36c7a6f.", "task_scope": "benchmark-level", "title": "Fix: argument report_dir does not work", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/186", "summary": "SWE-bench docker_build called find_dependent_images with wrong argument, and PR fixed incorrect function call plus typings.", "task_scope": "benchmark-level", "title": "fix: correct some typings and a incorrect function call", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/308", "summary": "SWE-bench PR fixes loading predictions when the JSON is dict-formatted and adds type validation before returning predictions.", "task_scope": "benchmark-level", "title": "fix: handle dict-format JSON correctly when loading predictions", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/189", "summary": "SWE-bench PR removes incorrect exec_run tar/rm calls after explaining Docker put_archive already extracts files into the container, preventing harness copy behavior from being wrong.", "task_scope": "benchmark-level", "title": "fix: remove incorrect `exec_run` calls", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/415", "summary": "PR attempts to fix swebench.harness.constants import errors from missing/incomplete language constants and an emptied Python install map.", "task_scope": "benchmark-level", "title": "Fix: Resolve import errors and refactor constants module", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/270", "summary": "SWE-bench pytest_v2 parser crashed with IndexError on invalid log lines, aborting evaluation for astropy__astropy-14995.", "task_scope": "astropy__astropy-14995", "title": "Fix: Skip invalid log lines in pytest_v2 log parser.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/151", "summary": "SWE-bench PR fixed JSON dataset support to avoid loading JSON twice.", "task_scope": "benchmark-level", "title": "Fix: Support JSON datasets (avoid loading json twice)", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/212", "summary": "SWE-bench PR fixed SymPy log parser dropping the last test line when it ended with [FAIL]/[OK].", "task_scope": "benchmark-level", "title": "fix: sympy log parser fails to recognize the last test case", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/653", "summary": "SWE-bench harness writes container-bound eval.sh and patch.diff with host-default newlines, so Windows hosts can poison Linux evaluation with CRLF.", "task_scope": "benchmark-level", "title": "fix: write eval.sh and patch.diff with LF on Windows hosts", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/401", "summary": "PR attempts to fix harness failures where requirements.txt cannot be found for some tasks by falling back to main, though maintainer questioned stability of that approach.", "task_scope": "benchmark-level", "title": "Fixes #400 (Harness no longer works: Could not find requirements.txt for some tasks)", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/531", "summary": "SWE-bench fail-only grading could treat missing or errored test output as resolved; maintainer says errored tests are now failures via #592/395389c.", "task_scope": "benchmark-level", "title": "Guard against empty eval_status_map in grading", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/169", "summary": "SWE-bench PR handled failures from None/empty patches in evaluation.", "task_scope": "benchmark-level", "title": "Handle failures because of None/empty patches", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/67", "summary": "PR fixes conda env list parsing when an environment is selected with '*', resolving issue #66.", "task_scope": "benchmark-level", "title": "Handle the case when a conda environment is selected", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/197", "summary": "SWE-bench PR updated grading so expected xfail outcomes count as passed rather than failed.", "task_scope": "benchmark-level", "title": "Handle xfail in grading", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/248", "summary": "Live issue reports SWE-bench harness cannot find Django requirements.txt for Verified gold evaluation, suggesting environment/repo-spec drift.", "task_scope": "benchmark-level", "title": "Harness no longer works: Could not find requirements.txt for repo django", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/294", "summary": "SWE-bench Verified gold-patch evaluation is unstable across the 15 task ids reported in issue #294; L3 reproduced the defect with 6 failures in one full run.", "task_scope": "astropy__astropy-7166, astropy__astropy-7336, astropy__astropy-7606, astropy__astropy-7671, astropy__astropy-8707, astropy__astropy-8872, django__django-10097, matplotlib__matplotlib-20488, psf__requests-1766, psf__requests-2317, pylint-dev__pylint-6528, pylint-dev__pylint-7080, pylint-dev__pylint-7277, sphinx-doc__sphinx-10323, sphinx-doc__sphinx-10435", "title": "How stable is run_evaluation.py with gold patch for SWE-bench_Verified?", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/271", "summary": "Reporter found three Flask SWE-bench instances whose images failed to build; maintainer recreated and verified the fix.", "task_scope": "pallets__flask-5014", "title": "images don't build for some flask problems", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/534", "summary": "PR improves SWE-bench Docker reliability by raising client timeouts, cleaning ghost containers, retrying unique names, and adding skip_patch support after large eval runs hit ReadTimeout and conflicts.", "task_scope": "benchmark-level", "title": "Improve Docker reliability: timeouts, ghost containers, skip_patch", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/563", "summary": "SWE-bench astropy__astropy-13033 PR description asks for missing-column information but gold tests require an exact list-formatted error string never specified to the model.", "task_scope": "astropy__astropy-13033", "title": "Inconsistency between PR description and gold tests in `astropy__astropy-13033`", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/324", "summary": "SWE-bench uses Python 3.9 for old requests tasks from 2013, causing a reported test not to fail at base commit and undermining FAIL_TO_PASS validity for the task.", "task_scope": "psf__requests-1724", "title": "Incorrect Python versions used to reproduce issues and verify fixes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/275", "summary": "SWE-bench FAIL_TO_PASS and PASS_TO_PASS fields for Django can contain docstrings rather than full test identifiers due to parser limitations; commenter says more than half of test names in Lite/Verified are wrong and over 100 patches may be mislabeled.", "task_scope": "benchmark-level", "title": "Incorrect unit tests in `FAIL_TO_PASS` and `PASS_TO_PASS`", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/149", "summary": "SWE-bench PR fixed run_id=None causing PosixPath / NoneType TypeError.", "task_scope": "benchmark-level", "title": "Interface fix: run_id is required", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/241", "summary": "Sphinx instance failed with ImportError from unpinned Jinja2>=3.1; reporter identified Jinja2<3.1 as the fix.", "task_scope": "sphinx-doc__sphinx-8595", "title": "Jinja2 is not pinned for sphinx", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/389", "summary": "SWE-bench jsonl dataset loading was implemented as JSON parsing, causing JSONDecodeError on valid jsonl files; maintainer confirmed bug.", "task_scope": "benchmark-level", "title": "jsonl dataset loading incorrect", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/207", "summary": "SWE-bench run_report ignored the instance_ids evaluation filter, producing confusing error_ids and performance accounting for filtered runs.", "task_scope": "benchmark-level", "title": "Make `run_report` more intuitive when using `instance_ids` filter during evaluation", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/437", "summary": "SWE-bench allows submitted patches to modify tests; discussion says test edits should be disabled/filtered during evaluation.", "task_scope": "benchmark-level", "title": "Make sure submitted patch doesn't modify tests", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/514", "summary": "Live issue reports sympy__sympy-13031 problem statement targets Dense Matrix while the gold patch changes SparseMatrix.", "task_scope": "sympy__sympy-13031", "title": "Mismatch between Problem Statement and Gold Patch in sympy__sympy-13031", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/427", "summary": "SWE-bench Modal grading parsed only part of test_output.txt, so correct solutions including gold patches were reported as failed.", "task_scope": "benchmark-level", "title": "Modal evaluation log parsing fix", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/538", "summary": "Live issue shows submitted patches can overwrite files later added by test_patch and create false positives.", "task_scope": "django__django-16877", "title": "Model can manipulate test results by overwriting new test patches", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/103", "summary": "PR fixes SWE-bench metric logic that marked a prediction as no-apply when either full patch or minimal patch failed, even if the other apply path succeeded.", "task_scope": "benchmark-level", "title": "modify metrics", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/223", "summary": "SWE-bench Verified astropy__astropy-7606 contained nonexistent PASS_TO_PASS test, making gold/valid patches score unresolved until dataset update.", "task_scope": "astropy__astropy-7606", "title": "nonexistent PASS_TO_PASS test in dataset for astropy__astropy-7606", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/530", "summary": "Live issue reports astropy__astropy-8707 gold behavior is misgraded because nose-era Astropy tests break under the pinned pytest/setuptools environment; maintainer says v5 Dockerfiles fix it.", "task_scope": "astropy__astropy-8707", "title": "Nose tests make swebench.harness.run_evaluation gives the wrong result for `astropy__astropy-8707`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/337", "summary": "SWE-bench parse_log_pytest_v2 reversed test name and status for older pytest output, causing astropy gold paths to fail.", "task_scope": "astropy__astropy-7166", "title": "parse_log_pytest_v2 reverses test name and status in map assignment", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/165", "summary": "Django test output containing noisy stdout was parsed as failure even though the test case passed, causing passed tests to count as failure.", "task_scope": "benchmark-level", "title": "Passed test case count as failure?", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/243", "summary": "SWE-bench Sphinx specs needed Jinja2<3.1 to avoid environment build failures; maintainer reproduced and verified the fix.", "task_scope": "sphinx-doc__sphinx-8595", "title": "Pin Jinja2 to version to 3.0.3 for Sphinx", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/598", "summary": "The SWE-bench harness mis-parses pytest output: 'nodeid SKIPPED ...' lines lose their real node id, and 'SKIPPED [N] path:line: reason' summary lines are recorded as a fake [N] test, distorting grading for tasks with skipped tests.", "task_scope": "benchmark-level", "title": "Pytest SKIPPED lines mis-parsed during grading", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/620", "summary": "채점기가 테스트 종료 코드를 보지 않고 stdout의 PASSED 문자열만 읽어 미해결 패치를 해결로 만들 수 있다는 신고와 수정", "task_scope": "benchmark-level", "title": "pytest 출력 위조를 통과시키는 채점 문제", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/402", "summary": "SWE-bench Modal evaluation sliced test output content and marked a locally resolved SymPy patch unresolved; PR conditions/removes the problematic slicing.", "task_scope": "benchmark-level", "title": "Remove problematic content slicing in test output parsing", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/465", "summary": "SWE-bench Verified evaluation images leaked future repository state through git history/tags, letting agents recover solution details; upstream adapted the harness and rebuilt images.", "task_scope": "benchmark-level", "title": "Repo State Loopholes During Agentic Evaluation", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/113", "summary": "Live issue reports Flask/Scikit-Learn environments fail to initialize in a clean Docker setup due suspected Cython/version mismatch despite repair report claims.", "task_scope": "benchmark-level", "title": "Reproducer Docker image", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/216", "summary": "SWE-bench PR fixed decode() on streamed temporarily invalid UTF-8 by buffering bytes before decoding.", "task_scope": "benchmark-level", "title": "Resolve an issue when attempting to call decode() on a streamed/potentially temporarily invalid utf-8 string.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/352", "summary": "SWE-bench setup_repo.sh scripts could leave the git worktree dirty after install steps, polluting generated patches.", "task_scope": "benchmark-level", "title": "setup_repo.sh scripts can leave the git working tree in a dirty state", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/292", "summary": "Issue reports SWE-bench Verified parametrized test specs missing required parameters; maintainer confirms it is legitimate.", "task_scope": "benchmark-level", "title": "Some SWE-bench_Verified test specs are missing test parameters", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/158", "summary": "Issue reports SWE-bench hanging in future.result() during evaluation, apparently around instance image building.", "task_scope": "benchmark-level", "title": "swe-bench can get badly stuck in `future.result()`", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/341", "summary": "Live issue shows astropy__astropy-7606 is marked unresolved even though top-agent output logs show all tests passed; maintainer says it was resolved by a fix.", "task_scope": "astropy__astropy-7606", "title": "Task `astropy__astropy-7606` Incorrectly Marked as Unresolved", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/289", "summary": "SWE-bench Verified PASS_TO_PASS contains the pytest progress marker [100%] as a test name for two pytest tasks.", "task_scope": "pytest-dev__pytest-5262, pytest-dev__pytest-7521", "title": "Tests incorrectly named [100%] in SWE-bench-Verified instances pytest-5262 and pytest-7521", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/290", "summary": "SWE-bench Verified PASS_TO_PASS contains hundreds of truncated parametrized pytest names; maintainer confirmed the issue and later resolved grading for those ids with commit 535db3e.", "task_scope": "benchmark-level", "title": "Truncated parametrized test names in SWE-bench_Verified", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/487", "summary": "SWE-bench Verified django__django-10097 gold patch could not run successfully; maintainer now says current main gold passes this task.", "task_scope": "django__django-10097", "title": "Unable to run gold patch for django__django-10097", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/57", "summary": "SWE-bench scikit-learn evaluation failed because installations were underspecified, especially missing numpy; maintainer says latest harness addressed it.", "task_scope": "scikit-learn__scikit-learn-13328", "title": "Unable to test for Scikit-Learn", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/66", "summary": "Issue reports UnboundLocalError in harness/utils.py when parsing conda env list with an active environment marked by '*'.", "task_scope": "benchmark-level", "title": "UnboundLocalError: local variable 'env_name' referenced before assignment in harness/utils.py line 56", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/130", "summary": "SWE-bench PR updated astropy pre_install only for versions >4.0, connecting to incorrect install specs.", "task_scope": "benchmark-level", "title": "Update astropy pre_install for only 4.0> versions", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/86", "summary": "Maintainer PR updates pytest log parsing to remove filesystem paths from parameterized test names, preventing mismatches between logs and expected test identifiers.", "task_scope": "benchmark-level", "title": "Update log parsing to remove paths from test names", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/651", "summary": "The SWE-bench README local evaluation setup omitted the required swe-bench-tasks checkout and --task-repo argument for current v5 CLI image builds, leaving the verified example with obsolete namespace guidance.", "task_scope": "benchmark-level", "title": "Update SWE-bench v5 local eval docs for task repos", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/368", "summary": "SWE-bench create_text_dataset generated prompt text column as None after PR #309; maintainer thanked reporter and fixed it.", "task_scope": "benchmark-level", "title": "Urgent, there is a bug when generate prompt_col in create_text_dataset.py", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "swebench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/654", "summary": "SWE-bench Verified grading counts XFAIL as a passed FAIL_TO_PASS test, so xfail(run=False) can mark an unresolved instance as resolved without executing the target test.", "task_scope": "benchmark-level", "title": "xfail(run=False) on a FAIL_TO_PASS test grades as resolved without the test running", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "swebench_multilingual", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/506", "summary": "Live PR fixes SWE-bench Java parser race conditions causing inconsistent evaluation, but author/commenters are project members.", "task_scope": "apache__druid-13704, apache__druid-14092, apache__druid-14136, apache__druid-15402, apache__druid-16875, apache__lucene-11760, apache__lucene-12022, apache__lucene-12196, apache__lucene-12212, apache__lucene-12626, apache__lucene-13170, apache__lucene-13301, apache__lucene-13494, apache__lucene-13704, google__gson-1014, google__gson-1093, google__gson-1100, google__gson-2024, google__gson-2061, google__gson-2134, google__gson-2158, google__gson-2311, google__gson-2479, javaparser__javaparser-4538, javaparser__javaparser-4561, reactivex__rxjava-7597", "title": "Fix flaky java evals", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebench_multilingual", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/532", "summary": "Live PR follow-up fixes SWE-bench Java Maven/Gradle log parser races; scoped to current multilingual Maven/Gradle Java tasks.", "task_scope": "apache__lucene-11760", "title": "Fix flaky Java log parsers: race condition handling", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench_multilingual", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/616", "summary": "SWE-bench Multilingual Jest parser can produce inconsistent test-name keys when Jest omits the duration suffix, hiding real fail-to-pass transitions.", "task_scope": "benchmark-level", "title": "Fix inconsistent test name whitespace in parse_log_jest", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench_multilingual", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/536", "summary": "SWE-bench Multilingual dataset PR pins versions and fixes eval_scripts across languages; gold/no-patch validation confirms dataset-level fixes.", "task_scope": "benchmark-level", "title": "Fix Multilingual dataset: version pinning and eval_script corrections", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebench_multilingual", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/507", "summary": "Fixes multiple SWE-bench Multilingual Docker image build errors by pinning Rust/Cargo fixture state and other constants; prerequisite to rebuilding images and preventing git-log leakage.", "task_scope": "apache__lucene-13170, micropython__micropython-10095, micropython__micropython-12158, micropython__micropython-13039, micropython__micropython-13569, micropython__micropython-15898, tokio-rs__axum-1119, tokio-rs__axum-1730, tokio-rs__axum-1934, tokio-rs__axum-2096, tokio-rs__axum-682, tokio-rs__axum-691, tokio-rs__axum-734, tokio-rs__tokio-4384, tokio-rs__tokio-4867, tokio-rs__tokio-6551, tokio-rs__tokio-6603, tokio-rs__tokio-6724, tokio-rs__tokio-6752, tokio-rs__tokio-6838, tokio-rs__tokio-7139", "title": "Fix multilingual instance build errors", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "swebench_multilingual", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/pull/656", "summary": "SWE-bench Multilingual task Dockerfiles write an unguarded .bashrc that sources a missing Miniconda profile, adding shell noise and preventing conda activation for agent harnesses.", "task_scope": "benchmark-level", "title": "Flag task Dockerfiles that ship a broken .bashrc (#638)", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebench_multilingual", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-bench/issues/578", "summary": "SWE-bench Multilingual images leak git tags with future commits reachable from tags, similar to prior future-history leakage.", "task_scope": "benchmark-level", "title": "SWE-Bench Multilingual Images Leak Git Tags", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/issues/115", "summary": "SWE-bench Pro Navidrome instance requirements say Register should return nil transcoding, while the gold implementation and required TestCore expect transcoding ID 1.", "task_scope": "instance_navidrome__navidrome-97434c1789a6444b30aae5ff5aa124a96a88f504", "title": "[Broken task] Navidrome 97434c: requirements mandate nil transcoding but TestCore expects trc.ID", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/pull/24", "summary": "PR fixed SWE-bench Pro eval script Docker Hub image URI derivation so gold patches could run successfully on a subset.", "task_scope": "benchmark-level", "title": "Fix eval script to correctly find dockerhub images", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/pull/18", "summary": "SWE-bench Pro eval script used uppercase FAIL_TO_PASS/PASS_TO_PASS keys where the dataset has lowercase fail_to_pass/pass_to_pass.", "task_scope": "benchmark-level", "title": "fix key error", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/pull/111", "summary": "SWE-bench Pro local Docker evaluation can silently score long-running instances unresolved because the Docker SDK default 60s read timeout applies to container.wait().", "task_scope": "benchmark-level", "title": "Fix: raise docker client timeout so slow evals aren't silently scored unresolved", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/pull/116", "summary": "SWE-bench Pro local Docker evaluation treats Linux arm64 hosts as Apple Silicon, forcing amd64 emulation and recording infrastructure failures as ordinary patch failures.", "task_scope": "benchmark-level", "title": "Graviton arm64", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/issues/19", "summary": "SWE-bench Pro Jest parser was incorrectly implemented and included tests that should not have been in FAIL_TO_PASS.", "task_scope": "benchmark-level", "title": "Jest parser is not correctly implemented", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/issues/74", "summary": "SWE-bench Pro issue shows git apply fails atomically when agent patches contain binary hunks, rejecting otherwise valid source-code fixes.", "task_scope": "benchmark-level", "title": "Patches with binary hunks fail to apply", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/pull/94", "summary": "SWE-bench Pro instance Dockerfiles retain future git history after checkout, allowing agents to mine future commits, branches, or tags for reference fixes.", "task_scope": "benchmark-level", "title": "Remove Future Git History from Dockerfiles", "upstream_status": "fixing" }, { "audit_label": "L1", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/issues/108", "summary": "External preregistered determinacy audit of the public SWE-bench Pro set documents an under-determination floor of 109/728 (15.0%) where the behavior the hidden test grades is not pinned by what a solver receives (problem statement, requirements, interface, base-commit source). Explicitly not a contamination claim. Archived with a citable Zenodo DOI.", "task_scope": "benchmark-level", "title": "Right of reply: an external determinacy audit of the SWE-bench Pro public set", "upstream_status": "found" }, { "audit_label": "L3", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/issues/76", "summary": "SWE-bench Pro issue identifies whitespace and truncated-quote mismatches in fail_to_pass test names that mark instances unresolved even when tests pass.", "task_scope": "instance_nodebb__nodebb-00c70ce7b0541cfc94afe567921d7668cdc8f4ac-vnan", "title": "Test name mismatches cause false negatives in eval scoring (whitespace + truncated quotes in fail_to_pass)", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swebenchpro", "evidence_label": "external_source_reported", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os/issues/72", "summary": "Live issue reports SWE-bench Pro instance tests treating 2026 as future, causing a current-date false negative; maintainer agrees on removing tests.", "task_scope": "instance_internetarchive__openlibrary-1894cb48d6e7fb498295a5d3ed0596f6f603b784-v0f5aece3601a5b4419f7ccec1dbda2071be28ee4", "title": "Time-sensitive test causes false negative in one instance", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/95", "summary": "Merged SWE-Smith PR fixes evaluation assumptions around missing test output and explicit passes/failures.", "task_scope": "benchmark-level", "title": "Fix #24: Fix eval script to look for explicit passes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/180", "summary": "SWE-smith get_container used instance_id as Docker container name, causing name conflicts when parallel workers handle the same instance; PR adds unique suffixes.", "task_scope": "benchmark-level", "title": "Fix container name conflict in get_container() for parallel execution", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/241", "summary": "SWE-smith container execution helpers hardcode Docker platform=\"linux/x86_64\", ignoring RepoProfile.pltf and breaking or forcing emulation on arm64/aarch64 hosts.", "task_scope": "benchmark-level", "title": "Fix hardcoded x86_64 platform in container creation (#arm64)", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/218", "summary": "SWE-smith PR adds missing jinja2 dependency used during issue generation.", "task_scope": "benchmark-level", "title": "fix: add missing jinja2 dependency", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/244", "summary": "SWE-smith RepoProfile._docker_ssh_arg passes a discovered private key path to docker build before considering ssh-agent forwarding, so passphrase-protected keys can make BuildKit fail before building even public repo images.", "task_scope": "benchmark-level", "title": "fix: docker build fails with passphrase-protected SSH keys — prefer ssh-agent", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/243", "summary": "SWE-smith configs/install_repo.sh creates the testbed conda environment with only python, then immediately runs python -m pip; conda-forge/miniforge installs can omit pip unless it is requested explicitly.", "task_scope": "benchmark-level", "title": "fix: explicitly install pip when creating testbed env in install_repo.sh", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/242", "summary": "SWE-smith setup.sh runs plain uv sync even though pyproject.toml keeps runtime dependencies only in the all optional extra, leaving fresh installs without required packages.", "task_scope": "benchmark-level", "title": "fix: setup.sh installs no dependencies — use 'uv sync --extra all'", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/issues/245", "summary": "SWE-smith gold evaluation can reverse the bug patch for parse.py, then restore parse.py because it is also listed in PASS_TO_PASS, causing the gold repair to be discarded before pytest runs.", "task_scope": "r1chardj0n3s__parse.30da9e4f.func_basic__j0z3z0co", "title": "Gold evaluation restores a patched source file listed in PASS_TO_PASS", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/198", "summary": "SWE-smith PR adds explicit architecture handling and platform flags so x86_64 images can be built on arm64 Macs without arch/platform mismatch.", "task_scope": "benchmark-level", "title": "Improve Docker architecture handling", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/issues/193", "summary": "SWE-smith issue reports validation failing because instance_id parsing compares the wrong suffix after collect_patches adds .ref.", "task_scope": "benchmark-level", "title": "Maybe here is a bug about `instance_id` when running validation?", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/33", "summary": "SWE-smith missing __init__.py in harness submodule was fixed, likely packaging/import related.", "task_scope": "benchmark-level", "title": "Missing __init__.py in harness submodule", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/issues/24", "summary": "Live issue shows SWE-smith can be reward-hacked by causing pytest collection failures so missing tests are treated as passing; maintainers acknowledge and discuss fixes.", "task_scope": "benchmark-level", "title": "SWE-Smith Reward Hacking Exploit", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/issues/194", "summary": "SWE-smith issue says valid.py uses upstream tests rather than the mirrored/pruned repo, causing missing tests to be graded as failures.", "task_scope": "benchmark-level", "title": "valid.py runs test suite from upstream repo, not the mirrored and pruned repo", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swesmith", "evidence_label": "external_source_reported", "source_url": "https://github.com/SWE-bench/SWE-smith/pull/90", "summary": "Merged SWE-Smith PR bundles harness/validation fixes, including run_id convention and error handling in bug generation.", "task_scope": "benchmark-level", "title": "Various Fixes", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/pull/34", "summary": "Merged PR fixes SWTBench evaluation/report comparability by filtering full SWE predictions down to the official SWT subset.", "task_scope": "benchmark-level", "title": "Allow filtering of instances to official SWT during evaluation.", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/issues/4", "summary": "SWT-Bench Django image build failed because requirements.txt could not be found at commit; reporter linked it to a SWE-bench header/request issue and submitted a PR.", "task_scope": "benchmark-level", "title": "Cannot build the env image for django", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/pull/35", "summary": "SWT-bench PR fixes a harness bug where copying large files into Docker containers via command-line hacks could fail during execution.", "task_scope": "benchmark-level", "title": "Fix copying too large files into docker containers", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/pull/33", "summary": "swt-bench grading treated tests missing from post-state as failures even when they were spurious or skipped, inflating failures.", "task_scope": "benchmark-level", "title": "Fix grading of unmatched tests.", "upstream_status": "fixed" }, { "audit_label": "L3", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/issues/40", "summary": "SWT-Bench Verified gold/oracle evaluation still fails on 12 of the 14 instances reported in issue #40.", "task_scope": "pylint-dev__pylint-6386, pylint-dev__pylint-6903, pylint-dev__pylint-7080, sphinx-doc__sphinx-8459, sphinx-doc__sphinx-8475, sphinx-doc__sphinx-8548, sphinx-doc__sphinx-8551, sphinx-doc__sphinx-8593, sphinx-doc__sphinx-8595, sphinx-doc__sphinx-8621, sphinx-doc__sphinx-8638, sphinx-doc__sphinx-8721", "title": "Gold Patch Failures/Errors in SWT-Bench Verified", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/pull/31", "summary": "swt-bench parser fix required a space after testcase status reports, indicating grading parsing was previously wrong.", "task_scope": "benchmark-level", "title": "Implement suggested Fix #29", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/pull/32", "summary": "SWT-Bench grading errored when coverage computation was disabled; PR reuses raw output for non-coverage grading.", "task_scope": "benchmark-level", "title": "Just eval raw output when not computing coverage", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "swtbench-verified", "evidence_label": "external_source_reported", "source_url": "https://github.com/logic-star-ai/swt-bench/pull/12", "summary": "SWT-bench reproduces SWE-bench docker image dependency pinning fixes for broken instances like Flask and Sphinx.", "task_scope": "sphinx-doc__sphinx-11445", "title": "Reproduce docker image fixes (pinning versions) from SWE-Bench", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "termigen-environments", "evidence_label": "external_source_reported", "source_url": "https://github.com/ucsb-mlsec/terminal-bench-env/issues/41", "summary": "terminal-bench-env did not clearly tell the model how to finish, so successful task completion could still time out; open PR #42 adds an explicit completion phrase to the system prompt.", "task_scope": "benchmark-level", "title": "The system prompt does not clearly tell the model how to finish", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/59", "summary": "Oracle solution/environment defect.", "task_scope": "build-pmars", "title": "`build-pmars` failing with oracle solution", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/44", "summary": "Not counted for terminal-bench@2.0 health: covered_by_terminal_bench_2_pr_50.", "task_scope": "make-doom-for-mips", "title": "`caffe-cifar-10` and `make-doom-for-mips` tasks appear infeasible under current resource constraints", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/46", "summary": "Verifier/oracle bug.", "task_scope": "filter-js-from-html", "title": "`filter-js-from-html` verifier uses BeautifulSoup normalization as ground truth", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/52", "summary": "Environment dependency defect.", "task_scope": "git-multibranch", "title": "`git-multibranch` `solve.sh` calls tmux without install", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/62", "summary": "Instruction/verifier loophole.", "task_scope": "gpt2-codegolf", "title": "`gpt2-codegolf` should specify/verify no #includes", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/60", "summary": "Information leakage.", "task_scope": "model-extraction-relu-logits", "title": "`model-extraction-relu-logits` makes no attempt to hide forward.py", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/61", "summary": "Information leakage.", "task_scope": "path-tracing", "title": "`path-tracing` exposes `orig` binary", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/45", "summary": "Task prompt/verifier mismatch.", "task_scope": "sam-cell-seg", "title": "`sam-cell-seg` task prompt specifies positional args but verifier expects named flags", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/71", "summary": "The build-cython-ext verifier runs the full pyknotid 0.5.3 upstream test suite and fails on test_reconstructed_space_curve, a flaky upstream space-curve test unrelated to the task Cython extension checks.", "task_scope": "build-cython-ext", "title": "build-cython-ext oracle blocked by upstream pyknotid flaky test", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/76", "summary": "build-pov-ray oracle solve.sh downloads POV-Ray 2.2 archives directly from povray.org, which now returns HTTP 403 and prevents the oracle build from starting.", "task_scope": "build-pov-ray", "title": "build-pov-ray: oracle solve.sh fails — povray.org now returns HTTP 403 (Cloudflare) to the source downloads", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/68", "summary": "Verifier filesystem portability defect.", "task_scope": "path-tracing-reverse", "title": "fix(path-tracing-reverse): use shutil.move for cross-fs robustness", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/83", "summary": "qemu-alpine-ssh oracle exits before guest SSH setup finishes, causing intermittent verifier failure.", "task_scope": "qemu-alpine-ssh", "title": "fix(qemu-alpine-ssh): make the oracle wait for the guest SSH setup to finish", "upstream_status": "fixing" }, { "audit_label": "L3", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/82", "summary": "Debian 11 EOL breaks apt-based verifier and image setup for qemu-alpine-ssh and qemu-startup.", "task_scope": "qemu-alpine-ssh, qemu-startup", "title": "fix(qemu-startup, qemu-alpine-ssh): keep apt working after Debian 11 EOL", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/57", "summary": "External data drift task defect.", "task_scope": "protein-assembly", "title": "Fix/protein assembly", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/38", "summary": "Instruction ambiguity defect.", "task_scope": "build-pmars", "title": "fix: Clarify build-pmars source directory instructions", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/40", "summary": "Instruction underspecification.", "task_scope": "mteb-retrieve", "title": "fix: Clarify mteb-retrieve to require BGE query/passage prompt types", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/36", "summary": "Verifier path defect.", "task_scope": "break-filter-js-from-html", "title": "fix: Correct test file paths in break-filter-js-from-html", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/39", "summary": "Verifier/auth defect.", "task_scope": "configure-git-webserver", "title": "fix: Fix configure-git-webserver verifier auth and username", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/47", "summary": "Oracle reliability defect.", "task_scope": "rstan-to-pystan", "title": "Improve reliability of rstan-to-python", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/67", "summary": "Verifier reward-hacking loophole.", "task_scope": "make-doom-for-mips", "title": "make-doom-for-mips passes without a DOOM binary built (because verifier uses agent-controlled vm.js)", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/72", "summary": "The mcmc-sampling-stan oracle installs rstan through RcppParallel/StanHeaders, but the task environment lacks cmake, so RcppParallel 6.2.0 fails to configure and rstan never becomes available.", "task_scope": "mcmc-sampling-stan", "title": "mcmc-sampling-stan oracle cannot build rstan without cmake", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/28", "summary": "Oracle/verifier defect.", "task_scope": "crack-7z-hash", "title": "oracle update for crack-7z-hash", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/78", "summary": "Terminal-Bench v2.1 make-doom-for-mips has broken setup evidence: a 404 dependency URL, missing MIPS/LLVM toolchain dependencies, and a nondeterministic frame comparison.", "task_scope": "make-doom-for-mips", "title": "terminal_bench v2.1 - issues with make-doom-for-mips", "upstream_status": "found" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/48", "summary": "Verifier fragility defect.", "task_scope": "portfolio-optimization", "title": "Update the expected speedup in portfolio-optimization", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/pull/14", "summary": "Not counted for terminal-bench@2.0 health: repo_upkeep_or_new_task_not_health_defect.", "task_scope": "reshard-c4-data", "title": "Upgrade uv installation in Dockerfile", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench", "evidence_label": "external_source_reported", "source_url": "https://github.com/harbor-framework/terminal-bench-2/issues/75", "summary": "planarity가 최신 버전으로 바뀌며 노드 속성 이름이 달라져 verifier가 KeyError를 내는 신고와 pin 수정", "task_scope": "build-cython-ext", "title": "고정되지 않은 의존성이 검증기를 깨뜨리는 문제", "upstream_status": "fixing" }, { "audit_label": "L2", "benchmark": "terminal-bench-pro", "evidence_label": "external_source_reported", "source_url": "https://github.com/alibaba/terminal-bench-pro/pull/8", "summary": "terminal-bench-pro PR removes apache-log-security-analyzer due to quality issues and replaces it with a task with clearer requirements and better tests.", "task_scope": "apache-log-security-analyzer", "title": "data: replace apache-log-security-analyzer with git-bundle-ml-pipeline-reconstructor", "upstream_status": "fixed" }, { "audit_label": "L2", "benchmark": "usaco", "evidence_label": "external_source_reported", "source_url": "https://github.com/princeton-nlp/USACO/pull/4", "summary": "USACO PR fixes the judge execution bug by enabling main() execution in generated code.", "task_scope": "benchmark-level", "title": "enable main() function execution in generated code", "upstream_status": "fixed" } ], "count": 600, "evidence_label": "external_source_reported", "schema": "defectlab.public-ledger-snapshot/v1", "snapshot_date": "2026-09-19" }