dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-28 08:14:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix: Correctly extract language and testcase from results

Browse source
This commit is contained in:
Paul Gauthier 2024-12-18 12:38:40 -08:00 • committed by Paul Gauthier (aider)
parent 42d8650058
commit a168403d68
1 changed files with 7 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

9
benchmark/problem_stats.py
View file

@ -30,8 +30,9 @@ def load_results(dirname):
try:
results = json.loads(fname.read_text())
# Add language info to results
lang = fname.parts[-4] # Get language from path
lang = fname.parts[-5] # Get language from path
results["language"] = lang
dump(results)
all_results.append(results)
except json.JSONDecodeError:
print(f"Failed to parse {fname}")
@ -87,7 +88,7 @@ def analyze_exercise_solutions(dirs=None, topn=None):
if results:
for result in results:
try:
all_exercises.add(result["testcase"])
all_exercises.add(result["language"] + "/" + result["testcase"])
except KeyError:
print(f"Warning: Missing testcase in {dirname}")
@ -100,7 +101,11 @@ def analyze_exercise_solutions(dirs=None, topn=None):
testcase = result.get("testcase")
if not testcase:
continue
lang = result.get("language")
if not lang:
continue
testcase = f"{lang}/{testcase}"
# Consider it solved if the last test attempt passed
tests_outcomes = result.get("tests_outcomes", [])
if tests_outcomes and tests_outcomes[-1]: