dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-27 15:55:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

fix: Correctly calculate and display never solved exercises

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-17 18:14:48 -08:00
parent 12491c4983
commit 5c55453a0e
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
benchmark/problem_stats.py
View file

@ -78,6 +78,9 @@ def analyze_exercise_solutions(topn=None):
if tests_outcomes and tests_outcomes[-1]:
exercise_solutions[testcase].append(model)
# Calculate never solved exercises
never_solved = len(all_exercises - set(exercise_solutions.keys()))
# Print per-exercise statistics
print("\nExercise Solution Statistics:")
print("-" * 40)
@ -100,8 +103,8 @@ def analyze_exercise_solutions(topn=None):
print(f"{i:>3}. {testcase:<{max_name_len}} : {num_solved:>3} solved ({percent:>5.1f}%)")
print("\nSummary:")
print(f"Total exercises solved at least once: {len(exercise_solutions)}")
never_solved = 133 - len(exercise_solutions)
solved_at_least_once = len([ex for ex, models in exercise_solutions.items() if models])
print(f"Total exercises solved at least once: {solved_at_least_once}")
print(f"Never solved by any model: {never_solved}")