dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 00:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix: Correctly calculate and display never solved exercises

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-17 18:14:48 -08:00
parent 12491c4983
commit 5c55453a0e
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

7
benchmark/problem_stats.py
View file

@ -78,6 +78,9 @@ def analyze_exercise_solutions(topn=None):
if tests_outcomes and tests_outcomes[-1]: if tests_outcomes and tests_outcomes[-1]:
exercise_solutions[testcase].append(model) exercise_solutions[testcase].append(model)
# Calculate never solved exercises
never_solved = len(all_exercises - set(exercise_solutions.keys()))
# Print per-exercise statistics # Print per-exercise statistics
print("\nExercise Solution Statistics:") print("\nExercise Solution Statistics:")
print("-" * 40) print("-" * 40)
@ -100,8 +103,8 @@ def analyze_exercise_solutions(topn=None):
print(f"{i:>3}. {testcase:<{max_name_len}} : {num_solved:>3} solved ({percent:>5.1f}%)") print(f"{i:>3}. {testcase:<{max_name_len}} : {num_solved:>3} solved ({percent:>5.1f}%)")
print("\nSummary:") print("\nSummary:")
print(f"Total exercises solved at least once: {len(exercise_solutions)}") solved_at_least_once = len([ex for ex, models in exercise_solutions.items() if models])
never_solved = 133 - len(exercise_solutions) print(f"Total exercises solved at least once: {solved_at_least_once}")
print(f"Never solved by any model: {never_solved}") print(f"Never solved by any model: {never_solved}")