dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-20 12:24:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: Print list of exercises never solved by any model

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-19 15:49:13 -08:00
parent 7eb7533d42
commit 2d32f77ed0
1 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

7
benchmark/problem_stats.py
View file

@ -182,7 +182,12 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False):
print(f"Total exercises solved at least once: {solved_at_least_once}")
print(f"Never solved by any model: {solved_by_none}")
print(f"Solved by all models: {solved_by_all}")
if solved_by_none > 0:
print("\nExercises never solved by any model:")
unsolved = [ex for ex, models in exercise_solutions.items() if not models]
for ex in sorted(unsolved):
print(f" {ex}")
print(f"\nSolved by all models: {solved_by_all}")
print(
f"Total exercises: {len(all_exercises)} = {solved_by_none} (none) + {solved_by_all} (all) +"
f" {len(all_exercises) - solved_by_none - solved_by_all} (some)"