dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-25 14:55:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat: Print never solved exercises and remove dumps

Browse source
This commit is contained in:
Paul Gauthier 2024-12-19 15:50:10 -08:00 • committed by Paul Gauthier (aider)
parent dddf192e5a
commit 6185ddf76a
1 changed files with 1 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
benchmark/problem_stats.py
View file

@ -181,6 +181,7 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False):
) )
print(f"Total exercises solved at least once: {solved_at_least_once}") print(f"Total exercises solved at least once: {solved_at_least_once}")
# print out these never solved use lang/exercises/practice/ex ai!
print(f"Never solved by any model: {solved_by_none}") print(f"Never solved by any model: {solved_by_none}")
if solved_by_none > 0: if solved_by_none > 0:
print("\nExercises never solved by any model:") print("\nExercises never solved by any model:")
@ -235,9 +236,6 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False):
} }
print(f"Total hard set exercises: {len(hard_set)}") print(f"Total hard set exercises: {len(hard_set)}")
dump(disqualified_exercises)
dump(hard_set)
# Count total problems, unsolved problems, and hard set problems by language # Count total problems, unsolved problems, and hard set problems by language
lang_totals = defaultdict(int) lang_totals = defaultdict(int)
lang_unsolved = defaultdict(int) lang_unsolved = defaultdict(int)