dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 08:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

feat: Show exercises solved by all and total breakdown

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-18 12:38:41 -08:00
parent a168403d68
commit b8647c0481
1 changed files with 7 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

8
benchmark/problem_stats.py
View file

@ -159,8 +159,14 @@ def analyze_exercise_solutions(dirs=None, topn=None):
print("\nSummary:") print("\nSummary:")
solved_at_least_once = len([ex for ex, models in exercise_solutions.items() if models]) solved_at_least_once = len([ex for ex, models in exercise_solutions.items() if models])
solved_by_none = never_solved
solved_by_all = len([ex for ex, models in exercise_solutions.items() if len(models) == total_models])
print(f"Total exercises solved at least once: {solved_at_least_once}") print(f"Total exercises solved at least once: {solved_at_least_once}")
print(f"Never solved by any model: {never_solved}") print(f"Never solved by any model: {solved_by_none}")
print(f"Solved by all models: {solved_by_all}")
print(f"Total exercises: {len(all_exercises)} = {solved_by_none} (none) + {solved_by_all} (all) + "
f"{len(all_exercises) - solved_by_none - solved_by_all} (some)")
if __name__ == "__main__": if __name__ == "__main__":