dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-28 00:05:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat: Print hard set problem counts by language

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-18 13:08:48 -08:00
parent d4b62608a9
commit 3e4500f9fd
1 changed files with 13 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
benchmark/problem_stats.py
View file

@ -188,6 +188,19 @@ def analyze_exercise_solutions(dirs=None, topn=None):
hard_set = {ex for ex, models in exercise_solutions.items() if len(models) <= HARD_SET_NUM}
print(f"Total hard set exercises: {len(hard_set)}")
# Count problems by language in hard set
lang_counts = defaultdict(int)
for exercise in hard_set:
lang = exercise.split('/')[1] # Get language from path
lang_counts[lang] += 1
print("\nHard set problems by language:")
print(f"{'Language':<12} {'Count':>5}")
print("-" * 18)
for lang, count in sorted(lang_counts.items()):
print(f"{lang:<12} {count:>5}")
print()
# For each model, compute performance on hard set
model_hard_stats = []
for (dirname, model), results, _ in valid_entries: