dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-21 12:55:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: Add RevCumulative column to problem stats

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-28 11:45:41 -04:00
parent c21f7afdcb
commit 8eaefb57d3
1 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
benchmark/problem_stats.py
View file

@ -205,16 +205,18 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False):
# Distribution table of how many models solved each exercise # Distribution table of how many models solved each exercise
print("\nDistribution of solutions:") print("\nDistribution of solutions:")
print("Models Exercises Cumulative") print("Models Exercises Cumulative RevCumulative")
print("-" * 35) print("-" * 50)
counts = [0] * (total_models + 1) counts = [0] * (total_models + 1)
for ex, models in exercise_solutions.items(): for ex, models in exercise_solutions.items():
counts[len(models)] += 1 counts[len(models)] += 1
cumsum = 0 cumsum = 0
revcumsum = sum(counts) # Start with total number of exercises
for i, count in enumerate(counts): for i, count in enumerate(counts):
cumsum += count cumsum += count
print(f"{i:>6d} {count:>9d} {cumsum:>10d}") print(f"{i:>6d} {count:>9d} {cumsum:>10d} {revcumsum:>12d}")
revcumsum -= count # Decrement the reverse cumulative sum
# Count parse errors per exercise # Count parse errors per exercise
parse_error_counts = defaultdict(int) parse_error_counts = defaultdict(int)