dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 16:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

style: Run linter on benchmark script

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-17 14:06:56 -08:00
parent 9cc674c283
commit 7bfc2e0e74
1 changed files with 21 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

16
benchmark/problem_stats.py
View file

@ -1,9 +1,11 @@
#!/usr/bin/env python
import yaml
from pathlib import Path
from collections import defaultdict
import json
from collections import defaultdict
from pathlib import Path
import yaml
def load_results(dirname):
"""Load all result files from a benchmark directory"""
@ -22,6 +24,7 @@ def load_results(dirname):
continue
return all_results
def analyze_exercise_solutions():
# Load the leaderboard data
with open("aider/website/_data/edit_leaderboard.yml") as f:
@ -54,11 +57,7 @@ def analyze_exercise_solutions():
print("-" * 40)
# Sort by number of models that solved each exercise
sorted_exercises = sorted(
exercise_solutions.items(),
key=lambda x: len(x[1]),
reverse=True
)
sorted_exercises = sorted(exercise_solutions.items(), key=lambda x: len(x[1]), reverse=True)
for testcase, models in sorted_exercises:
print(f"{testcase}: solved by {len(models)} models")
@ -78,5 +77,6 @@ def analyze_exercise_solutions():
for count in sorted(solved_by_counts.keys()):
print(f"Solved by {count} models: {solved_by_counts[count]} exercises")
if __name__ == "__main__":
analyze_exercise_solutions()