dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-24 06:15:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

fix: Handle missing results in topn leaderboard calculation

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-17 14:15:02 -08:00
parent 674e3846e2
commit 7f16757bbe
1 changed files with 16 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

21
benchmark/problem_stats.py
View file

@ -31,11 +31,22 @@ def analyze_exercise_solutions(topn=None):
with open("aider/website/_data/edit_leaderboard.yml") as f: with open("aider/website/_data/edit_leaderboard.yml") as f:
leaderboard = yaml.safe_load(f) leaderboard = yaml.safe_load(f)
# Sort models by pass rate to get top N if specified # Filter out entries that don't load and sort by pass rate
if topn: valid_entries = []
leaderboard.sort(key=lambda x: float(x.get("pass_rate_2", 0)), reverse=True) for entry in leaderboard:
leaderboard = leaderboard[:topn] dirname = entry["dirname"]
results = load_results(dirname)
if results:
valid_entries.append((entry, results))
# Sort by pass rate and take top N if specified
valid_entries.sort(key=lambda x: float(x[0].get("pass_rate_2", 0)), reverse=True)
if topn:
valid_entries = valid_entries[:topn]
# Unpack the filtered and sorted entries
leaderboard = [entry for entry, _ in valid_entries]
# Get all exercise names from a complete run # Get all exercise names from a complete run
all_exercises = set() all_exercises = set()
exercise_solutions = defaultdict(list) exercise_solutions = defaultdict(list)
@ -81,7 +92,7 @@ def analyze_exercise_solutions(topn=None):
# Calculate max length for alignment # Calculate max length for alignment
max_name_len = max(len(testcase) for testcase in all_exercises) max_name_len = max(len(testcase) for testcase in all_exercises)
total_models = len({model for models in exercise_solutions.values() for model in models}) total_models = len(leaderboard)
for testcase, models in sorted_exercises: for testcase, models in sorted_exercises:
num_solved = len(models) num_solved = len(models)