dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 00:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat: Add --topn argument to limit models by pass rate

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-17 14:13:16 -08:00
parent 91f5fca5e9
commit 00d7c3a05a
1 changed files with 12 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

14
benchmark/problem_stats.py
View file

@ -1,5 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
import argparse
import json import json
from collections import defaultdict from collections import defaultdict
from pathlib import Path from pathlib import Path
@ -25,11 +26,16 @@ def load_results(dirname):
return all_results return all_results
def analyze_exercise_solutions(): def analyze_exercise_solutions(topn=None):
# Load the leaderboard data # Load the leaderboard data
with open("aider/website/_data/edit_leaderboard.yml") as f: with open("aider/website/_data/edit_leaderboard.yml") as f:
leaderboard = yaml.safe_load(f) leaderboard = yaml.safe_load(f)
# Sort models by pass rate to get top N if specified
if topn:
leaderboard.sort(key=lambda x: float(x.get('pass_rate_2', '0').rstrip('%')), reverse=True)
leaderboard = leaderboard[:topn]
# Get all exercise names from a complete run # Get all exercise names from a complete run
all_exercises = set() all_exercises = set()
exercise_solutions = defaultdict(list) exercise_solutions = defaultdict(list)
@ -89,4 +95,8 @@ def analyze_exercise_solutions():
if __name__ == "__main__": if __name__ == "__main__":
analyze_exercise_solutions() parser = argparse.ArgumentParser()
parser.add_argument('--topn', type=int, help='Only consider top N models by pass rate')
args = parser.parse_args()
analyze_exercise_solutions(args.topn)