dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 08:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

style: Run linter on problem_stats.py

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-18 13:03:31 -08:00
parent f3be2fa66b
commit 04916a6e97
1 changed files with 4 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

8
benchmark/problem_stats.py
View file

@ -189,7 +189,7 @@ def analyze_exercise_solutions(dirs=None, topn=None):
for (dirname, model), results, _ in valid_entries: for (dirname, model), results, _ in valid_entries:
if not results: if not results:
continue continue
solved_hard = 0 solved_hard = 0
for result in results: for result in results:
testcase = result.get("testcase") testcase = result.get("testcase")
@ -198,19 +198,19 @@ def analyze_exercise_solutions(dirs=None, topn=None):
lang = result.get("language") lang = result.get("language")
if not lang: if not lang:
continue continue
testcase = f"{testcase}/{lang}" testcase = f"{testcase}/{lang}"
if testcase in hard_set: if testcase in hard_set:
tests_outcomes = result.get("tests_outcomes", []) tests_outcomes = result.get("tests_outcomes", [])
if tests_outcomes and tests_outcomes[-1]: if tests_outcomes and tests_outcomes[-1]:
solved_hard += 1 solved_hard += 1
pct = (solved_hard / len(hard_set)) * 100 pct = (solved_hard / len(hard_set)) * 100
model_hard_stats.append((model, solved_hard, pct)) model_hard_stats.append((model, solved_hard, pct))
# Sort by number solved # Sort by number solved
model_hard_stats.sort(key=lambda x: x[1], reverse=True) model_hard_stats.sort(key=lambda x: x[1], reverse=True)
print("\nModel performance on hard set:") print("\nModel performance on hard set:")
print(f"{'Model':<30} {'Solved':<8} {'Percent':>7}") print(f"{'Model':<30} {'Solved':<8} {'Percent':>7}")
print("-" * 50) print("-" * 50)