From 0ae53ce1a1bfc343258e728d61a8e62e1b177e92 Mon Sep 17 00:00:00 2001 From: "Paul Gauthier (aider)" Date: Tue, 17 Dec 2024 14:08:47 -0800 Subject: [PATCH] feat: Output per-exercise stats, sort by solvers --- benchmark/problem_stats.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/benchmark/problem_stats.py b/benchmark/problem_stats.py index 0729f247b..810b48f42 100755 --- a/benchmark/problem_stats.py +++ b/benchmark/problem_stats.py @@ -52,7 +52,7 @@ def analyze_exercise_solutions(): if tests_outcomes and tests_outcomes[-1]: exercise_solutions[testcase].append(model) - # Print statistics + # Print per-exercise statistics print("\nExercise Solution Statistics:") print("-" * 40) @@ -60,8 +60,7 @@ def analyze_exercise_solutions(): sorted_exercises = sorted(exercise_solutions.items(), key=lambda x: len(x[1]), reverse=True) for testcase, models in sorted_exercises: - print(f"{testcase}: solved by {len(models)} models") - # print(f" Models: {', '.join(models)}") + print(f"{testcase}: {len(models)} solved") print("\nSummary:") print(f"Total exercises solved at least once: {len(exercise_solutions)}")