From 5c55453a0eee092e3ed8d4ce31aac08b9c285891 Mon Sep 17 00:00:00 2001 From: "Paul Gauthier (aider)" Date: Tue, 17 Dec 2024 18:14:48 -0800 Subject: [PATCH] fix: Correctly calculate and display never solved exercises --- benchmark/problem_stats.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmark/problem_stats.py b/benchmark/problem_stats.py index 910bfc3b7..46f43e606 100755 --- a/benchmark/problem_stats.py +++ b/benchmark/problem_stats.py @@ -78,6 +78,9 @@ def analyze_exercise_solutions(topn=None): if tests_outcomes and tests_outcomes[-1]: exercise_solutions[testcase].append(model) + # Calculate never solved exercises + never_solved = len(all_exercises - set(exercise_solutions.keys())) + # Print per-exercise statistics print("\nExercise Solution Statistics:") print("-" * 40) @@ -100,8 +103,8 @@ def analyze_exercise_solutions(topn=None): print(f"{i:>3}. {testcase:<{max_name_len}} : {num_solved:>3} solved ({percent:>5.1f}%)") print("\nSummary:") - print(f"Total exercises solved at least once: {len(exercise_solutions)}") - never_solved = 133 - len(exercise_solutions) + solved_at_least_once = len([ex for ex, models in exercise_solutions.items() if models]) + print(f"Total exercises solved at least once: {solved_at_least_once}") print(f"Never solved by any model: {never_solved}")