diff --git a/benchmark/problem_stats.py b/benchmark/problem_stats.py index 004e4f24f..1e9925559 100755 --- a/benchmark/problem_stats.py +++ b/benchmark/problem_stats.py @@ -181,6 +181,7 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False): ) print(f"Total exercises solved at least once: {solved_at_least_once}") + # print out these never solved use lang/exercises/practice/ex ai! print(f"Never solved by any model: {solved_by_none}") if solved_by_none > 0: print("\nExercises never solved by any model:") @@ -235,9 +236,6 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False): } print(f"Total hard set exercises: {len(hard_set)}") - dump(disqualified_exercises) - dump(hard_set) - # Count total problems, unsolved problems, and hard set problems by language lang_totals = defaultdict(int) lang_unsolved = defaultdict(int)