diff --git a/benchmark/problem_stats.py b/benchmark/problem_stats.py index 3178c1fb8..7358a3827 100755 --- a/benchmark/problem_stats.py +++ b/benchmark/problem_stats.py @@ -182,7 +182,12 @@ def analyze_exercise_solutions(dirs=None, topn=None, copy_hard_set=False): print(f"Total exercises solved at least once: {solved_at_least_once}") print(f"Never solved by any model: {solved_by_none}") - print(f"Solved by all models: {solved_by_all}") + if solved_by_none > 0: + print("\nExercises never solved by any model:") + unsolved = [ex for ex, models in exercise_solutions.items() if not models] + for ex in sorted(unsolved): + print(f" {ex}") + print(f"\nSolved by all models: {solved_by_all}") print( f"Total exercises: {len(all_exercises)} = {solved_by_none} (none) + {solved_by_all} (all) +" f" {len(all_exercises) - solved_by_none - solved_by_all} (some)"