From 3e4500f9fdebd01c120d759b92ac0a837960aa45 Mon Sep 17 00:00:00 2001 From: "Paul Gauthier (aider)" Date: Wed, 18 Dec 2024 13:08:48 -0800 Subject: [PATCH] feat: Print hard set problem counts by language --- benchmark/problem_stats.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/benchmark/problem_stats.py b/benchmark/problem_stats.py index ccb531bb0..b8f3a3c1d 100755 --- a/benchmark/problem_stats.py +++ b/benchmark/problem_stats.py @@ -188,6 +188,19 @@ def analyze_exercise_solutions(dirs=None, topn=None): hard_set = {ex for ex, models in exercise_solutions.items() if len(models) <= HARD_SET_NUM} print(f"Total hard set exercises: {len(hard_set)}") + # Count problems by language in hard set + lang_counts = defaultdict(int) + for exercise in hard_set: + lang = exercise.split('/')[1] # Get language from path + lang_counts[lang] += 1 + + print("\nHard set problems by language:") + print(f"{'Language':<12} {'Count':>5}") + print("-" * 18) + for lang, count in sorted(lang_counts.items()): + print(f"{lang:<12} {count:>5}") + print() + # For each model, compute performance on hard set model_hard_stats = [] for (dirname, model), results, _ in valid_entries: