diff --git a/assets/swe_bench.jpg b/assets/swe_bench.jpg index 4ce2881c3..f5aaf8ebc 100644 Binary files a/assets/swe_bench.jpg and b/assets/swe_bench.jpg differ diff --git a/assets/swe_bench.svg b/assets/swe_bench.svg index ffd8dbe1d..def4a99e8 100644 --- a/assets/swe_bench.svg +++ b/assets/swe_bench.svg @@ -6,7 +6,7 @@ - 2024-06-01T07:02:59.687095 + 2024-06-01T07:22:29.978035 image/svg+xml @@ -30,8 +30,8 @@ z - - - + - + - + - + - + - + - + - + - + - + @@ -707,19 +707,19 @@ z - + - + - + @@ -727,7 +727,7 @@ z - + - + @@ -813,12 +813,12 @@ z - + - + - + - + @@ -1004,7 +1004,7 @@ z - + @@ -1017,12 +1017,12 @@ z - + - + - + - + - - - - - - + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + + + + + + - - - - - - + + + + + + + + + + + + + - + - - - - - - + + + + + + - - - - - - - + + + + + + + - + - + + + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -1245,23 +1631,23 @@ z - + - - + - + - + - + - + @@ -1299,18 +1685,18 @@ L 690 274.534192 - + - + - + @@ -1319,18 +1705,18 @@ L 690 242.032134 - + - + - + @@ -1339,18 +1725,18 @@ L 690 209.530076 - + - + - + - + - + - + @@ -1394,18 +1780,18 @@ L 690 144.52596 - + - + - + @@ -1415,18 +1801,18 @@ L 690 112.023902 - + - + - + @@ -1436,7 +1822,7 @@ L 690 79.521844 - + - - - @@ -1563,64 +1949,64 @@ L 690 50.4 " style="fill: none; stroke: #dddddd; stroke-width: 0.5; stroke-linejoin: miter; stroke-linecap: square"/> - +" clip-path="url(#pa45e7e5613)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#pa45e7e5613)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#pa45e7e5613)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#pa45e7e5613)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#pa45e7e5613)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#pa45e7e5613)" style="fill: #17965a; opacity: 0.9"/> - +" clip-path="url(#pa45e7e5613)" style="fill: #17965a; opacity: 0.9"/> - + @@ -1630,7 +2016,7 @@ z - + - + @@ -1682,7 +2068,7 @@ z - + - + @@ -1777,7 +2163,7 @@ z - + - - - + - + + diff --git a/assets/swe_bench_lite.jpg b/assets/swe_bench_lite.jpg index 37cfd3cda..0286d6134 100644 Binary files a/assets/swe_bench_lite.jpg and b/assets/swe_bench_lite.jpg differ diff --git a/assets/swe_bench_lite.svg b/assets/swe_bench_lite.svg index ff7a2ec84..b3dc7cdfb 100644 --- a/assets/swe_bench_lite.svg +++ b/assets/swe_bench_lite.svg @@ -1,12 +1,12 @@ - + - 2024-05-30T15:26:12.767905 + 2024-06-01T07:22:45.471733 image/svg+xml @@ -21,8 +21,8 @@ - - - - + - + - + - + - + - + - + - + - + - + @@ -502,7 +502,7 @@ z - + - + - + - + - + - + - + - + @@ -928,108 +928,410 @@ z - + - - - - - - + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + + + + + + - + - - - - - - + + + + + + - - - - - - - + + + + + + + - + - + + + + + - - - - - - + + + + + + @@ -1037,23 +1339,23 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + @@ -1180,18 +1482,18 @@ L 690 152.407339 - + - + - + - + - + - + @@ -1245,7 +1547,7 @@ L 690 71.545398 - + - - - @@ -1402,64 +1704,64 @@ L 690 50.4 " style="fill: none; stroke: #dddddd; stroke-width: 0.5; stroke-linejoin: miter; stroke-linecap: square"/> - +" clip-path="url(#p837c6e1836)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#p837c6e1836)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#p837c6e1836)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#p837c6e1836)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#p837c6e1836)" style="fill: #b3d1e6; opacity: 0.3"/> - +" clip-path="url(#p837c6e1836)" style="fill: #17965a; opacity: 0.9"/> - +" clip-path="url(#p837c6e1836)" style="fill: #17965a; opacity: 0.9"/> - + - + - + - + - + - + - + - + + diff --git a/benchmark/swe_bench.py b/benchmark/swe_bench.py index 2151cf53c..52b0dcf82 100644 --- a/benchmark/swe_bench.py +++ b/benchmark/swe_bench.py @@ -5,6 +5,8 @@ import matplotlib.pyplot as plt from imgcat import imgcat from matplotlib import rc +from aider.dump import dump # noqa: F401 + def plot_swe_bench(data_file, is_lite): with open(data_file, "r") as file: @@ -50,6 +52,10 @@ def plot_swe_bench(data_file, is_lite): bar = ax.bar(model, pass_rate, color=color, alpha=alpha, zorder=3, hatch=hatch) bars.append(bar[0]) + for label in ax.get_xticklabels(): + if "Aider" in str(label): + label.set_fontfamily("Helvetica Bold") + for model, bar in zip(models, bars): yval = bar.get_height() y = yval - 1