dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-28 16:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

svg

Browse source
This commit is contained in:
Paul Gauthier 2024-05-15 11:54:00 -07:00
parent 590cbbddfd
commit 74c0df8df8
4 changed files with 1768 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

40
benchmark/over_time.py
View file

@ -1,11 +1,11 @@
import matplotlib.pyplot as plt
import yaml
from datetime import datetime
from matplotlib import rc
from imgcat import imgcat
from matplotlib import rc
def plot_over_time(yaml_file):
with open(yaml_file, 'r') as file:
with open(yaml_file, "r") as file:
data = yaml.safe_load(file)
dates = []
@ -13,10 +13,10 @@ def plot_over_time(yaml_file):
models = []
for entry in data:
if 'released' in entry and 'pass_rate_2' in entry:
dates.append(entry['released'])
pass_rates.append(entry['pass_rate_2'])
models.append(entry['model'].split('(')[0].strip())
if "released" in entry and "pass_rate_2" in entry:
dates.append(entry["released"])
pass_rates.append(entry["pass_rate_2"])
models.append(entry["model"].split("(")[0].strip())
plt.rcParams["hatch.linewidth"] = 0.5
plt.rcParams["hatch.color"] = "#444444"
@ -26,22 +26,32 @@ def plot_over_time(yaml_file):
fig, ax = plt.subplots(figsize=(10, 5))
ax.grid(axis="y", zorder=0, lw=0.2)
for spine in ax.spines.values():
spine.set_edgecolor('#DDDDDD')
spine.set_edgecolor("#DDDDDD")
spine.set_linewidth(0.5)
colors = ['red' if 'gpt-4' in model else 'green' if 'gpt-3.5' in model else 'blue' for model in models]
colors = [
"red" if "gpt-4" in model else "green" if "gpt-3.5" in model else "blue" for model in models
]
ax.scatter(dates, pass_rates, c=colors, alpha=0.5, s=120)
for i, model in enumerate(models):
ax.annotate(model, (dates[i], pass_rates[i]), fontsize=12, alpha=0.75,
xytext=(5, 5), textcoords='offset points')
ax.annotate(
model,
(dates[i], pass_rates[i]),
fontsize=12,
alpha=0.75,
xytext=(5, 5),
textcoords="offset points",
)
ax.set_xlabel('Model release date', fontsize=18)
ax.set_ylabel('Aider code editing benchmark,\npercent completed correctly', fontsize=18)
ax.set_title('LLM code editing skill by model release date', fontsize=20)
ax.set_xlabel("Model release date", fontsize=18)
ax.set_ylabel("Aider code editing benchmark,\npercent completed correctly", fontsize=18)
ax.set_title("LLM code editing skill by model release date", fontsize=20)
plt.tight_layout()
plt.savefig("tmp_over_time.png")
plt.savefig("tmp_over_time.svg")
imgcat(fig)
# Example usage
plot_over_time('_data/edit_leaderboard.yml')
plot_over_time("_data/edit_leaderboard.yml")