dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 08:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

show both tries; repeats

Browse source
This commit is contained in:
Paul Gauthier 2023-06-30 10:20:06 -07:00
parent a9cc3f81b2
commit f84e3d6ab7
1 changed files with 35 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

55
benchmark/benchmark.py
View file

@ -53,6 +53,9 @@ def show_stats(dirnames):
if row.edit_format == "diff-func-string": if row.edit_format == "diff-func-string":
row.edit_format = "diff-func" row.edit_format = "diff-func"
if "repeat" in row.dir_name:
row.edit_format = "-".join(row.dir_name.split("-")[-2:])
if row.completed_tests < 133: if row.completed_tests < 133:
print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}") print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}")
@ -68,7 +71,7 @@ def show_stats(dirnames):
df = pd.DataFrame.from_records(rows) df = pd.DataFrame.from_records(rows)
df.sort_values(by=["model", "edit_format"], inplace=True) df.sort_values(by=["model", "edit_format"], inplace=True)
# df_grouped1 = df.groupby(["model", "edit_format"])["pass_rate_1"].mean() df_grouped1 = df.groupby(["model", "edit_format"])["pass_rate_1"].mean()
df_grouped2 = df.groupby(["model", "edit_format"])["pass_rate_2"].mean() df_grouped2 = df.groupby(["model", "edit_format"])["pass_rate_2"].mean()
plt.rcParams["hatch.linewidth"] = 0.5 plt.rcParams["hatch.linewidth"] = 0.5
@ -88,28 +91,40 @@ def show_stats(dirnames):
color= colors, color= colors,
) )
""" """
df = df_grouped2.unstack() zorder = 1
num_models, num_formats = df.shape for grouped in (df_grouped2, df_grouped1):
zorder += 1
df = grouped.unstack()
num_models, num_formats = df.shape
pos = np.array(range(num_models)) pos = np.array(range(num_models))
width = 0.8 / num_formats width = 0.8 / num_formats
formats = df.columns formats = df.columns
models = df.index models = df.index
for i, fmt in enumerate(formats): for i, fmt in enumerate(formats):
color = "#b3e6a8" if "diff" in fmt else "#b3d1e6" if zorder:
hatch = "///" if "func" in fmt else "" edge = dict(
rects = ax.bar( edgecolor="#444444",
pos + i * width, linewidth=0.25,
df[fmt], )
width * 0.95, if zorder == 2:
label=fmt, edge["label"] = fmt
color=color,
hatch=hatch, color = "#b3e6a8" if "diff" in fmt else "#b3d1e6"
zorder=3, hatch = "///" if "func" in fmt else ""
) rects = ax.bar(
ax.bar_label(rects, padding=2, labels=[f"{v:.0f}%" for v in df[fmt]], size=12) pos + i * width,
df[fmt],
width * 0.90,
color=color,
hatch=hatch,
zorder=zorder,
**edge,
)
if zorder == 2:
ax.bar_label(rects, padding=2, labels=[f"{v:.0f}%" for v in df[fmt]], size=12)
ax.set_xticks([p + 1.5 * width for p in pos]) ax.set_xticks([p + 1.5 * width for p in pos])
ax.set_xticklabels(models, rotation=45) ax.set_xticklabels(models, rotation=45)