dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-30 01:04:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

show both tries; repeats

Browse source
This commit is contained in:
Paul Gauthier 2023-06-30 10:20:06 -07:00
parent a9cc3f81b2
commit f84e3d6ab7
1 changed files with 35 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

25
benchmark/benchmark.py
View file

@ -53,6 +53,9 @@ def show_stats(dirnames):
if row.edit_format == "diff-func-string": if row.edit_format == "diff-func-string":
row.edit_format = "diff-func" row.edit_format = "diff-func"
if "repeat" in row.dir_name:
row.edit_format = "-".join(row.dir_name.split("-")[-2:])
if row.completed_tests < 133: if row.completed_tests < 133:
print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}") print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}")
@ -68,7 +71,7 @@ def show_stats(dirnames):
df = pd.DataFrame.from_records(rows) df = pd.DataFrame.from_records(rows)
df.sort_values(by=["model", "edit_format"], inplace=True) df.sort_values(by=["model", "edit_format"], inplace=True)
# df_grouped1 = df.groupby(["model", "edit_format"])["pass_rate_1"].mean() df_grouped1 = df.groupby(["model", "edit_format"])["pass_rate_1"].mean()
df_grouped2 = df.groupby(["model", "edit_format"])["pass_rate_2"].mean() df_grouped2 = df.groupby(["model", "edit_format"])["pass_rate_2"].mean()
plt.rcParams["hatch.linewidth"] = 0.5 plt.rcParams["hatch.linewidth"] = 0.5
@ -88,7 +91,10 @@ def show_stats(dirnames):
color= colors, color= colors,
) )
""" """
df = df_grouped2.unstack() zorder = 1
for grouped in (df_grouped2, df_grouped1):
zorder += 1
df = grouped.unstack()
num_models, num_formats = df.shape num_models, num_formats = df.shape
pos = np.array(range(num_models)) pos = np.array(range(num_models))
@ -98,17 +104,26 @@ def show_stats(dirnames):
models = df.index models = df.index
for i, fmt in enumerate(formats): for i, fmt in enumerate(formats):
if zorder:
edge = dict(
edgecolor="#444444",
linewidth=0.25,
)
if zorder == 2:
edge["label"] = fmt
color = "#b3e6a8" if "diff" in fmt else "#b3d1e6" color = "#b3e6a8" if "diff" in fmt else "#b3d1e6"
hatch = "///" if "func" in fmt else "" hatch = "///" if "func" in fmt else ""
rects = ax.bar( rects = ax.bar(
pos + i * width, pos + i * width,
df[fmt], df[fmt],
width * 0.95, width * 0.90,
label=fmt,
color=color, color=color,
hatch=hatch, hatch=hatch,
zorder=3, zorder=zorder,
**edge,
) )
if zorder == 2:
ax.bar_label(rects, padding=2, labels=[f"{v:.0f}%" for v in df[fmt]], size=12) ax.bar_label(rects, padding=2, labels=[f"{v:.0f}%" for v in df[fmt]], size=12)
ax.set_xticks([p + 1.5 * width for p in pos]) ax.set_xticks([p + 1.5 * width for p in pos])