dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 00:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

check for dup model/edit rows

Browse source
This commit is contained in:
Paul Gauthier 2023-06-29 19:11:28 -07:00
parent 0c9deaa118
commit a577a76a44
1 changed files with 10 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

11
benchmark/benchmark.py
View file

@ -41,6 +41,7 @@ def show_stats(dirnames):
row = summarize_results(dirname) row = summarize_results(dirname)
raw_rows.append(row) raw_rows.append(row)
seen = dict()
rows = [] rows = []
for row in raw_rows: for row in raw_rows:
if not row: if not row:
@ -52,12 +53,20 @@ def show_stats(dirnames):
if row.completed_tests < 133: if row.completed_tests < 133:
print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}") print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}")
kind = (row.model, row.edit_format)
if kind in seen:
dump(row.dir_name)
dump(seen[kind])
return
seen[kind] = row.dir_name
rows.append(vars(row)) rows.append(vars(row))
df = pd.DataFrame.from_records(rows) df = pd.DataFrame.from_records(rows)
df.sort_values(by=["model", "edit_format"], inplace=True) df.sort_values(by=["model", "edit_format"], inplace=True)
df_grouped = df.groupby(["model", "edit_format"])["pass_rate_1"].mean().sort_values(ascending=False) df_grouped = df.groupby(["model", "edit_format"])["pass_rate_1"].mean()
fig, ax = plt.subplots(figsize=(10, 6)) fig, ax = plt.subplots(figsize=(10, 6))
df_grouped.unstack().plot(kind="barh", ax=ax) df_grouped.unstack().plot(kind="barh", ax=ax)