dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-30 17:24:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

cleanup

Browse source
This commit is contained in:
Paul Gauthier 2024-05-06 10:46:09 -07:00
parent 5fb7a323ec
commit 3162d42262
1 changed files with 2 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

57
benchmark/benchmark.py
View file

@ -42,50 +42,15 @@ def show_stats(dirnames, graphs):
# return # return
repeats = []
seen = dict() seen = dict()
rows = [] rows = []
for row in raw_rows: for row in raw_rows:
if not row: if not row:
continue continue
if row.model == "gpt-3.5-turbo": if row.completed_tests not in (89, 133):
row.model = "gpt-3.5-turbo-0613"
if row.model == "gpt-4":
row.model = "gpt-4-0613"
if row.edit_format == "diff-func-string":
row.edit_format = "diff-func"
if (
row.model == "gpt-3.5-turbo-0613"
and row.edit_format == "whole"
and "repeat" not in row.dir_name
):
# remember this row, so we can update it with the repeat_avg
repeat_row = len(rows)
# gpt35 = "gpt-3.5-turbo"
# gpt4 = "gpt-4"
# if row.model.startswith(gpt35):
# row.model = gpt35 + "\n" + row.model[len(gpt35) :]
# elif row.model.startswith(gpt4):
# row.model = gpt4 + "\n" + row.model[len(gpt4) :]
if "folk" in row.dir_name:
row.edit_format += "folk"
# if row.model == "gpt-4-0613":
# row.model += "\n(8k context window is\ntoo small for benchmark)"
if row.completed_tests < 89:
print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}") print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}")
# if "repeat" in row.dir_name:
# repeats.append(vars(row))
# continue
kind = (row.model, row.edit_format) kind = (row.model, row.edit_format)
if kind in seen: if kind in seen:
dump(row.dir_name) dump(row.dir_name)
@ -95,26 +60,6 @@ def show_stats(dirnames, graphs):
seen[kind] = row.dir_name seen[kind] = row.dir_name
rows.append(vars(row)) rows.append(vars(row))
if repeats:
dump(repeats)
extra = rows[repeat_row]
dump(extra)
repeats.append(extra)
repeats = pd.DataFrame.from_records(repeats)
repeat_max = repeats["pass_rate_2"].max()
repeat_min = repeats["pass_rate_2"].min()
repeat_avg = repeats["pass_rate_2"].mean()
repeat_lo = repeat_avg - repeat_min
repeat_hi = repeat_max - repeat_avg
dump(repeat_max)
dump(repeat_min)
dump(repeat_avg)
# use the average in the main bar
rows[repeat_row]["pass_rate_2"] = repeat_avg
else:
repeat_hi = repeat_lo = repeat_avg = None # noqa: F841 repeat_hi = repeat_lo = repeat_avg = None # noqa: F841
df = pd.DataFrame.from_records(rows) df = pd.DataFrame.from_records(rows)