dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-25 14:55:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

run repeats

Browse source
This commit is contained in:
Paul Gauthier 2023-07-02 06:09:27 -07:00
parent 92a09f3deb
commit 411c744a15
2 changed files with 41 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

61
benchmark/rungrid.py
View file

@ -9,45 +9,50 @@ from aider.dump import dump # noqa: F401
def main():
models = [
# "gpt-3.5-turbo-0301",
# "gpt-3.5-turbo-0613",
"gpt-3.5-turbo-0613",
# "gpt-3.5-turbo-16k-0613",
"gpt-4-0314",
"gpt-4-0613",
# "gpt-4-0314",
# "gpt-4-0613",
]
edit_formats = [
"diff",
"diff-func",
# "diff",
# "diff-func",
"whole",
"whole-func",
# "whole-func",
]
for model in models:
for edit_format in edit_formats:
# dump(model, edit_format)
for repeat in range(1, 10, 1):
for model in models:
for edit_format in edit_formats:
# dump(model, edit_format)
if "-func" in edit_format and "-03" in model:
continue
if "-func" in edit_format and "-03" in model:
continue
if (model, edit_format) == ("gpt-3.5-turbo-16k-0613", "whole-func"):
# sublist reliably hangs the API?
continue
# if (model, edit_format) == ("gpt-3.5-turbo-16k-0613", "whole-func"):
# # sublist reliably hangs the API?
# continue
dirname = f"rungrid-{model}-{edit_format}"
# dirname = f"rungrid-{model}-{edit_format}"
dirname = f"rungrid-{model}-{edit_format}-repeat-{repeat}"
run(dirname, model, edit_format)
cmd = [
"./benchmark/benchmark.py",
dirname,
"--model",
model,
"--edit-format",
edit_format,
"--threads",
"10",
"--cont",
]
print(" ".join(cmd))
subprocess.run(cmd, check=True)
def run(dirname, model, edit_format):
cmd = [
"./benchmark/benchmark.py",
dirname,
"--model",
model,
"--edit-format",
edit_format,
"--threads",
"10",
"--cont",
]
print(" ".join(cmd))
subprocess.run(cmd, check=True)
if __name__ == "__main__":