dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 00:35:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

rename --retries to --tries

Browse source
This commit is contained in:
Paul Gauthier 2023-06-27 17:45:57 -07:00
parent 11dddad4c1
commit e24d03060d
1 changed files with 10 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

20
benchmark/benchmark.py
View file

@ -50,7 +50,7 @@ def main(
stats_only: bool = typer.Option( stats_only: bool = typer.Option(
False, "--stats-only", "-s", help="Do not run tests, just collect stats on completed tests" False, "--stats-only", "-s", help="Do not run tests, just collect stats on completed tests"
), ),
retries: int = typer.Option(2, "--retries", "-r", help="Number of retries for running tests"), tries: int = typer.Option(2, "--tries", "-r", help="Number of tries for running tests"),
threads: int = typer.Option(1, "--threads", "-t", help="Number of threads to run in parallel"), threads: int = typer.Option(1, "--threads", "-t", help="Number of threads to run in parallel"),
num_tests: int = typer.Option(-1, "--num-tests", "-n", help="Number of tests to run"), num_tests: int = typer.Option(-1, "--num-tests", "-n", help="Number of tests to run"),
): ):
@ -123,7 +123,7 @@ def main(
dirname / testname, dirname / testname,
model, model,
edit_format, edit_format,
retries, tries,
no_unit_tests, no_unit_tests,
verbose, verbose,
commit_hash, commit_hash,
@ -138,7 +138,7 @@ def main(
dirname / testname, dirname / testname,
model, model,
edit_format, edit_format,
retries, tries,
no_unit_tests, no_unit_tests,
verbose, verbose,
commit_hash, commit_hash,
@ -158,11 +158,11 @@ def summarize_results(dirname):
completed_tests = 0 completed_tests = 0
try: try:
retries = max(len(results["tests_outcomes"]) for results in all_results if results) tries = max(len(results["tests_outcomes"]) for results in all_results if results)
except ValueError: except ValueError:
retries = 0 tries = 0
passed_tests = [0] * retries passed_tests = [0] * tries
duration = 0 duration = 0
total_cost = 0 total_cost = 0
total_error_outputs = 0 total_error_outputs = 0
@ -177,7 +177,7 @@ def summarize_results(dirname):
completed_tests += 1 completed_tests += 1
passed = results["tests_outcomes"][-1] passed = results["tests_outcomes"][-1]
if passed: if passed:
for i in range(len(results["tests_outcomes"]) - 1, retries): for i in range(len(results["tests_outcomes"]) - 1, tries):
passed_tests[i] += 1 passed_tests[i] += 1
total_cost += results["cost"] total_cost += results["cost"]
@ -208,7 +208,7 @@ def summarize_results(dirname):
print("num_user_asks:", total_user_asks) print("num_user_asks:", total_user_asks)
console.print() console.print()
for i in range(retries): for i in range(tries):
pass_rate = 100 * passed_tests[i] / completed_tests pass_rate = 100 * passed_tests[i] / completed_tests
console.print(f"{pass_rate:.1f}% correct after try {i}") console.print(f"{pass_rate:.1f}% correct after try {i}")
@ -234,7 +234,7 @@ def summarize_results(dirname):
console.rule() console.rule()
def run_test(testdir, model_name, edit_format, retries, no_unit_tests, verbose, commit_hash): def run_test(testdir, model_name, edit_format, tries, no_unit_tests, verbose, commit_hash):
if not os.path.isdir(testdir): if not os.path.isdir(testdir):
print("Not a dir:", testdir) print("Not a dir:", testdir)
return return
@ -304,7 +304,7 @@ Only use standard python libraries, don't suggest installing any packages.
dur = 0 dur = 0
test_outcomes = [] test_outcomes = []
for i in range(retries): for i in range(tries):
start = time.time() start = time.time()
coder.run(with_message=instructions) coder.run(with_message=instructions)
dur += time.time() - start dur += time.time() - start