dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-22 21:34:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

refac

Browse source
This commit is contained in:
Paul Gauthier 2023-06-24 20:22:10 -07:00
parent 6172a70ed3
commit 959861ee2e
1 changed files with 20 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

34
scripts/benchmark.py
View file

@ -72,13 +72,7 @@ def main():
test_dnames = sorted(os.listdir(dirname)) test_dnames = sorted(os.listdir(dirname))
total_tests = len(test_dnames) all_results = []
completed_tests = 0
passed_tests = [0] * args.retries
duration = 0
total_cost = 0
for testname in test_dnames: for testname in test_dnames:
if args.keyword and args.keyword not in testname: if args.keyword and args.keyword not in testname:
continue continue
@ -94,17 +88,33 @@ def main():
) )
os.chdir(cwd) os.chdir(cwd)
all_results.append(results)
summarize_results(all_results)
def summarize_results(all_results, total_tests=None):
if not total_tests:
total_tests = len(all_results)
completed_tests = 0
retries = max(len(results["tests_outcomes"]) for results in all_results if results)
passed_tests = [0] * retries
duration = 0
total_cost = 0
for results in all_results:
if not results: if not results:
continue continue
completed_tests += 1 completed_tests += 1
passed = results["tests_outcomes"][-1] passed = results["tests_outcomes"][-1]
if passed: if passed:
for i in range(len(results["tests_outcomes"]) - 1, args.retries): for i in range(len(results["tests_outcomes"]) - 1, retries):
passed_tests[i] += 1 passed_tests[i] += 1
dump(completed_tests, total_tests) dump(completed_tests, total_tests)
for i in range(args.retries): for i in range(retries):
pass_rate = 100 * passed_tests[i] / completed_tests pass_rate = 100 * passed_tests[i] / completed_tests
dump(i, pass_rate) dump(i, pass_rate)
@ -131,11 +141,6 @@ def main():
print() print()
###
# input('next?')
print(dirname / testname)
def run_test(testdir, model_name, edit_format, retries, no_test, verbose): def run_test(testdir, model_name, edit_format, retries, no_test, verbose):
if not os.path.isdir(testdir): if not os.path.isdir(testdir):
@ -230,6 +235,7 @@ def run_test(testdir, model_name, edit_format, retries, no_test, verbose):
results = dict( results = dict(
testdir=str(testdir), testdir=str(testdir),
testcase=testdir.name,
model=main_model.name, model=main_model.name,
edit_format=edit_format, edit_format=edit_format,
tests_outcomes=test_outcomes, tests_outcomes=test_outcomes,