dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-21 21:04:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Added --stats-only

Browse source
This commit is contained in:
Paul Gauthier 2023-06-25 08:33:24 -07:00
parent 5eaa50fe6c
commit c5a232eeaf
1 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

18
scripts/benchmark.py
View file

@ -47,6 +47,12 @@ def main():
action="store_true",
help="Verbose output",
)
parser.add_argument(
"--stats-only",
"-s",
action="store_true",
help="Do not run tests, just collect stats on completed tests",
)
parser.add_argument(
"--retries",
"-r",
@ -109,10 +115,12 @@ def main():
args.retries,
args.no_test,
args.verbose,
args.stats_only,
)
all_results.append(results)
summarize_results(all_results, total_tests)
if not args.stats_only:
summarize_results(all_results, total_tests)
else:
run_test_threaded = lox.thread(args.threads)(run_test)
for testname in test_dnames:
@ -177,7 +185,7 @@ def summarize_results(all_results, total_tests=None):
console.rule()
def run_test(testdir, model_name, edit_format, retries, no_test, verbose):
def run_test(testdir, model_name, edit_format, retries, no_test, verbose, stats_only):
dump(testdir)
if not os.path.isdir(testdir):
@ -191,11 +199,15 @@ def run_test(testdir, model_name, edit_format, retries, no_test, verbose):
results_fname = testdir / ".aider.results.json"
if results_fname.exists():
try:
return json.loads(results_fname.read_text())
res = json.loads(results_fname.read_text())
return res
except JSONDecodeError:
print(f"{results_fname} failed to parse, skipping")
return
if stats_only:
return
fnames = []
for fname in testdir.glob("*"):
if "test" not in fname.name and fname.is_file() and fname.name[0] != ".":