dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-22 05:14:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Added --stats-only

Browse source
This commit is contained in:
Paul Gauthier 2023-06-25 08:33:24 -07:00
parent 5eaa50fe6c
commit c5a232eeaf
1 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

16
scripts/benchmark.py
View file

@ -47,6 +47,12 @@ def main():
action="store_true", action="store_true",
help="Verbose output", help="Verbose output",
) )
parser.add_argument(
"--stats-only",
"-s",
action="store_true",
help="Do not run tests, just collect stats on completed tests",
)
parser.add_argument( parser.add_argument(
"--retries", "--retries",
"-r", "-r",
@ -109,9 +115,11 @@ def main():
args.retries, args.retries,
args.no_test, args.no_test,
args.verbose, args.verbose,
args.stats_only,
) )
all_results.append(results) all_results.append(results)
if not args.stats_only:
summarize_results(all_results, total_tests) summarize_results(all_results, total_tests)
else: else:
run_test_threaded = lox.thread(args.threads)(run_test) run_test_threaded = lox.thread(args.threads)(run_test)
@ -177,7 +185,7 @@ def summarize_results(all_results, total_tests=None):
console.rule() console.rule()
def run_test(testdir, model_name, edit_format, retries, no_test, verbose): def run_test(testdir, model_name, edit_format, retries, no_test, verbose, stats_only):
dump(testdir) dump(testdir)
if not os.path.isdir(testdir): if not os.path.isdir(testdir):
@ -191,11 +199,15 @@ def run_test(testdir, model_name, edit_format, retries, no_test, verbose):
results_fname = testdir / ".aider.results.json" results_fname = testdir / ".aider.results.json"
if results_fname.exists(): if results_fname.exists():
try: try:
return json.loads(results_fname.read_text()) res = json.loads(results_fname.read_text())
return res
except JSONDecodeError: except JSONDecodeError:
print(f"{results_fname} failed to parse, skipping") print(f"{results_fname} failed to parse, skipping")
return return
if stats_only:
return
fnames = [] fnames = []
for fname in testdir.glob("*"): for fname in testdir.glob("*"):
if "test" not in fname.name and fname.is_file() and fname.name[0] != ".": if "test" not in fname.name and fname.is_file() and fname.name[0] != ".":