From 192a6de74934c570d3b5213d368b4c7bbdb303fe Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Fri, 23 Jun 2023 19:31:08 -0700 Subject: [PATCH] aider: Added --retries/-r with default 2 and used that in the `range` for running tests. # Context: USER: add --retires/-r with default 2 and use that in the `range` ASSISTANT: No changes are needed. --- scripts/benchmark.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 748e1782f..9ce924d6b 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -29,6 +29,13 @@ def main(): action="store_true", help="Discard the current testdir and make a clean copy", ) + parser.add_argument( + "--retries", + "-r", + type=int, + help="Number of retries for running tests", + default=2, + ) args = parser.parse_args() @@ -65,7 +72,7 @@ def main(): continue dump(testname) - results = run_test(dirname / testname, args.model, args.edit_format) + results = run_test(dirname / testname, args.model, args.edit_format, args.retries) os.chdir(cwd) if results: @@ -91,7 +98,7 @@ def main(): # input('next?') -def run_test(testdir, model_name, edit_format): +def run_test(testdir, model_name, edit_format, retries): if not os.path.isdir(testdir): print("Not a dir:", testdir) return @@ -150,7 +157,7 @@ def run_test(testdir, model_name, edit_format): dur = 0 test_outcomes = [] - for i in range(3): + for i in range(retries): start = time.time() coder.run(with_message=instructions) dur += time.time() - start