dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-28 16:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

added timeout reporting

Browse source
This commit is contained in:
Paul Gauthier 2023-06-29 11:55:28 -07:00
parent b852c35828
commit 5eb1b2bdec
1 changed files with 25 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

19
benchmark/benchmark.py
View file

@ -168,6 +168,7 @@ def summarize_results(dirname):
total_cost = 0
total_error_outputs = 0
total_user_asks = 0
total_test_timeouts = 0
variants = defaultdict(set)
@ -183,6 +184,7 @@ def summarize_results(dirname):
total_cost += results["cost"]
duration += results["duration"]
total_test_timeouts += results.get("test_timeouts", 0)
total_error_outputs += results.get("num_error_outputs", 0)
total_user_asks += results.get("num_user_asks", 0)
@ -208,6 +210,9 @@ def summarize_results(dirname):
print("num_error_outputs:", total_error_outputs)
print("num_user_asks:", total_user_asks)
style = "red" if total_test_timeouts else None
console.print("test_timeouts:", total_test_timeouts, style=style)
console.print()
for i in range(tries):
pass_rate = 100 * passed_tests[i] / completed_tests
@ -300,6 +305,8 @@ def run_test(
verbose=verbose,
)
timeouts = 0
dur = 0
test_outcomes = []
for i in range(tries):
@ -314,7 +321,11 @@ def run_test(
if no_unit_tests:
break
try:
errors = run_unit_tests(testdir, history_fname)
except subprocess.TimeoutExpired:
errors = "Tests timed out!"
timeouts += 1
if errors:
test_outcomes.append(False)
@ -337,6 +348,7 @@ def run_test(
tests_outcomes=test_outcomes,
cost=coder.total_cost,
duration=dur,
test_timeouts=timeouts,
commit_hash=commit_hash,
num_error_outputs=io.num_error_outputs,
num_user_asks=io.num_user_asks,
@ -370,7 +382,7 @@ def run_unit_tests(testdir, history_fname):
print(" ".join(command))
timeout = 60
try:
result = subprocess.run(
command,
stdout=subprocess.PIPE,
@ -378,10 +390,7 @@ def run_unit_tests(testdir, history_fname):
text=True,
timeout=timeout,
)
except subprocess.TimeoutExpired:
success = False
res = f"Tests timed out after {timeout} secs!"
else:
success = result.returncode == 0
res = result.stdout
res = cleanup_test_output(res)