dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-30 01:04:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

added timeout reporting

Browse source
This commit is contained in:
Paul Gauthier 2023-06-29 11:55:28 -07:00
parent b852c35828
commit 5eb1b2bdec
1 changed files with 25 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

41
benchmark/benchmark.py
View file

@ -168,6 +168,7 @@ def summarize_results(dirname):
total_cost = 0 total_cost = 0
total_error_outputs = 0 total_error_outputs = 0
total_user_asks = 0 total_user_asks = 0
total_test_timeouts = 0
variants = defaultdict(set) variants = defaultdict(set)
@ -183,6 +184,7 @@ def summarize_results(dirname):
total_cost += results["cost"] total_cost += results["cost"]
duration += results["duration"] duration += results["duration"]
total_test_timeouts += results.get("test_timeouts", 0)
total_error_outputs += results.get("num_error_outputs", 0) total_error_outputs += results.get("num_error_outputs", 0)
total_user_asks += results.get("num_user_asks", 0) total_user_asks += results.get("num_user_asks", 0)
@ -208,6 +210,9 @@ def summarize_results(dirname):
print("num_error_outputs:", total_error_outputs) print("num_error_outputs:", total_error_outputs)
print("num_user_asks:", total_user_asks) print("num_user_asks:", total_user_asks)
style = "red" if total_test_timeouts else None
console.print("test_timeouts:", total_test_timeouts, style=style)
console.print() console.print()
for i in range(tries): for i in range(tries):
pass_rate = 100 * passed_tests[i] / completed_tests pass_rate = 100 * passed_tests[i] / completed_tests
@ -300,6 +305,8 @@ def run_test(
verbose=verbose, verbose=verbose,
) )
timeouts = 0
dur = 0 dur = 0
test_outcomes = [] test_outcomes = []
for i in range(tries): for i in range(tries):
@ -314,7 +321,11 @@ def run_test(
if no_unit_tests: if no_unit_tests:
break break
errors = run_unit_tests(testdir, history_fname) try:
errors = run_unit_tests(testdir, history_fname)
except subprocess.TimeoutExpired:
errors = "Tests timed out!"
timeouts += 1
if errors: if errors:
test_outcomes.append(False) test_outcomes.append(False)
@ -337,6 +348,7 @@ def run_test(
tests_outcomes=test_outcomes, tests_outcomes=test_outcomes,
cost=coder.total_cost, cost=coder.total_cost,
duration=dur, duration=dur,
test_timeouts=timeouts,
commit_hash=commit_hash, commit_hash=commit_hash,
num_error_outputs=io.num_error_outputs, num_error_outputs=io.num_error_outputs,
num_user_asks=io.num_user_asks, num_user_asks=io.num_user_asks,
@ -370,21 +382,18 @@ def run_unit_tests(testdir, history_fname):
print(" ".join(command)) print(" ".join(command))
timeout = 60 timeout = 60
try:
result = subprocess.run( result = subprocess.run(
command, command,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
text=True, text=True,
timeout=timeout, timeout=timeout,
) )
except subprocess.TimeoutExpired:
success = False success = result.returncode == 0
res = f"Tests timed out after {timeout} secs!" res = result.stdout
else: res = cleanup_test_output(res)
success = result.returncode == 0
res = result.stdout
res = cleanup_test_output(res)
with history_fname.open("a") as fh: with history_fname.open("a") as fh:
fh.write(f"```\n{res}\n```") fh.write(f"```\n{res}\n```")