dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-25 14:55:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

track num_exhausted_context_windows

Browse source
This commit is contained in:
Paul Gauthier 2023-06-29 15:32:45 -07:00
parent 995fdc3415
commit b11a6fa7ee
2 changed files with 8 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
benchmark/benchmark.py
View file

@ -174,6 +174,7 @@ def summarize_results(dirname):
total_error_outputs = 0
total_user_asks = 0
total_test_timeouts = 0
num_exhausted_context_windows = 0
variants = defaultdict(set)
@ -193,6 +194,7 @@ def summarize_results(dirname):
total_error_outputs += results.get("num_error_outputs", 0)
total_user_asks += results.get("num_user_asks", 0)
num_exhausted_context_windows += results.get("num_exhausted_context_windows", 0)
for key in "model edit_format commit_hash".split():
val = results.get(key)
@ -215,6 +217,9 @@ def summarize_results(dirname):
print("num_error_outputs:", total_error_outputs)
print("num_user_asks:", total_user_asks)
style = "red" if num_exhausted_context_windows else None
print("num_exhausted_context_windows", num_exhausted_context_windows, style=style)
style = "red" if total_test_timeouts else None
console.print("test_timeouts:", total_test_timeouts, style=style)
@ -352,6 +357,7 @@ def run_test(
commit_hash=commit_hash,
num_error_outputs=io.num_error_outputs,
num_user_asks=io.num_user_asks,
num_exhausted_context_windows=coder.num_exhausted_context_windows,
chat_hashes=list(
zip(
coder.chat_completion_call_hashes,