dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-25 06:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

track num_exhausted_context_windows

Browse source
This commit is contained in:
Paul Gauthier 2023-06-29 15:32:45 -07:00
parent 995fdc3415
commit b11a6fa7ee
2 changed files with 8 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
aider/coders/base_coder.py
View file

@ -44,6 +44,7 @@ class Coder:
repo_map = None repo_map = None
functions = None functions = None
total_cost = 0.0 total_cost = 0.0
num_exhausted_context_windows = 0
@classmethod @classmethod
def create( def create(
@ -447,6 +448,7 @@ class Coder:
exhausted = True exhausted = True
if exhausted: if exhausted:
self.num_exhausted_context_windows += 1
self.io.tool_error("The chat session is larger than the context window!\n") self.io.tool_error("The chat session is larger than the context window!\n")
self.commands.cmd_tokens("") self.commands.cmd_tokens("")
self.io.tool_error("\nTo reduce token usage:") self.io.tool_error("\nTo reduce token usage:")

6
benchmark/benchmark.py
View file

@ -174,6 +174,7 @@ def summarize_results(dirname):
total_error_outputs = 0 total_error_outputs = 0
total_user_asks = 0 total_user_asks = 0
total_test_timeouts = 0 total_test_timeouts = 0
num_exhausted_context_windows = 0
variants = defaultdict(set) variants = defaultdict(set)
@ -193,6 +194,7 @@ def summarize_results(dirname):
total_error_outputs += results.get("num_error_outputs", 0) total_error_outputs += results.get("num_error_outputs", 0)
total_user_asks += results.get("num_user_asks", 0) total_user_asks += results.get("num_user_asks", 0)
num_exhausted_context_windows += results.get("num_exhausted_context_windows", 0)
for key in "model edit_format commit_hash".split(): for key in "model edit_format commit_hash".split():
val = results.get(key) val = results.get(key)
@ -215,6 +217,9 @@ def summarize_results(dirname):
print("num_error_outputs:", total_error_outputs) print("num_error_outputs:", total_error_outputs)
print("num_user_asks:", total_user_asks) print("num_user_asks:", total_user_asks)
style = "red" if num_exhausted_context_windows else None
print("num_exhausted_context_windows", num_exhausted_context_windows, style=style)
style = "red" if total_test_timeouts else None style = "red" if total_test_timeouts else None
console.print("test_timeouts:", total_test_timeouts, style=style) console.print("test_timeouts:", total_test_timeouts, style=style)
@ -352,6 +357,7 @@ def run_test(
commit_hash=commit_hash, commit_hash=commit_hash,
num_error_outputs=io.num_error_outputs, num_error_outputs=io.num_error_outputs,
num_user_asks=io.num_user_asks, num_user_asks=io.num_user_asks,
num_exhausted_context_windows=coder.num_exhausted_context_windows,
chat_hashes=list( chat_hashes=list(
zip( zip(
coder.chat_completion_call_hashes, coder.chat_completion_call_hashes,