From b11a6fa7ee8f0f8fd2032cf71a7074bd9ab5be57 Mon Sep 17 00:00:00 2001
From: Paul Gauthier <aider@paulg.org>
Date: Thu, 29 Jun 2023 15:32:45 -0700
Subject: [PATCH] track num_exhausted_context_windows

---
 aider/coders/base_coder.py | 2 ++
 benchmark/benchmark.py     | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/aider/coders/base_coder.py b/aider/coders/base_coder.py
index 7b578c0bb..e1f2a567c 100755
--- a/aider/coders/base_coder.py
+++ b/aider/coders/base_coder.py
@@ -44,6 +44,7 @@ class Coder:
     repo_map = None
     functions = None
     total_cost = 0.0
+    num_exhausted_context_windows = 0
 
     @classmethod
     def create(
@@ -447,6 +448,7 @@ class Coder:
                 exhausted = True
 
         if exhausted:
+            self.num_exhausted_context_windows += 1
             self.io.tool_error("The chat session is larger than the context window!\n")
             self.commands.cmd_tokens("")
             self.io.tool_error("\nTo reduce token usage:")
diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index b692073d3..e57e9d4a3 100755
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -174,6 +174,7 @@ def summarize_results(dirname):
     total_error_outputs = 0
     total_user_asks = 0
     total_test_timeouts = 0
+    num_exhausted_context_windows = 0
 
     variants = defaultdict(set)
 
@@ -193,6 +194,7 @@ def summarize_results(dirname):
 
         total_error_outputs += results.get("num_error_outputs", 0)
         total_user_asks += results.get("num_user_asks", 0)
+        num_exhausted_context_windows += results.get("num_exhausted_context_windows", 0)
 
         for key in "model edit_format commit_hash".split():
             val = results.get(key)
@@ -215,6 +217,9 @@ def summarize_results(dirname):
     print("num_error_outputs:", total_error_outputs)
     print("num_user_asks:", total_user_asks)
 
+    style = "red" if num_exhausted_context_windows else None
+    print("num_exhausted_context_windows", num_exhausted_context_windows, style=style)
+
     style = "red" if total_test_timeouts else None
     console.print("test_timeouts:", total_test_timeouts, style=style)
 
@@ -352,6 +357,7 @@ def run_test(
         commit_hash=commit_hash,
         num_error_outputs=io.num_error_outputs,
         num_user_asks=io.num_user_asks,
+        num_exhausted_context_windows=coder.num_exhausted_context_windows,
         chat_hashes=list(
             zip(
                 coder.chat_completion_call_hashes,