diff --git a/aider/coders/base_coder.py b/aider/coders/base_coder.py
index ece25596d..a21cb420e 100755
--- a/aider/coders/base_coder.py
+++ b/aider/coders/base_coder.py
@@ -104,6 +104,7 @@ class Coder:
             fnames = []
 
         self.chat_completion_call_hashes = []
+        self.chat_completion_response_hashes = []
 
         self.verbose = verbose
         self.abs_fnames = set()
@@ -611,6 +612,13 @@ class Coder:
         except AttributeError as content_err:
             show_content_err = content_err
 
+        resp_hash = dict(
+            function_call=self.partial_response_function_call,
+            content=self.partial_response_content,
+        )
+        resp_hash = hashlib.sha1(json.dumps(resp_hash, sort_keys=True).encode())
+        self.chat_completion_response_hashes.append(resp_hash.hexdigest())
+
         if show_func_err and show_content_err:
             self.io.tool_error(show_func_err)
             self.io.tool_error(show_content_err)
diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index f45f12e29..60e482c2e 100755
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -65,7 +65,7 @@ def main(
         priors = list(BENCHMARK_DNAME.glob(f"*--{dirname}"))
         if len(priors) == 1 and stats_only:
             dirname = priors[0].name
-            print(f'Using pre-existing {dirname}')
+            print(f"Using pre-existing {dirname}")
         elif len(priors):
             if not make_new:
                 print(f"Prior runs of {dirname} exist, use --new or name one explicitly")
@@ -348,6 +348,8 @@ Fix the code in {file_list} to resolve the errors.
         commit_hash=commit_hash,
         num_error_outputs=io.num_error_outputs,
         num_user_asks=io.num_user_asks,
+        chat_completion_call_hashes=coder.chat_completion_call_hashes,
+        chat_completion_response_hashes=coder.chat_completion_response_hashes,
     )
     dump(results)
 
@@ -376,7 +378,7 @@ def run_unit_tests(testdir, history_fname):
             DOCKER_IMAGE,
             "bash",
             "-c",
-            f"pip install pytest && pytest /app/{test_file.name}",
+            f"pytest /app/{test_file.name}",
         ]
         print(" ".join(command))