diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index d26bb55ed..b692073d3 100755
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -86,6 +86,10 @@ def main(
         summarize_results(dirname)
         return
 
+    if "AIDER_DOCKER" not in os.environ:
+        print("Warning: benchmarking runs unvetted code from GPT, run in a docker container")
+        return
+
     assert BENCHMARK_DNAME.exists() and BENCHMARK_DNAME.is_dir(), BENCHMARK_DNAME
     assert ORIGINAL_DNAME.exists() and ORIGINAL_DNAME.is_dir(), ORIGINAL_DNAME
 
diff --git a/benchmark/docker.sh b/benchmark/docker.sh
index 51803404a..0182cdd34 100755
--- a/benchmark/docker.sh
+++ b/benchmark/docker.sh
@@ -6,5 +6,6 @@ docker run \
        -v `pwd`/tmp.benchmarks/.:/benchmarks \
        -e OPENAI_API_KEY=$OPENAI_API_KEY \
        -e HISTFILE=/aider/.bash_history \
+       -e AIDER_DOCKER=1 \
        aider-benchmark \
        bash