From 5ee455a05b3b574e4c2638ff0e129ebbb1c0cbed Mon Sep 17 00:00:00 2001
From: Paul Gauthier <aider@paulg.org>
Date: Thu, 29 Jun 2023 14:19:00 -0700
Subject: [PATCH] docker environment for the whole benchmark

---
 .dockerignore             |  9 +++++++++
 benchmark/Dockerfile      |  7 +++++++
 benchmark/benchmark.py    |  8 ++++----
 benchmark/docker.sh       | 10 +++++++++-
 benchmark/docker_build.sh |  8 ++++++++
 5 files changed, 37 insertions(+), 5 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 benchmark/Dockerfile
 mode change 100644 => 100755 benchmark/docker.sh
 create mode 100755 benchmark/docker_build.sh

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 000000000..ce03212b5
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,9 @@
+.BC.*
+tmp*
+*~
+OLD*
+*.pyc
+.DS_Store
+.env
+.venv
+.aider.*
diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
new file mode 100644
index 000000000..b868000c2
--- /dev/null
+++ b/benchmark/Dockerfile
@@ -0,0 +1,7 @@
+FROM python:3.8-slim
+RUN apt-get update && apt-get install -y less git
+COPY requirements.txt /aider/requirements.txt
+RUN pip install --upgrade pip && pip install -r /aider/requirements.txt
+RUN pip install lox typer
+WORKDIR /aider
+
diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py
index ccb36b1df..1e83b10ca 100755
--- a/benchmark/benchmark.py
+++ b/benchmark/benchmark.py
@@ -23,9 +23,9 @@ from aider.coders import Coder
 from aider.dump import dump  # noqa: F401
 from aider.io import InputOutput
 
-BENCHMARK_DNAME = Path("tmp.benchmark/.")
+BENCHMARK_DNAME = Path("/benchmarks")
 
-ORIGINAL_DNAME = BENCHMARK_DNAME / "practice/."
+ORIGINAL_DNAME = BENCHMARK_DNAME / "exercism-python"
 
 app = typer.Typer(add_completion=False, pretty_exceptions_enable=False)
 
@@ -52,8 +52,8 @@ def main(
     threads: int = typer.Option(1, "--threads", "-t", help="Number of threads to run in parallel"),
     num_tests: int = typer.Option(-1, "--num-tests", "-n", help="Number of tests to run"),
 ):
-    assert BENCHMARK_DNAME.exists() and BENCHMARK_DNAME.is_dir()
-    assert ORIGINAL_DNAME.exists() and ORIGINAL_DNAME.is_dir()
+    assert BENCHMARK_DNAME.exists() and BENCHMARK_DNAME.is_dir(), BENCHMARK_DNAME
+    assert ORIGINAL_DNAME.exists() and ORIGINAL_DNAME.is_dir(), ORIGINAL_DNAME
 
     repo = git.Repo(search_parent_directories=True)
     commit_hash = repo.head.object.hexsha[:7]
diff --git a/benchmark/docker.sh b/benchmark/docker.sh
old mode 100644
new mode 100755
index d4660c441..51803404a
--- a/benchmark/docker.sh
+++ b/benchmark/docker.sh
@@ -1,2 +1,10 @@
 #!/bin/bash
-docker run -it --rm python:3.8 bash
+
+docker run \
+       -it --rm \
+       -v `pwd`:/aider \
+       -v `pwd`/tmp.benchmarks/.:/benchmarks \
+       -e OPENAI_API_KEY=$OPENAI_API_KEY \
+       -e HISTFILE=/aider/.bash_history \
+       aider-benchmark \
+       bash
diff --git a/benchmark/docker_build.sh b/benchmark/docker_build.sh
new file mode 100755
index 000000000..a6619bb5c
--- /dev/null
+++ b/benchmark/docker_build.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+docker build \
+       --file benchmark/Dockerfile \
+       -t aider-benchmark \
+       .