feat: Add openrouter/openai/o1 model and update prompts

benchmark/Dockerfile

@@ -15,6 +15,17 @@ RUN apt-get update && apt-get install -y \
 # Make python3.11 the default python3
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
 
+# Install Go
+RUN curl -OL https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go1.21.5.linux-amd64.tar.gz && \
+    rm go1.21.5.linux-amd64.tar.gz
+ENV PATH="/usr/local/go/bin:${PATH}"
+
+# Install Rust
+ADD https://sh.rustup.rs /tmp/rustup.sh
+RUN chmod +x /tmp/rustup.sh && /tmp/rustup.sh -y && rm /tmp/rustup.sh
+ENV PATH="/root/.cargo/bin:${PATH}"
+
 # Install Node.js and dependencies
 RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
     apt-get install -y nodejs && \
@@ -33,17 +44,6 @@ RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
     core-js@3.37.1 \
     eslint@8.49.0
 
-# Install Go
-RUN curl -OL https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && \
-    tar -C /usr/local -xzf go1.21.5.linux-amd64.tar.gz && \
-    rm go1.21.5.linux-amd64.tar.gz
-ENV PATH="/usr/local/go/bin:${PATH}"
-
-# Install Rust
-ADD https://sh.rustup.rs /tmp/rustup.sh
-RUN chmod +x /tmp/rustup.sh && /tmp/rustup.sh -y && rm /tmp/rustup.sh
-ENV PATH="/root/.cargo/bin:${PATH}"
-
 COPY . /aider
 RUN pip3 install --no-cache-dir --upgrade pip uv
 RUN uv pip install --system --no-cache-dir -e /aider[dev]

benchmark/benchmark.py

@@ -416,6 +416,7 @@ def load_results(dirname):
             results = json.loads(fname.read_text())
             all_results.append(results)
         except json.JSONDecodeError:
+            print("json.JSONDecodeError", fname)
             continue
     return all_results
 
@@ -655,7 +656,9 @@ def run_test_real(
         config = json.loads(f.read())
 
     # Get solution and test files from config
-    solution_files = config.get("files", {}).get("solution", [])
+    solution_files = set(config.get("files", {}).get("solution", []))
+    solution_files.discard("Cargo.toml")
+
     test_files = config.get("files", {}).get("test", [])
 
     # Copy all solution files
@@ -743,7 +746,10 @@ def run_test_real(
         # auto_lint=False,  # disabled for code-in-json experiments
         cache_prompts=True,
         suggest_shell_commands=False,
+        ignore_mentions=set(test_files),
     )
+    dump(coder.ignore_mentions)
+
     coder.max_apply_update_errors = max_apply_update_errors
     coder.show_announcements()
 
@@ -851,7 +857,7 @@ def run_unit_tests(original_dname, testdir, history_fname, test_files):
     # Map of file extensions to test commands
     TEST_COMMANDS = {
         ".py": ["pytest"],
-        ".rs": ["cargo", "test", "--", "--include-ignored"],
+        ".rs": ["cargo", "test", "--offline", "--", "--include-ignored"],
         ".cs": ["dotnet", "test"],
         ".go": ["go", "test", "./..."],
         ".js": ["/aider/benchmark/npm-test.sh"],

benchmark/prompts.py

@@ -3,7 +3,7 @@ instructions_addendum = """
 
 Use the above instructions to modify the supplied files: {file_list}
 Don't change the names of existing functions or classes, as they may be referenced from other code like unit tests, etc.
-Only use standard python libraries, don't suggest installing any packages.
+Only use standard libraries, don't suggest installing any packages.
 """  # noqa: E501
 
 
@@ -11,6 +11,6 @@ test_failures = """
 ####
 
 See the testing errors above.
-The tests are correct.
+The tests are correct, don't try and change them.
 Fix the code in {file_list} to resolve the errors.
 """