From 8c29ecff28ebd37c3fe3774232155dc628da968b Mon Sep 17 00:00:00 2001
From: Paul Gauthier <aider@paulg.org>
Date: Sat, 1 Jul 2023 06:21:22 -0700
Subject: [PATCH] added rungrid

---
 benchmark/rungrid.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100755 benchmark/rungrid.py

diff --git a/benchmark/rungrid.py b/benchmark/rungrid.py
new file mode 100755
index 000000000..3a3d54323
--- /dev/null
+++ b/benchmark/rungrid.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import sys
+
+from aider.dump import dump
+from benchmark import main as benchmark_main
+
+
+def main():
+    models = [
+        "gpt-3.5-turbo-0301",
+        "gpt-3.5-turbo-0613",
+        "gpt-3.5-turbo-16k-0613",
+    ]
+    edit_formats = [
+        "diff",
+        "diff-func",
+        "whole",
+        "whole-func",
+    ]
+
+    for model in models:
+        for edit_format in edit_formats:
+            # dump(model, edit_format)
+            dirname = f"/benchmarks/{model}-{edit_format}"
+            dump(dirname)
+
+            benchmark_main(
+                dirnames=[dirname],
+                model=model,
+                edit_format=edit_format,
+                threads=10,
+                cont=True,
+            )
+
+
+if __name__ == "__main__":
+    status = main()
+    sys.exit(status)