From 8cb83afcc413fdd0555fd544bac52d49768e769f Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Thu, 12 Sep 2024 17:21:18 -0700 Subject: [PATCH] ask transient whole, o1-preview deep --- benchmark/benchmark.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 7deb1d3c4..5ed06c20f 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -549,17 +549,18 @@ def run_test_real( chat_history_file=history_fname, ) + ask_model = models.Model("o1-preview") # ask_model = models.Model("gpt-4o") - ask_model = models.Model("openrouter/anthropic/claude-3.5-sonnet") + # ask_model = models.Model("openrouter/anthropic/claude-3.5-sonnet") # ask_model = models.Model("openrouter/deepseek/deepseek-coder") # whole_model = models.Model("gpt-4o") # whole_model = models.Model("openrouter/anthropic/claude-3.5-sonnet") - # whole_model = models.Model("openrouter/deepseek/deepseek-coder") + whole_model = models.Model("openrouter/deepseek/deepseek-coder") # whole_model = models.Model("openrouter/anthropic/claude-3-haiku-20240307") # whole_model = models.Model("gpt-4o-mini") # whole_model = models.Model("openrouter/meta-llama/llama-3.1-8b-instruct") - whole_model = models.Model("openrouter/meta-llama/llama-3-70b-instruct") + # whole_model = models.Model("openrouter/meta-llama/llama-3-70b-instruct") main_model = ask_model edit_format = "ask-whole"