examples_as_sys_msg for sonnet gets 100% compliance on exercism eval

This commit is contained in:
Paul Gauthier 2024-06-24 11:03:29 -07:00
parent 30154957bb
commit 42fbe28897
2 changed files with 13 additions and 9 deletions

View file

@ -1,12 +1,12 @@
import difflib
import json
import yaml
import math
import os
import sys
from dataclasses import dataclass, fields
from typing import Optional
import yaml
from PIL import Image
from aider import urls
@ -184,18 +184,21 @@ MODEL_SETTINGS = [
"diff",
weak_model_name="claude-3-haiku-20240307",
use_repo_map=True,
examples_as_sys_msg=True,
),
ModelSettings(
"anthropic/claude-3-5-sonnet-20240620",
"diff",
weak_model_name="claude-3-haiku-20240307",
use_repo_map=True,
examples_as_sys_msg=True,
),
ModelSettings(
"openrouter/anthropic/claude-3.5-sonnet",
"diff",
weak_model_name="openrouter/anthropic/claude-3-haiku-20240307",
use_repo_map=True,
examples_as_sys_msg=True,
),
# Vertex AI Claude models
ModelSettings(
@ -471,21 +474,24 @@ class Model:
return validate_variables(["GROQ_API_KEY"])
return res
def register_models(model_settings_fnames):
files_loaded = []
for model_settings_fname in model_settings_fnames:
if not os.path.exists(model_settings_fname):
continue
try:
with open(model_settings_fname, "r") as model_settings_file:
model_settings_list = yaml.safe_load(model_settings_file)
for model_settings_dict in model_settings_list:
model_settings = ModelSettings(**model_settings_dict)
existing_model_settings = next((ms for ms in MODEL_SETTINGS if ms.name == model_settings.name), None)
existing_model_settings = next(
(ms for ms in MODEL_SETTINGS if ms.name == model_settings.name), None
)
if existing_model_settings:
MODEL_SETTINGS.remove(existing_model_settings)
MODEL_SETTINGS.append(model_settings)
@ -501,14 +507,14 @@ def register_litellm_models(model_fnames):
for model_fname in model_fnames:
if not os.path.exists(model_fname):
continue
try:
with open(model_fname, "r") as model_def_file:
model_def = json.load(model_def_file)
litellm.register_model(model_def)
except Exception as e:
raise Exception(f"Error loading model definition from {model_fname}: {e}")
files_loaded.append(model_fname)
return files_loaded