dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-30 09:14:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

senior-junior

Browse source
This commit is contained in:
Paul Gauthier 2024-09-25 08:25:11 -07:00
parent a9e9f9cdbe
commit c912982747
1 changed files with 23 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

49
benchmark/benchmark.py
View file

@ -549,29 +549,26 @@ def run_test_real(
chat_history_file=history_fname, chat_history_file=history_fname,
) )
ask_model = models.Model("o1-preview") # senior_model = models.Model("o1-mini")
# ask_model = models.Model("gpt-4o") # senior_model = models.Model("o1-preview")
# ask_model = models.Model("openrouter/anthropic/claude-3.5-sonnet") # senior_model = models.Model("gpt-4o")
# ask_model = models.Model("openrouter/deepseek/deepseek-coder") senior_model = models.Model("openrouter/anthropic/claude-3.5-sonnet")
# senior_model = models.Model("openrouter/deepseek/deepseek-chat")
# whole_model = models.Model("gpt-4o") # junior_model = models.Model("gpt-4o")
# whole_model = models.Model("openrouter/anthropic/claude-3.5-sonnet") # junior_model = models.Model("openrouter/anthropic/claude-3.5-sonnet")
whole_model = models.Model("openrouter/deepseek/deepseek-coder") junior_model = models.Model("openrouter/deepseek/deepseek-chat")
# whole_model = models.Model("openrouter/anthropic/claude-3-haiku-20240307") # junior_model = models.Model("openrouter/anthropic/claude-3-haiku-20240307")
# whole_model = models.Model("gpt-4o-mini") # junior_model = models.Model("gpt-4o")
# whole_model = models.Model("openrouter/meta-llama/llama-3.1-8b-instruct") # junior_model = models.Model("gpt-4o-mini")
# whole_model = models.Model("openrouter/meta-llama/llama-3-70b-instruct") # junior_model = models.Model("openrouter/meta-llama/llama-3.1-8b-instruct")
# junior_model = models.Model("openrouter/meta-llama/llama-3-70b-instruct")
main_model = ask_model junior_edit_format = "whole"
edit_format = "ask-whole"
# weak_model_name = model_name edit_format = "senior-junior-" + junior_edit_format
weak_model_name = None show_model_name = senior_model.name + "--" + junior_model.name
main_model = models.Model(model_name, weak_model=weak_model_name)
edit_format = edit_format or main_model.edit_format
dump(main_model)
dump(edit_format)
show_fnames = ",".join(map(str, fnames)) show_fnames = ",".join(map(str, fnames))
print("fnames:", show_fnames) print("fnames:", show_fnames)
@ -586,7 +583,7 @@ def run_test_real(
suggest_shell_commands=False, suggest_shell_commands=False,
) )
coder = Coder.create( coder = Coder.create(
main_model=ask_model, main_model=senior_model,
edit_format="ask", edit_format="ask",
**coder_kwargs, **coder_kwargs,
) )
@ -618,19 +615,19 @@ def run_test_real(
""" """
coder = Coder.create( coder = Coder.create(
from_coder=coder, from_coder=coder,
main_model=ask_model, main_model=senior_model,
edit_format="ask", edit_format="ask",
**coder_kwargs, **coder_kwargs,
) )
""" """
response = coder.run(with_message=instructions, preproc=False) response = coder.run(with_message=instructions, preproc=False)
whole_coder = Coder.create( junior_coder = Coder.create(
from_coder=coder, from_coder=coder,
main_model=whole_model, main_model=junior_model,
edit_format="whole", edit_format=junior_edit_format,
**coder_kwargs, **coder_kwargs,
) )
response = whole_coder.run(with_message="make those changes", preproc=False) response = junior_coder.run(with_message="make those changes", preproc=False)
dur += time.time() - start dur += time.time() - start
@ -676,7 +673,7 @@ def run_test_real(
results = dict( results = dict(
testdir=str(testdir), testdir=str(testdir),
testcase=testdir.name, testcase=testdir.name,
model=main_model.name, model=show_model_name,
edit_format=edit_format, edit_format=edit_format,
tests_outcomes=test_outcomes, tests_outcomes=test_outcomes,
cost=coder.total_cost, cost=coder.total_cost,