move benchmark prompts into their own file

This commit is contained in:
Paul Gauthier 2023-06-28 18:25:39 -07:00
parent 0845de7a7d
commit d1f0590911
2 changed files with 19 additions and 14 deletions

View file

@ -14,6 +14,7 @@ from pathlib import Path
import git
import lox
import prompts
import typer
from rich.console import Console
@ -268,12 +269,7 @@ def run_test(testdir, model_name, edit_format, tries, no_unit_tests, verbose, co
else:
instructions = ""
instructions += (testdir / ".docs/instructions.md").read_text()
instructions += f"""
=====
Use the above instructions to modify the supplied files: {file_list}
Keep and implement the existing function or class stubs, they will be called from unit tests.
Only use standard python libraries, don't suggest installing any packages.
"""
instructions += prompts.instructions_addendum.format(file_list=file_list)
io = InputOutput(
pretty=True,
@ -327,14 +323,7 @@ Only use standard python libraries, don't suggest installing any packages.
errors = errors[:50]
errors = "\n".join(errors)
instructions = errors
instructions += f"""
####
See the testing errors above.
The tests are correct.
Fix the code in {file_list} to resolve the errors.
"""
instructions += prompts.test_failures.format(file_list=file_list)
results = dict(
testdir=str(testdir),

16
benchmark/prompts.py Normal file
View file

@ -0,0 +1,16 @@
instructions_addendum = """
####
Use the above instructions to modify the supplied files: {file_list}
Keep and implement the existing function or class stubs, they will be called from unit tests.
Only use standard python libraries, don't suggest installing any packages.
"""
test_failures = """
####
See the testing errors above.
The tests are correct.
Fix the code in {file_list} to resolve the errors.
"""