dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-24 14:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

support --keywords

Browse source
This commit is contained in:
Paul Gauthier 2023-08-09 11:12:27 -03:00
parent 00512e3d1c
commit 2eeee67ee5
1 changed files with 13 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

17
benchmark/benchmark.py
View file

@ -239,8 +239,8 @@ def main(
dirnames: List[str] = typer.Argument(..., help="Directory names"), dirnames: List[str] = typer.Argument(..., help="Directory names"),
model: str = typer.Option("gpt-3.5-turbo", "--model", "-m", help="Model name"), model: str = typer.Option("gpt-3.5-turbo", "--model", "-m", help="Model name"),
edit_format: str = typer.Option(None, "--edit-format", "-e", help="Edit format"), edit_format: str = typer.Option(None, "--edit-format", "-e", help="Edit format"),
keyword: str = typer.Option( keywords: str = typer.Option(
None, "--keyword", "-k", help="Only run tests that contain keyword" None, "--keywords", "-k", help="Only run tests that contain keywords (comma sep)"
), ),
clean: bool = typer.Option( clean: bool = typer.Option(
False, "--clean", "-c", help="Discard the existing testdir and make a clean copy" False, "--clean", "-c", help="Discard the existing testdir and make a clean copy"
@ -311,8 +311,9 @@ def main(
test_dnames = sorted(os.listdir(dirname)) test_dnames = sorted(os.listdir(dirname))
if keyword: if keywords:
test_dnames = [dn for dn in test_dnames if keyword in dn] keywords = keywords.split(",")
test_dnames = [dn for dn in test_dnames for keyword in keywords if keyword in dn]
random.shuffle(test_dnames) random.shuffle(test_dnames)
if num_tests > 0: if num_tests > 0:
@ -367,6 +368,8 @@ def show_diffs(dirnames):
testcases = sorted(testcases) testcases = sorted(testcases)
unchanged = set()
for testcase in testcases: for testcase in testcases:
all_outcomes = [] all_outcomes = []
for dirname in dirnames: for dirname in dirnames:
@ -377,6 +380,7 @@ def show_diffs(dirnames):
all_outcomes.append(outcomes) all_outcomes.append(outcomes)
if len(set(all_outcomes)) == 1: if len(set(all_outcomes)) == 1:
unchanged.add(testcase)
continue continue
print() print()
@ -384,6 +388,11 @@ def show_diffs(dirnames):
for outcome, dirname in zip(all_outcomes, dirnames): for outcome, dirname in zip(all_outcomes, dirnames):
print(outcome, f"{dirname}/{testcase}/.aider.chat.history.md") print(outcome, f"{dirname}/{testcase}/.aider.chat.history.md")
changed = set(testcases) - unchanged
print()
print("changed:", len(changed), ",".join(sorted(changed)))
print("unchanged:", len(unchanged), ",".join(sorted(unchanged)))
def load_results(dirname): def load_results(dirname):
dirname = Path(dirname) dirname = Path(dirname)