dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-28 16:25:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat: Add --stats-languages option to filter results

Browse source
This commit is contained in:
Paul Gauthier (aider) 2024-12-20 16:03:19 -08:00
parent bddf6e9017
commit e3ac8ab19d
1 changed files with 14 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

20
benchmark/benchmark.py
View file

@ -195,6 +195,9 @@ def main(
stats_only: bool = typer.Option(
False, "--stats", "-s", help="Do not run tests, just collect stats on completed tests"
),
stats_languages: str = typer.Option(
None, "--stats-languages", help="Only include stats for specific languages (comma separated)"
),
diffs_only: bool = typer.Option(False, "--diffs", help="Just diff the provided stats dirs"),
tries: int = typer.Option(2, "--tries", "-r", help="Number of tries for running tests"),
threads: int = typer.Option(1, "--threads", "-t", help="Number of threads to run in parallel"),
@ -413,13 +416,18 @@ def show_diffs(dirnames):
print("unchanged:", len(unchanged), ",".join(sorted(unchanged)))
def load_results(dirname):
def load_results(dirname, stats_languages=None):
dirname = Path(dirname)
all_results = []
# do this ai!
# add a --stats-languages switch that takes like java,cpp,python
# limit the glob here to those languages followed by /exercises/practice/*/.aider.results.json
for fname in dirname.glob("*/exercises/practice/*/.aider.results.json"):
if stats_languages:
languages = [lang.strip().lower() for lang in stats_languages.split(",")]
glob_patterns = [f"{lang}/exercises/practice/*/.aider.results.json" for lang in languages]
else:
glob_patterns = ["*/exercises/practice/*/.aider.results.json"]
for pattern in glob_patterns:
for fname in dirname.glob(pattern):
try:
results = json.loads(fname.read_text())
all_results.append(results)
@ -430,7 +438,7 @@ def load_results(dirname):
def summarize_results(dirname):
all_results = load_results(dirname)
all_results = load_results(dirname, stats_languages)
res = SimpleNamespace()
res.total_tests = len(list(Path(dirname).glob("*/exercises/practice/*")))