dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-06-01 10:14:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Record commit_hash

Browse source
This commit is contained in:
Paul Gauthier 2023-06-25 15:46:15 -07:00
parent 976f84bac5
commit 68640c9bb2
1 changed files with 14 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

19
scripts/benchmark.py
View file

@ -10,6 +10,7 @@ from collections import defaultdict
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError
from pathlib import Path from pathlib import Path
import git
import lox import lox
from rich.console import Console from rich.console import Console
@ -26,6 +27,11 @@ assert ORIGINAL_DNAME.exists() and ORIGINAL_DNAME.is_dir()
def main(): def main():
repo = git.Repo(search_parent_directories=True)
commit_hash = repo.head.object.hexsha[:7]
if repo.is_dirty():
commit_hash += "-dirty"
parser = argparse.ArgumentParser(description="Aider Benchmark") parser = argparse.ArgumentParser(description="Aider Benchmark")
parser.add_argument("dirname", type=str, help="Directory name") parser.add_argument("dirname", type=str, help="Directory name")
parser.add_argument("--model", "-m", type=str, help="Model name", default="gpt-3.5-turbo") parser.add_argument("--model", "-m", type=str, help="Model name", default="gpt-3.5-turbo")
@ -117,6 +123,7 @@ def main():
args.no_test, args.no_test,
args.verbose, args.verbose,
args.stats_only, args.stats_only,
commit_hash,
) )
all_results.append(results) all_results.append(results)
@ -133,6 +140,7 @@ def main():
args.no_test, args.no_test,
args.verbose, args.verbose,
args.stats_only, args.stats_only,
commit_hash,
) )
all_results = run_test_threaded.gather(tqdm=True) all_results = run_test_threaded.gather(tqdm=True)
@ -172,9 +180,9 @@ def summarize_results(dirname, all_results, total_tests=None):
total_cost += results["cost"] total_cost += results["cost"]
duration += results["duration"] duration += results["duration"]
for key in "model edit_format".split(): for key in "model edit_format commit_hash".split():
if key in results: val = results.get(key)
variants[key].add(results[key]) variants[key].add(val)
dump(completed_tests) dump(completed_tests)
if not completed_tests: if not completed_tests:
@ -189,7 +197,7 @@ def summarize_results(dirname, all_results, total_tests=None):
style = "red" style = "red"
else: else:
style = None style = None
val = ", ".join(val) val = ", ".join(map(str, val))
console.print(f"{key}: {val}", style=style) console.print(f"{key}: {val}", style=style)
console.print() console.print()
@ -213,7 +221,7 @@ def summarize_results(dirname, all_results, total_tests=None):
console.rule() console.rule()
def run_test(testdir, model_name, edit_format, retries, no_test, verbose, stats_only): def run_test(testdir, model_name, edit_format, retries, no_test, verbose, stats_only, commit_hash):
if not stats_only: if not stats_only:
dump(testdir) dump(testdir)
@ -319,6 +327,7 @@ def run_test(testdir, model_name, edit_format, retries, no_test, verbose, stats_
tests_outcomes=test_outcomes, tests_outcomes=test_outcomes,
cost=coder.total_cost, cost=coder.total_cost,
duration=dur, duration=dur,
commit_hash=commit_hash,
) )
dump(results) dump(results)