dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-30 09:14:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Add pass rates to CSV output in benchmark results summary.

Browse source
This commit is contained in:
Paul Gauthier 2024-05-04 07:13:40 -07:00
parent d8a18f2c67
commit 01282674d4
1 changed files with 16 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

16
benchmark/benchmark.py
View file

@ -848,8 +848,10 @@ def summarize_results(dirname):
show("test_timeouts") show("test_timeouts")
console.print() console.print()
percents = dict()
for i in range(tries): for i in range(tries):
pass_rate = 100 * passed_tests[i] / res.completed_tests pass_rate = 100 * passed_tests[i] / res.completed_tests
percents[i] = pass_rate
console.print(f"{pass_rate:.1f}% correct after try {i}") console.print(f"{pass_rate:.1f}% correct after try {i}")
setattr(res, f"pass_rate_{i+1}", pass_rate) setattr(res, f"pass_rate_{i+1}", pass_rate)
@ -867,6 +869,20 @@ def summarize_results(dirname):
f" ${projected_cost:.2f} projected" f" ${projected_cost:.2f} projected"
) )
csv = []
csv.append(' '.join(variants['model']))
csv.append(f"{percents[1]:.1f}")
csv.append(f"{percents[0]:.1f}")
csv.append(' '.join(variants['edit_format']))
csv.append('aider')
csv.append('version')
for hsh in variants['commit_hash']):
# TODO: get the output from `git show {hsh}:aider/__init__.py`
csv.append(' '.join(variants['commit_hash']))
csv.append(dirname.name[:10])
csv = ','.join(csv)
print()
print(csv)
console.rule() console.rule()
# print(json.dumps(vars(res), indent=4, sort_keys=True)) # print(json.dumps(vars(res), indent=4, sort_keys=True))