dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-23 22:04:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

track and report errors/asks during benchmarking

Browse source
This commit is contained in:
Paul Gauthier 2023-06-26 10:33:16 -07:00
parent cbfda391bb
commit 1370da14fb
2 changed files with 16 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
scripts/benchmark.py
View file

@ -180,7 +180,7 @@ def summarize_results(dirname, all_results, total_tests=None):
total_cost += results["cost"]
duration += results["duration"]
for key in "model edit_format commit_hash".split():
for key in "model edit_format commit_hash num_error_outputs num_user_asks".split():
val = results.get(key)
variants[key].add(val)
@ -339,6 +339,8 @@ Only use standard python libraries, don't suggest installing any packages.
cost=coder.total_cost,
duration=dur,
commit_hash=commit_hash,
num_error_outputs=io.num_error_outputs,
num_user_asks=io.num_user_asks,
)
dump(results)