dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 08:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Show variants

Browse source
This commit is contained in:
Paul Gauthier 2023-06-25 13:18:43 -07:00
parent 5b57024c84
commit 0c3d4198f5
1 changed files with 13 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
scripts/benchmark.py
View file

@ -6,6 +6,7 @@ import random
import shutil import shutil
import subprocess import subprocess
import time import time
from collections import defaultdict
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError
from pathlib import Path from pathlib import Path
@ -152,6 +153,8 @@ def summarize_results(all_results, total_tests=None):
duration = 0 duration = 0
total_cost = 0 total_cost = 0
variants = defaultdict(set)
for results in all_results: for results in all_results:
if not results: if not results:
continue continue
@ -165,13 +168,23 @@ def summarize_results(all_results, total_tests=None):
total_cost += results["cost"] total_cost += results["cost"]
duration += results["duration"] duration += results["duration"]
for key in "model edit_format".split():
if key in results:
variants[key].add(results[key])
console.rule() console.rule()
console.print(f"{completed_tests} test-cases") console.print(f"{completed_tests} test-cases")
for key, val in variants.items():
val = ", ".join(val)
console.print(f"{key}: {val}")
console.print()
for i in range(retries): for i in range(retries):
pass_rate = 100 * passed_tests[i] / completed_tests pass_rate = 100 * passed_tests[i] / completed_tests
console.print(f"{pass_rate:.1f}% correct after try {i}") console.print(f"{pass_rate:.1f}% correct after try {i}")
console.print()
avg_duration = duration / completed_tests avg_duration = duration / completed_tests
console.print(f"{avg_duration:.1f} sec/test-case") console.print(f"{avg_duration:.1f} sec/test-case")