dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-28 00:05:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Show variants

Browse source
This commit is contained in:
Paul Gauthier 2023-06-25 13:18:43 -07:00
parent 5b57024c84
commit 0c3d4198f5
1 changed files with 13 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

13
scripts/benchmark.py
View file

@ -6,6 +6,7 @@ import random
import shutil
import subprocess
import time
from collections import defaultdict
from json.decoder import JSONDecodeError
from pathlib import Path
@ -152,6 +153,8 @@ def summarize_results(all_results, total_tests=None):
duration = 0
total_cost = 0
variants = defaultdict(set)
for results in all_results:
if not results:
continue
@ -165,13 +168,23 @@ def summarize_results(all_results, total_tests=None):
total_cost += results["cost"]
duration += results["duration"]
for key in "model edit_format".split():
if key in results:
variants[key].add(results[key])
console.rule()
console.print(f"{completed_tests} test-cases")
for key, val in variants.items():
val = ", ".join(val)
console.print(f"{key}: {val}")
console.print()
for i in range(retries):
pass_rate = 100 * passed_tests[i] / completed_tests
console.print(f"{pass_rate:.1f}% correct after try {i}")
console.print()
avg_duration = duration / completed_tests
console.print(f"{avg_duration:.1f} sec/test-case")