dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-06-01 02:05:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #1200 from cheahjs/patch-1

Browse source 
docs: add benchmark results for new gemini experimental models
This commit is contained in:
paul-gauthier 2024-08-28 05:40:31 -07:00 • committed by GitHub
commit 70d05fc6f0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 69 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

68
aider/website/_data/edit_leaderboard.yml
View file

@ -879,3 +879,71 @@
seconds_per_case: 26.3
total_cost: 3.6113
- dirname: 2024-08-28-07-10-50--gemini-1.5-pro-exp-0827-diff-fenced
test_cases: 133
model: gemini-1.5-pro-exp-0827
edit_format: diff-fenced
commit_hash: d8adc75
pass_rate_1: 54.9
pass_rate_2: 66.9
percent_cases_well_formed: 94.7
error_outputs: 112
num_malformed_responses: 26
num_with_malformed_responses: 7
user_asks: 38
lazy_comments: 0
syntax_errors: 1
indentation_errors: 0
exhausted_context_windows: 0
test_timeouts: 1
command: aider --model gemini/gemini-1.5-pro-exp-0827
date: 2024-08-28
versions: 0.53.1-dev
seconds_per_case: 14.5
total_cost: 0.0000
- dirname: 2024-08-27-19-20-19--gemini-1.5-flash-exp-0827
test_cases: 133
model: gemini-1.5-flash-exp-0827
edit_format: whole
commit_hash: d8adc75
pass_rate_1: 40.6
pass_rate_2: 52.6
percent_cases_well_formed: 100.0
error_outputs: 1
num_malformed_responses: 0
num_with_malformed_responses: 0
user_asks: 1
lazy_comments: 3
syntax_errors: 1
indentation_errors: 0
exhausted_context_windows: 0
test_timeouts: 4
command: aider --model gemini/gemini-1.5-flash-exp-0827
date: 2024-08-27
versions: 0.53.1-dev
seconds_per_case: 6.3
total_cost: 0.0000
- dirname: 2024-08-27-19-42-05--gemini-1.5-flash-8b-exp-0827
test_cases: 133
model: gemini-1.5-flash-8b-exp-0827
edit_format: whole
commit_hash: d8adc75
pass_rate_1: 31.6
pass_rate_2: 38.3
percent_cases_well_formed: 100.0
error_outputs: 12
num_malformed_responses: 0
num_with_malformed_responses: 0
user_asks: 10
lazy_comments: 250
syntax_errors: 6
indentation_errors: 1
exhausted_context_windows: 0
test_timeouts: 0
command: aider --model gemini/gemini-1.5-flash-8b-exp-0827
date: 2024-08-27
versions: 0.53.1-dev
seconds_per_case: 7.2
total_cost: 0.0000