mirror of
https://github.com/Aider-AI/aider.git
synced 2025-06-16 09:34:59 +00:00
Merge branch 'main' into mixpanel
This commit is contained in:
Paul Gauthier
commit
93b8cb9cec
24 changed files with 2527 additions and 313 deletions
|
|
@ -16,6 +16,15 @@ cog.out(text)
|
|||
|
||||
# Release history
|
||||
|
||||
### main branch
|
||||
|
||||
- Improved editing performance on Jupyter Notebook `.ipynb` files.
|
||||
- Work around litellm tokenizer bug for images.
|
||||
|
||||
### Aider v0.50.1
|
||||
|
||||
- Bugfix for provider API exceptions.
|
||||
|
||||
### Aider v0.50.0
|
||||
|
||||
- Infinite output for DeepSeek Coder, Mistral models in addition to Anthropic's models.
|
||||
|
|
|
|||
927
aider/website/_data/code-in-json.yml
Normal file
927
aider/website/_data/code-in-json.yml
Normal file
|
|
@ -0,0 +1,927 @@
|
|||
- dirname: 2024-08-15-13-17-11--json-no-lint-gpt-4o-2024-08-06-whole
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 4.3
|
||||
total_cost: 0.7965
|
||||
- dirname: 2024-08-15-13-18-36--json-no-lint-gpt-4o-2024-08-06-func
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 57.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 1
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 5.7
|
||||
total_cost: 0.8417
|
||||
- dirname: 2024-08-15-13-21-55--json-no-lint-gpt-4o-2024-05-13-func
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 2
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 1
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 7.1
|
||||
total_cost: 1.2285
|
||||
- dirname: 2024-08-15-13-23-33--json-no-lint-claude-3.5-sonnet-whole
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 10.5
|
||||
total_cost: 1.6714
|
||||
- dirname: 2024-08-15-13-26-38--json-no-lint-deepseek-coder-whole
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 59.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 2
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 2
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 27.9
|
||||
total_cost: 0.0438
|
||||
- dirname: 2024-08-15-13-50-03--json-no-lint-gpt-4o-2024-08-06-whole-2
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 61.7
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 4.2
|
||||
total_cost: 0.7946
|
||||
- dirname: 2024-08-15-13-51-36--json-no-lint-gpt-4o-2024-08-06-func-2
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 56.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 1
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 6.4
|
||||
total_cost: 0.8390
|
||||
- dirname: 2024-08-15-13-54-53--json-no-lint-gpt-4o-2024-05-13-func-2
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 1
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 7.7
|
||||
total_cost: 1.2210
|
||||
- dirname: 2024-08-15-13-56-21--json-no-lint-claude-3.5-sonnet-whole-2
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 1
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 16.5
|
||||
total_cost: 1.6556
|
||||
- dirname: 2024-08-15-14-06-12--json-no-lint-deepseek-coder-whole-2
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 2
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 1
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 25.8
|
||||
total_cost: 0.0439
|
||||
- dirname: 2024-08-15-14-11-45--json-no-lint-gpt-4o-2024-08-06-whole-3
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 4.3
|
||||
total_cost: 0.7945
|
||||
- dirname: 2024-08-15-14-13-11--json-no-lint-gpt-4o-2024-08-06-func-3
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 56.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 1
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 5.6
|
||||
total_cost: 0.8220
|
||||
- dirname: 2024-08-15-14-16-34--json-no-lint-gpt-4o-2024-05-13-func-3
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 58.6
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 8.7
|
||||
total_cost: 1.2064
|
||||
- dirname: 2024-08-15-14-17-51--json-no-lint-claude-3.5-sonnet-whole-3
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 11.0
|
||||
total_cost: 1.6555
|
||||
- dirname: 2024-08-15-14-21-06--json-no-lint-deepseek-coder-whole-3
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 61.7
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 3
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 2
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 3
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 24.4
|
||||
total_cost: 0.0439
|
||||
- dirname: 2024-08-15-14-27-17--json-no-lint-gpt-4o-2024-08-06-whole-4
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 4.3
|
||||
total_cost: 0.8015
|
||||
- dirname: 2024-08-15-14-28-58--json-no-lint-gpt-4o-2024-08-06-func-4
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 6.0
|
||||
total_cost: 0.8394
|
||||
- dirname: 2024-08-15-14-32-58--json-no-lint-gpt-4o-2024-05-13-func-4
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 59.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 2
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 11.1
|
||||
total_cost: 1.2120
|
||||
- dirname: 2024-08-15-14-34-39--json-no-lint-claude-3.5-sonnet-whole-4
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 11.3
|
||||
total_cost: 1.6635
|
||||
- dirname: 2024-08-15-14-38-35--json-no-lint-deepseek-coder-whole-4
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 59.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 2
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 2
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 24.5
|
||||
total_cost: 0.0438
|
||||
- dirname: 2024-08-15-14-44-11--json-no-lint-gpt-4o-2024-08-06-whole-5
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 4.6
|
||||
total_cost: 0.8023
|
||||
- dirname: 2024-08-15-14-45-40--json-no-lint-gpt-4o-2024-08-06-func-5
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 57.1
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 3
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 6.3
|
||||
total_cost: 0.8354
|
||||
- dirname: 2024-08-15-14-49-44--json-no-lint-gpt-4o-2024-05-13-func-5
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: JSON
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 59.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 4
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 10.5
|
||||
total_cost: 1.2099
|
||||
- dirname: 2024-08-15-14-51-18--json-no-lint-claude-3.5-sonnet-whole-5
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 11.4
|
||||
total_cost: 1.6685
|
||||
- dirname: 2024-08-15-14-54-41--json-no-lint-deepseek-coder-whole-5
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: Markdown
|
||||
commit_hash: bac04a2
|
||||
pass_rate_1: 61.7
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 2
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 2
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 24.5
|
||||
total_cost: 0.0439
|
||||
- dirname: 2024-08-15-15-12-55--json-no-lint-strict-gpt-4o-2024-08-06-func-2
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON (strict)
|
||||
commit_hash: bf2d5fe
|
||||
pass_rate_1: 57.1
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 5.9
|
||||
total_cost: 0.8216
|
||||
- dirname: 2024-08-15-15-14-31--json-no-lint-strict-gpt-4o-2024-08-06-func-3
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON (strict)
|
||||
commit_hash: bf2d5fe
|
||||
pass_rate_1: 54.1
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 2
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 6.3
|
||||
total_cost: 0.8410
|
||||
- dirname: 2024-08-15-15-16-14--json-no-lint-strict-gpt-4o-2024-08-06-func-4
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON (strict)
|
||||
commit_hash: bf2d5fe
|
||||
pass_rate_1: 59.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 5.9
|
||||
total_cost: 0.8203
|
||||
- dirname: 2024-08-15-15-17-50--json-no-lint-strict-gpt-4o-2024-08-06-func-5
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-08-06
|
||||
edit_format: JSON (strict)
|
||||
commit_hash: bf2d5fe
|
||||
pass_rate_1: 57.1
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 1
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-08-06
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 6.1
|
||||
total_cost: 0.8415
|
||||
- dirname: 2024-08-15-17-36-22--json-no-lint-again-gpt-4o-2024-05-13-whole-1
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: Markdown
|
||||
commit_hash: ed94379
|
||||
pass_rate_1: 60.2
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 7
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 6.8
|
||||
total_cost: 1.5110
|
||||
- dirname: 2024-08-15-17-38-13--json-no-lint-again-gpt-4o-2024-05-13-whole-2
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: Markdown
|
||||
commit_hash: ed94379
|
||||
pass_rate_1: 60.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 7.0
|
||||
total_cost: 1.4954
|
||||
- dirname: 2024-08-15-17-40-10--json-no-lint-again-gpt-4o-2024-05-13-whole-3
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: Markdown
|
||||
commit_hash: ed94379
|
||||
pass_rate_1: 60.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 6.8
|
||||
total_cost: 1.4999
|
||||
- dirname: 2024-08-15-17-41-30--json-no-lint-again-gpt-4o-2024-05-13-whole-4
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: Markdown
|
||||
commit_hash: ed94379
|
||||
pass_rate_1: 58.6
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 7.4
|
||||
total_cost: 1.4848
|
||||
- dirname: 2024-08-15-17-43-12--json-no-lint-again-gpt-4o-2024-05-13-whole-5
|
||||
test_cases: 133
|
||||
model: gpt-4o-2024-05-13
|
||||
edit_format: Markdown
|
||||
commit_hash: ed94379
|
||||
pass_rate_1: 59.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model gpt-4o-2024-05-13
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 7.6
|
||||
total_cost: 1.4948
|
||||
|
||||
- dirname: 2024-08-15-19-35-32--json-no-lint-again-deepseek-coder-func-1
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: JSON
|
||||
commit_hash: 3a2ac02-dirty
|
||||
pass_rate_1: 50.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 2
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 17.8
|
||||
total_cost: 0.0330
|
||||
- dirname: 2024-08-15-19-37-50--json-no-lint-again-deepseek-coder-func-2
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 49.6
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 5
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 18.3
|
||||
total_cost: 0.0336
|
||||
- dirname: 2024-08-15-19-40-20--json-no-lint-again-deepseek-coder-func-3
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 48.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 1
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 5
|
||||
indentation_errors: 1
|
||||
exhausted_context_windows: 1
|
||||
test_timeouts: 2
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 18.4
|
||||
total_cost: 0.0337
|
||||
- dirname: 2024-08-15-19-44-07--json-no-lint-again-deepseek-coder-func-4
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 53.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 2
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 2
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 17.6
|
||||
total_cost: 0.0330
|
||||
- dirname: 2024-08-15-19-46-48--json-no-lint-again-deepseek-coder-func-5
|
||||
test_cases: 133
|
||||
model: deepseek-coder V2 0724
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28-dirty
|
||||
pass_rate_1: 53.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 11
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 2
|
||||
command: aider --model deepseek-coder
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 18.0
|
||||
total_cost: 0.0332
|
||||
|
||||
- dirname: 2024-08-15-20-07-59--json-no-lint-again-claude-3.5-sonnet-func-1
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 54.1
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 9.5
|
||||
total_cost: 1.5789
|
||||
- dirname: 2024-08-15-20-09-39--json-no-lint-again-claude-3.5-sonnet-func-2
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 55.6
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 9.2
|
||||
total_cost: 1.5916
|
||||
- dirname: 2024-08-15-20-11-39--json-no-lint-again-claude-3.5-sonnet-func-3
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 53.4
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 10.3
|
||||
total_cost: 1.5896
|
||||
- dirname: 2024-08-15-20-13-44--json-no-lint-again-claude-3.5-sonnet-func-4
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 55.6
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 9.2
|
||||
total_cost: 1.6000
|
||||
- dirname: 2024-08-15-20-15-51--json-no-lint-again-claude-3.5-sonnet-func-5
|
||||
test_cases: 133
|
||||
model: claude-3.5-sonnet
|
||||
edit_format: JSON
|
||||
commit_hash: 1a98c28
|
||||
pass_rate_1: 51.9
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 1
|
||||
command: aider --model claude-3.5-sonnet
|
||||
date: 2024-08-15
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 8.9
|
||||
total_cost: 1.5936
|
||||
|
||||
|
|
@ -577,6 +577,7 @@
|
|||
pass_rate_2: 77.4
|
||||
percent_cases_well_formed: 99.2
|
||||
error_outputs: 23
|
||||
released: 2024-06-20
|
||||
num_malformed_responses: 4
|
||||
num_with_malformed_responses: 1
|
||||
user_asks: 2
|
||||
|
|
@ -603,6 +604,7 @@
|
|||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
released: 2024-03-13
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
|
|
@ -644,6 +646,7 @@
|
|||
commit_hash: d31eef3-dirty
|
||||
pass_rate_1: 40.6
|
||||
pass_rate_2: 55.6
|
||||
released: 2024-07-18
|
||||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 1
|
||||
num_malformed_responses: 0
|
||||
|
|
@ -668,6 +671,7 @@
|
|||
pass_rate_1: 60.9
|
||||
pass_rate_2: 69.9
|
||||
percent_cases_well_formed: 97.7
|
||||
released: 2024-06-28
|
||||
error_outputs: 58
|
||||
num_malformed_responses: 13
|
||||
num_with_malformed_responses: 3
|
||||
|
|
@ -690,6 +694,7 @@
|
|||
commit_hash: f7ce78b-dirty
|
||||
pass_rate_1: 46.6
|
||||
pass_rate_2: 63.9
|
||||
released: 2024-07-23
|
||||
percent_cases_well_formed: 92.5
|
||||
error_outputs: 84
|
||||
num_malformed_responses: 19
|
||||
|
|
@ -716,6 +721,7 @@
|
|||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 0
|
||||
num_malformed_responses: 0
|
||||
released: 2024-07-23
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
lazy_comments: 0
|
||||
|
|
@ -738,6 +744,7 @@
|
|||
pass_rate_2: 72.9
|
||||
percent_cases_well_formed: 97.7
|
||||
error_outputs: 13
|
||||
released: 2024-07-24
|
||||
num_malformed_responses: 3
|
||||
num_with_malformed_responses: 3
|
||||
user_asks: 1
|
||||
|
|
@ -763,6 +770,7 @@
|
|||
error_outputs: 3
|
||||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
released: 2024-07-24
|
||||
user_asks: 3
|
||||
lazy_comments: 0
|
||||
syntax_errors: 1
|
||||
|
|
@ -785,6 +793,7 @@
|
|||
percent_cases_well_formed: 100.0
|
||||
error_outputs: 27
|
||||
num_malformed_responses: 0
|
||||
released: 2024-07-23
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 23
|
||||
lazy_comments: 8
|
||||
|
|
@ -810,6 +819,7 @@
|
|||
num_malformed_responses: 0
|
||||
num_with_malformed_responses: 0
|
||||
user_asks: 0
|
||||
released: 2024-07-23
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
|
|
@ -838,9 +848,34 @@
|
|||
indentation_errors: 2
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 5
|
||||
released: 2024-08-06
|
||||
command: aider --model openai/gpt-4o-2024-08-06
|
||||
date: 2024-08-06
|
||||
versions: 0.48.1-dev
|
||||
seconds_per_case: 6.5
|
||||
total_cost: 0.0000
|
||||
|
||||
- dirname: 2024-08-14-13-07-12--chatgpt-4o-latest-diff
|
||||
test_cases: 133
|
||||
model: chatgpt-4o-latest
|
||||
edit_format: diff
|
||||
commit_hash: b1c3769
|
||||
pass_rate_1: 53.4
|
||||
pass_rate_2: 69.2
|
||||
percent_cases_well_formed: 97.7
|
||||
error_outputs: 27
|
||||
num_malformed_responses: 5
|
||||
num_with_malformed_responses: 3
|
||||
user_asks: 7
|
||||
lazy_comments: 0
|
||||
syntax_errors: 0
|
||||
indentation_errors: 0
|
||||
exhausted_context_windows: 0
|
||||
test_timeouts: 0
|
||||
command: aider --model openai/chatgpt-4o-latest
|
||||
date: 2024-08-14
|
||||
released: 2024-08-08
|
||||
versions: 0.50.2-dev
|
||||
seconds_per_case: 26.3
|
||||
total_cost: 3.6113
|
||||
|
||||
|
|
@ -1,90 +1,126 @@
|
|||
<canvas id="blameChart" width="800" height="450" style="margin-top: 20px"></canvas>
|
||||
<canvas id="blameChart" width="800" height="360" style="margin-top: 20px"></canvas>
|
||||
<canvas id="linesChart" width="800" height="360" style="margin-top: 20px"></canvas>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/moment"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-moment"></script>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
var ctx = document.getElementById('blameChart').getContext('2d');
|
||||
var blameCtx = document.getElementById('blameChart').getContext('2d');
|
||||
var linesCtx = document.getElementById('linesChart').getContext('2d');
|
||||
|
||||
var labels = [{% for row in site.data.blame %}'{{ row.end_tag }}',{% endfor %}];
|
||||
|
||||
var blameData = {
|
||||
labels: labels,
|
||||
datasets: [{
|
||||
label: 'Aider\'s Contribution to Each Release',
|
||||
data: [
|
||||
{% for row in site.data.blame %}
|
||||
{
|
||||
x: '{{ row.end_date }}',
|
||||
y: {{ row.aider_percentage }},
|
||||
r: Math.sqrt({{ row.aider_total }}) * 1.5,
|
||||
label: '{{ row.end_tag }}',
|
||||
percentage: {{ row.aider_percentage }},
|
||||
lines: {{ row.aider_total }}
|
||||
},
|
||||
{% endfor %}
|
||||
],
|
||||
backgroundColor: 'rgba(54, 162, 235, 0.2)',
|
||||
label: 'Aider\'s percent of new code by release',
|
||||
data: [{% for row in site.data.blame %}{ x: '{{ row.end_tag }}', y: {{ row.aider_percentage }}, lines: {{ row.aider_total }} },{% endfor %}],
|
||||
backgroundColor: 'rgba(54, 162, 235, 0.8)',
|
||||
borderColor: 'rgba(54, 162, 235, 1)',
|
||||
borderWidth: 1
|
||||
}]
|
||||
};
|
||||
|
||||
var blameChart = new Chart(ctx, {
|
||||
type: 'bubble',
|
||||
var linesData = {
|
||||
labels: labels,
|
||||
datasets: [{
|
||||
label: 'Aider\'s lines of new code',
|
||||
data: [{% for row in site.data.blame %}{ x: '{{ row.end_tag }}', y: {{ row.aider_total }} },{% endfor %}],
|
||||
backgroundColor: 'rgba(255, 99, 132, 0.8)',
|
||||
borderColor: 'rgba(255, 99, 132, 1)',
|
||||
borderWidth: 1
|
||||
}]
|
||||
};
|
||||
|
||||
var blameChart = new Chart(blameCtx, {
|
||||
type: 'bar',
|
||||
data: blameData,
|
||||
options: {
|
||||
scales: {
|
||||
x: {
|
||||
type: 'time',
|
||||
time: {
|
||||
unit: 'month',
|
||||
displayFormats: {
|
||||
month: 'MMM YYYY'
|
||||
}
|
||||
},
|
||||
type: 'category',
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Release date'
|
||||
text: 'Version'
|
||||
},
|
||||
ticks: {
|
||||
maxRotation: 45,
|
||||
minRotation: 45
|
||||
},
|
||||
min: moment('{{ site.data.blame | first | map: "end_date" | first }}').subtract(1, 'month'),
|
||||
max: moment('{{ site.data.blame | last | map: "end_date" | first }}').add(1, 'month')
|
||||
}
|
||||
},
|
||||
y: {
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Aider Contribution (% of code)'
|
||||
text: 'Percent of new code'
|
||||
},
|
||||
beginAtZero: true
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
legend: {
|
||||
display: false
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
return `${context.raw.label}: ${Math.round(context.raw.percentage)}% (${context.raw.lines} lines)`;
|
||||
}
|
||||
}
|
||||
},
|
||||
legend: {
|
||||
display: true,
|
||||
position: 'top',
|
||||
labels: {
|
||||
generateLabels: function(chart) {
|
||||
return [{
|
||||
text: 'Y-axis is percent of code, bubble size is lines of code',
|
||||
fillStyle: 'rgba(54, 162, 235, 0.2)',
|
||||
strokeStyle: 'rgba(54, 162, 235, 1)',
|
||||
lineWidth: 1,
|
||||
hidden: false,
|
||||
index: 0
|
||||
}];
|
||||
var label = 'Aider\'s contribution';
|
||||
var value = context.parsed.y || 0;
|
||||
var lines = context.raw.lines || 0;
|
||||
return `${label}: ${Math.round(value)}% (${lines} lines)`;
|
||||
}
|
||||
}
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Aider\'s Contribution to Each Release',
|
||||
text: 'Percent of new code written by aider, by release',
|
||||
font: {
|
||||
size: 16
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
var linesChart = new Chart(linesCtx, {
|
||||
type: 'bar',
|
||||
data: linesData,
|
||||
options: {
|
||||
scales: {
|
||||
x: {
|
||||
type: 'category',
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Version'
|
||||
},
|
||||
ticks: {
|
||||
maxRotation: 45,
|
||||
minRotation: 45
|
||||
}
|
||||
},
|
||||
y: {
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Lines of new code'
|
||||
},
|
||||
beginAtZero: true
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
legend: {
|
||||
display: false
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
var label = 'New lines of code by aider';
|
||||
var value = context.parsed.y || 0;
|
||||
return `${label}: ${value}`;
|
||||
}
|
||||
}
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Lines of new code written by aider, by release',
|
||||
font: {
|
||||
size: 16
|
||||
}
|
||||
|
|
|
|||
165
aider/website/_includes/code-in-json-benchmark.js
Normal file
165
aider/website/_includes/code-in-json-benchmark.js
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
<style>
|
||||
.chart-container {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div class="chart-container">
|
||||
<canvas id="passRateChart"></canvas>
|
||||
</div>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
var ctx = document.getElementById('passRateChart').getContext('2d');
|
||||
var chartContainer = document.querySelector('.chart-container');
|
||||
|
||||
var yamlData = {{ site.data.code-in-json | jsonify }};
|
||||
|
||||
var models = [...new Set(yamlData.map(item => item.model))].sort();
|
||||
var editFormats = [...new Set(yamlData.map(item => item.edit_format))];
|
||||
|
||||
var datasets = editFormats.map(format => ({
|
||||
label: format,
|
||||
data: models.map(model => {
|
||||
var items = yamlData.filter(d => d.model === model && d.edit_format === format);
|
||||
if (items.length === 0) return null;
|
||||
var average = items.reduce((sum, item) => sum + item.pass_rate_1, 0) / items.length;
|
||||
return parseFloat(average.toFixed(1));
|
||||
}),
|
||||
backgroundColor: function(context) {
|
||||
const format = context.dataset.label;
|
||||
if (format === 'Markdown') {
|
||||
return 'rgba(54, 162, 235, 0.8)';
|
||||
} else if (format.startsWith('JSON')) {
|
||||
const ctx = context.chart.ctx;
|
||||
const gradient = ctx.createPattern(createStripedCanvas(format === 'JSON (strict)'), 'repeat');
|
||||
return gradient;
|
||||
} else {
|
||||
return 'rgba(75, 192, 192, 0.8)';
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
var data = {
|
||||
labels: models,
|
||||
datasets: datasets
|
||||
};
|
||||
|
||||
function getAspectRatio() {
|
||||
var width = chartContainer.offsetWidth;
|
||||
// Gradually change aspect ratio from 2 (landscape) to 1 (square)
|
||||
return Math.max(1, Math.min(2, width / 300));
|
||||
}
|
||||
|
||||
var config = {
|
||||
type: 'bar',
|
||||
data: data,
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: true,
|
||||
aspectRatio: getAspectRatio(),
|
||||
scales: {
|
||||
x: {
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Model'
|
||||
}
|
||||
},
|
||||
y: {
|
||||
beginAtZero: true,
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Pass Rate (%, average of 5 runs)'
|
||||
},
|
||||
max: 70
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Coding skill by model and code wrapping strategy',
|
||||
font: {
|
||||
size: 16
|
||||
}
|
||||
},
|
||||
legend: {
|
||||
position: 'top',
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
let label = context.dataset.label || '';
|
||||
if (label) {
|
||||
label += ': ';
|
||||
}
|
||||
if (context.parsed.y !== null) {
|
||||
label += context.parsed.y.toFixed(1) + '%';
|
||||
}
|
||||
return label;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
plugins: [{
|
||||
afterDraw: function(chart) {
|
||||
var ctx = chart.ctx;
|
||||
var isWideScreen = window.innerWidth > 768; // Assuming 768px as the breakpoint for wide screens
|
||||
if (isWideScreen) {
|
||||
chart.data.datasets.forEach(function(dataset, i) {
|
||||
var meta = chart.getDatasetMeta(i);
|
||||
meta.data.forEach(function(bar, index) {
|
||||
var data = dataset.data[index];
|
||||
if (data !== null) {
|
||||
ctx.fillStyle = '#000000';
|
||||
ctx.textAlign = 'center';
|
||||
ctx.textBaseline = 'bottom';
|
||||
var displayText = data.toFixed(1) + '%';
|
||||
ctx.fillText(displayText, bar.x, bar.y - 5);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}]
|
||||
};
|
||||
|
||||
var chart = new Chart(ctx, config);
|
||||
|
||||
function resizeChart() {
|
||||
chart.options.aspectRatio = getAspectRatio();
|
||||
chart.resize();
|
||||
}
|
||||
|
||||
window.addEventListener('resize', resizeChart);
|
||||
|
||||
// Initial resize to set correct size
|
||||
resizeChart();
|
||||
});
|
||||
|
||||
function createStripedCanvas(isStrict) {
|
||||
const patternCanvas = document.createElement('canvas');
|
||||
const patternContext = patternCanvas.getContext('2d');
|
||||
const size = 10;
|
||||
patternCanvas.width = size;
|
||||
patternCanvas.height = size;
|
||||
|
||||
patternContext.fillStyle = 'rgba(255, 99, 132, 0.8)';
|
||||
patternContext.fillRect(0, 0, size, size);
|
||||
|
||||
if (isStrict) {
|
||||
patternContext.strokeStyle = 'rgba(255, 255, 255, 0.8)';
|
||||
patternContext.lineWidth = 0.75;
|
||||
patternContext.beginPath();
|
||||
patternContext.moveTo(0, 0);
|
||||
patternContext.lineTo(size, size);
|
||||
patternContext.stroke();
|
||||
}
|
||||
|
||||
return patternCanvas;
|
||||
}
|
||||
</script>
|
||||
139
aider/website/_includes/code-in-json-syntax.js
Normal file
139
aider/website/_includes/code-in-json-syntax.js
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
<style>
|
||||
.chart-container {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
max-width: 800px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div class="chart-container">
|
||||
<canvas id="syntaxErrorsChart"></canvas>
|
||||
</div>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
var ctx = document.getElementById('syntaxErrorsChart').getContext('2d');
|
||||
var chartContainer = document.querySelector('.chart-container');
|
||||
|
||||
var yamlData = {{ site.data.code-in-json | jsonify }};
|
||||
|
||||
var models = [...new Set(yamlData.map(item => item.model))].sort();
|
||||
var editFormats = [...new Set(yamlData.map(item => item.edit_format))];
|
||||
|
||||
var datasets = editFormats.map(format => ({
|
||||
label: format,
|
||||
data: models.map(model => {
|
||||
var items = yamlData.filter(d => d.model === model && d.edit_format === format);
|
||||
if (items.length === 0) return null;
|
||||
var totalErrors = items.reduce((sum, item) => sum + item.syntax_errors + item.indentation_errors, 0);
|
||||
return totalErrors;
|
||||
}),
|
||||
backgroundColor: function(context) {
|
||||
const format = context.dataset.label;
|
||||
if (format === 'Markdown') {
|
||||
return 'rgba(54, 162, 235, 0.8)';
|
||||
} else if (format.startsWith('JSON')) {
|
||||
const ctx = context.chart.ctx;
|
||||
const gradient = ctx.createPattern(createStripedCanvas(format === 'JSON (strict)'), 'repeat');
|
||||
return gradient;
|
||||
} else {
|
||||
return 'rgba(75, 192, 192, 0.8)';
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
var data = {
|
||||
labels: models,
|
||||
datasets: datasets
|
||||
};
|
||||
|
||||
function getAspectRatio() {
|
||||
var width = chartContainer.offsetWidth;
|
||||
// Gradually change aspect ratio from 2 (landscape) to 1 (square)
|
||||
return Math.max(1, Math.min(2, width / 300));
|
||||
}
|
||||
|
||||
var config = {
|
||||
type: 'bar',
|
||||
data: data,
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: true,
|
||||
aspectRatio: getAspectRatio(),
|
||||
scales: {
|
||||
x: {
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Model'
|
||||
}
|
||||
},
|
||||
y: {
|
||||
beginAtZero: true,
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Total syntax errors from 5 runs'
|
||||
},
|
||||
max: 35
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Syntax errors by model and code wrapping strategy',
|
||||
font: {
|
||||
size: 16
|
||||
}
|
||||
},
|
||||
legend: {
|
||||
position: 'top',
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
let label = context.dataset.label || '';
|
||||
if (label) {
|
||||
label += ': ';
|
||||
}
|
||||
if (context.parsed.y !== null) {
|
||||
label += context.parsed.y;
|
||||
}
|
||||
return label;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
plugins: [{
|
||||
afterDraw: function(chart) {
|
||||
var ctx = chart.ctx;
|
||||
chart.data.datasets.forEach(function(dataset, i) {
|
||||
var meta = chart.getDatasetMeta(i);
|
||||
meta.data.forEach(function(bar, index) {
|
||||
var data = dataset.data[index];
|
||||
if (data !== null) {
|
||||
ctx.fillStyle = '#000000';
|
||||
ctx.textAlign = 'center';
|
||||
ctx.textBaseline = 'bottom';
|
||||
ctx.fillText(data, bar.x, bar.y - 5);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}]
|
||||
};
|
||||
|
||||
var chart = new Chart(ctx, config);
|
||||
|
||||
function resizeChart() {
|
||||
chart.options.aspectRatio = getAspectRatio();
|
||||
chart.resize();
|
||||
}
|
||||
|
||||
window.addEventListener('resize', resizeChart);
|
||||
|
||||
// Initial resize to set correct size
|
||||
resizeChart();
|
||||
});
|
||||
</script>
|
||||
248
aider/website/_posts/2024-08-14-code-in-json.md
Normal file
248
aider/website/_posts/2024-08-14-code-in-json.md
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
---
|
||||
title: LLMs are bad at returning code in JSON
|
||||
excerpt: LLMs write worse code if you ask them to return the code wrapped in JSON via a tool function call.
|
||||
highlight_image: /assets/code-in-json.jpg
|
||||
nav_exclude: true
|
||||
---
|
||||
{% if page.date %}
|
||||
<p class="post-date">{{ page.date | date: "%B %d, %Y" }}</p>
|
||||
{% endif %}
|
||||
|
||||
# LLMs are bad at returning code in JSON
|
||||
|
||||
|
||||
LLMs produce lower quality code if they’re asked to return it as part of a structured JSON response. This seems to be true for many top models, including those with specialized support for JSON. Benchmarks show that models struggle with syntactic issues related to quoting and escaping.
|
||||
The benchmark results also imply a decreased capacity for solving coding problems due to the burden of JSON formatting.
|
||||
|
||||
{% include code-in-json-benchmark.js %}
|
||||
|
||||
> Figure 1: Aider coding benchmark scores of models using either plain markdown text or JSON to return code.
|
||||
> Pass rate (%) averaged over 5 runs.
|
||||
> Models produce better code when they return it as markdown text,
|
||||
> as compared to returning code in a structured JSON response.
|
||||
|
||||
|
||||
## Background
|
||||
|
||||
People often ask why aider uses a plain text format for LLMs to specify code edits (below),
|
||||
rather than relying on LLM tools and structured JSON responses.
|
||||
|
||||
```python
|
||||
greeting.py
|
||||
<<<<<<< SEARCH
|
||||
def greeting():
|
||||
print("Hello")
|
||||
=======
|
||||
def greeting():
|
||||
print("Goodbye")
|
||||
>>>>>>> REPLACE
|
||||
```
|
||||
|
||||
People expect that it would be easier and more reliable to use tool calls,
|
||||
which would involve a structured JSON response more like this:
|
||||
|
||||
```json
|
||||
{
|
||||
"filename": "greeting.py",
|
||||
"search": "def greeting():\n print(\"Hello\")\n"
|
||||
"replace": "def greeting():\n print(\"Goodbye\")\n"
|
||||
}
|
||||
```
|
||||
|
||||
This question becomes increasingly relevant as LLM providers
|
||||
continue to improve their tooling for reliably generating JSON.
|
||||
For example,
|
||||
[OpenAI recently announced](https://openai.com/index/introducing-structured-outputs-in-the-api/)
|
||||
the ability to
|
||||
strictly enforce that JSON responses will be syntactically correct
|
||||
and conform to a specified schema.
|
||||
|
||||
But just producing valid JSON is not sufficient for AI code generation --
|
||||
the code inside the JSON matters too.
|
||||
It has to be high quality code that solves the assigned coding task without errors or bugs.
|
||||
Unfortunately,
|
||||
LLMs write worse code when they're asked to
|
||||
wrap it in JSON.
|
||||
|
||||
In some sense this shouldn't be surprising.
|
||||
Just look at the very simple
|
||||
JSON example above, with the escaped
|
||||
quotes `\"` and
|
||||
newlines `\n`
|
||||
mixed into the code.
|
||||
Imagine the additional
|
||||
complexity
|
||||
if the code itself contained quoted strings
|
||||
with their
|
||||
own escape sequences.
|
||||
|
||||
Would *you* write better code by
|
||||
typing it out normally
|
||||
or typing it as a properly escaped
|
||||
JSON string?
|
||||
|
||||
|
||||
## Quantifying the benefits of plain text
|
||||
|
||||
Previous [aider benchmark results](/2023/07/02/benchmarks.html)
|
||||
showed
|
||||
the superiority of returning code
|
||||
as plain text compared to JSON-wrapped function calls.
|
||||
Those results were obtained
|
||||
over a year ago, against models far less capable than those available today.
|
||||
OpenAI's newly announced support for "strict" JSON
|
||||
suggests the possibility that modern models might be able
|
||||
to return quality code inside a structured JSON response.
|
||||
|
||||
The results presented here are based on
|
||||
the
|
||||
[aider "code editing" benchmark](/2023/07/02/benchmarks.html#the-benchmark)
|
||||
of 133 practice exercises from the Exercism python repository.
|
||||
The benchmark was simplified somewhat to focus on the differences between
|
||||
plain text and JSON responses.
|
||||
In particular, models were
|
||||
restricted to a single attempt to solve each task
|
||||
without a second try to fix errors.
|
||||
|
||||
The performance of each model was compared across different strategies for returning code:
|
||||
|
||||
- **Markdown** -- the model returned the whole source code file in standard markdown triple-backtick fences.
|
||||
- **JSON** -- the model used a tool function call to return the whole source code file in a structured JSON response.
|
||||
- **JSON (strict)** -- the same as the "JSON" strategy, but with `strict=True`. Only gpt-4o-2024-08-06 supported this setting.
|
||||
|
||||
The markdown strategy was the same as
|
||||
aider's "whole" edit format, where the
|
||||
LLM returns an entire updated copy of the source file like this:
|
||||
|
||||
````
|
||||
Here is the program you asked for which prints "Hello":
|
||||
|
||||
greeting.py
|
||||
```
|
||||
def greeting():
|
||||
print("Hello")
|
||||
```
|
||||
````
|
||||
|
||||
Both JSON strategies required the LLM to call the `write_file` function with
|
||||
an explanation/plan and
|
||||
the entire updated copy of the source file.
|
||||
The LLM didn't have to specify the filename,
|
||||
since the benchmark operates on one source file at a time.
|
||||
|
||||
```json
|
||||
{
|
||||
"explanation": "Here is the program you asked for which prints \"Hello\"",
|
||||
"content": "def greeting():\n print(\"Hello\")\n"
|
||||
}
|
||||
```
|
||||
|
||||
This experimental setup was designed to quantify
|
||||
the effects of JSON-wrapping on the LLMs ability to write code to solve a task.
|
||||
|
||||
## Results
|
||||
|
||||
Four of the strongest code editing models were benchmarked
|
||||
to assess the impact of JSON-wrapping code:
|
||||
|
||||
- claude-3-5-sonnet-20240620
|
||||
- deepseek-coder (V2 0724)
|
||||
- gpt-4o-2024-05-13
|
||||
- gpt-4o-2024-08-06
|
||||
|
||||
Each combination of model and code wrapping strategy was benchmarked 5 times.
|
||||
|
||||
### Overall coding skill
|
||||
|
||||
As shown in Figure 1,
|
||||
all of the models did worse on the benchmark when asked to
|
||||
return code in a structured JSON response.
|
||||
Most did significantly worse, performing well below
|
||||
their result with the markdown strategy.
|
||||
|
||||
Some noteworthy observations:
|
||||
|
||||
- OpenAI's gpt-4o-2024-05-13 was the only model where the markdown and JSON results were
|
||||
close. Using JSON only dropped the score by 0.4 percent, a difference which is
|
||||
within the margin of error for 5 trials.
|
||||
- The use of OpenAI's new strict mode offered no improvement
|
||||
as compared to non-strict JSON.
|
||||
Both JSON results were well below the markdown result.
|
||||
- The results from Sonnet and DeepSeek Coder suffered the worst harm from JSON wrapping.
|
||||
|
||||
### Syntax errors
|
||||
|
||||
Models tend to make more syntax errors when asked to wrap code in JSON.
|
||||
Figure 2 shows the number of syntax errors found in the code produced by each
|
||||
model and code wrapping strategy.
|
||||
It totals up the `SyntaxError` and `IndentationError` errors from all 5 runs,
|
||||
for each model and strategy combination.
|
||||
|
||||
Below is an example of a `SyntaxError` created by gpt-4o-2024-05-13 using the
|
||||
JSON code wrapping strategy.
|
||||
It appears that the model got confused about escaping and quoting while trying
|
||||
to format the JSON response.
|
||||
|
||||
```python
|
||||
Traceback (most recent call last):
|
||||
...
|
||||
File "bottle-song/bottle_song.py", line 9
|
||||
lyrics.append(f'There'll be {i - 1} green bottles hanging on the wall.')
|
||||
^
|
||||
SyntaxError: unterminated string literal (detected at line 9)
|
||||
```
|
||||
|
||||
The problematic line of code contains a single-quoted string which also
|
||||
contains a single-quote character.
|
||||
It should have been output as the following chunk of JSON, with
|
||||
a double slash in `There\\'ll`.
|
||||
That is needed to JSON-escape the `\` so that it survives
|
||||
JSON-decoding to
|
||||
produce `There\'ll` in the resulting code.
|
||||
That would correctly escape the single-quote inside the single-quoted string.
|
||||
|
||||
```
|
||||
...lyrics.append(f'There\\'ll be {i - 1} green bottles hanging on the wall.')\n...
|
||||
```
|
||||
|
||||
|
||||
|
||||
{% include code-in-json-syntax.js %}
|
||||
|
||||
> Figure 2: Number of `SyntaxError` and `IndentationError` errors found in model generated code,
|
||||
> totaled from 5 runs.
|
||||
> Models tend to make more syntax and formatting errors when asked to wrap code in JSON.
|
||||
|
||||
### Beyond syntax errors
|
||||
|
||||
Sonnet's results seems to indicate that the negative effects of JSON-wrapping
|
||||
go beyond just syntactic difficulties.
|
||||
Sonnet avoided syntax errors regardless of the code wrapping strategy,
|
||||
but its benchmark scores in Figure 1 were nonetheless lower with JSON.
|
||||
This implies that JSON-wrapping may distract or challenge models in a way that
|
||||
reduces their ability to reason about solving coding problems.
|
||||
|
||||
|
||||
|
||||
## Conclusions
|
||||
|
||||
While the specific results differ from the similar
|
||||
[July 2023 experiments](/2023/07/02/benchmarks.html),
|
||||
the conclusion remains unchanged: LLMs are bad at returning code in
|
||||
structured JSON responses.
|
||||
|
||||
OpenAI appears to be making progress in allowing LLMs to
|
||||
return JSON-wrapped code
|
||||
without harming the code quality.
|
||||
But it seems premature to consider switching from plain text
|
||||
to JSON-wrapped code at this time.
|
||||
|
||||
---------
|
||||
|
||||
#### Notes on the aider leaderboard
|
||||
|
||||
*The results presented here are not directly comparable to results
|
||||
from the main
|
||||
[aider LLM leaderboard](https://aider.chat/docs/leaderboards/).
|
||||
A number of settings were changed to simplify the benchmark
|
||||
in order to focus on comparing plain text and JSON-wrapped code.*
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB After Width: | Height: | Size: 158 KiB |
BIN
aider/website/assets/code-in-json.jpg
Normal file
BIN
aider/website/assets/code-in-json.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 158 KiB |
File diff suppressed because it is too large
Load diff
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 74 KiB |
|
|
@ -28,7 +28,7 @@ Using a `.aider.conf.yml` file:
|
|||
dark-mode: true
|
||||
```
|
||||
|
||||
By setting an environgment variable:
|
||||
By setting an environment variable:
|
||||
|
||||
```
|
||||
export AIDER_DARK_MODE=true
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ The json file should be a dictionary with an entry for each model, as follows:
|
|||
|
||||
```
|
||||
{
|
||||
"deepseek-chat": {
|
||||
"deepseek/deepseek-chat": {
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 4096,
|
||||
|
|
@ -42,6 +42,11 @@ The json file should be a dictionary with an entry for each model, as follows:
|
|||
See
|
||||
[litellm's model_prices_and_context_window.json file](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) for more examples.
|
||||
|
||||
{: .tip }
|
||||
Use a fully qualified model name with a `provider/` at the front
|
||||
in the `.aider.model.metadata.json` file.
|
||||
For example, use `deepseek/deepseek-chat`, not just `deepseek-chat`.
|
||||
|
||||
## Model settings
|
||||
|
||||
Aider has a number of settings that control how it works with
|
||||
|
|
|
|||
|
|
@ -321,6 +321,6 @@ mod_dates = [get_last_modified_date(file) for file in files]
|
|||
latest_mod_date = max(mod_dates)
|
||||
cog.out(f"{latest_mod_date.strftime('%B %d, %Y.')}")
|
||||
]]]-->
|
||||
August 10, 2024.
|
||||
August 14, 2024.
|
||||
<!--[[[end]]]-->
|
||||
</p>
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ You can add images to the chat just like you would
|
|||
add any other file:
|
||||
|
||||
- Use `/add <image-filename>` from within the chat
|
||||
- Use `/add-clipboard-image` to paste an image from your clipboard into the chat.
|
||||
- Use `/clipboard` to paste an image from your clipboard into the chat.
|
||||
- Launch aider with image filenames on the command line: `aider <image-filename>` along with any other command line arguments you need.
|
||||
|
||||
## Web pages
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue