mirror of
https://github.com/Aider-AI/aider.git
synced 2025-06-17 01:55:01 +00:00
322 lines
No EOL
8.6 KiB
YAML
322 lines
No EOL
8.6 KiB
YAML
- dirname: 2024-11-09-11-09-15--Qwen2.5-Coder-32B-Instruct
|
|
test_cases: 133
|
|
model: "HuggingFace via GLHF: BF16"
|
|
released: 2024-11-12
|
|
edit_format: diff
|
|
commit_hash: ec9982a
|
|
pass_rate_1: 59.4
|
|
pass_rate_2: 71.4
|
|
percent_cases_well_formed: 94.7
|
|
error_outputs: 17
|
|
num_malformed_responses: 17
|
|
num_with_malformed_responses: 7
|
|
user_asks: 1
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 3
|
|
command: aider --model openai/hf:Qwen/Qwen2.5-Coder-32B-Instruct --openai-api-base https://glhf.chat/api/openai/v1
|
|
date: 2024-11-09
|
|
versions: 0.59.2.dev
|
|
seconds_per_case: 22.5
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-22-18-56-13--ollama-qwen2.5-coder:32b-instruct-fp16
|
|
test_cases: 132
|
|
model: "Ollama: fp16"
|
|
edit_format: diff
|
|
commit_hash: f06452c-dirty, 6a0a97c-dirty, 4e9ae16-dirty, 5506d0f-dirty
|
|
pass_rate_1: 58.3
|
|
pass_rate_2: 71.4
|
|
percent_cases_well_formed: 90.2
|
|
error_outputs: 27
|
|
num_malformed_responses: 26
|
|
num_with_malformed_responses: 13
|
|
user_asks: 2
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 0
|
|
command: aider --model ollama/qwen2.5-coder:32b-instruct-fp16
|
|
date: 2024-11-22
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 119.6
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-22-14-53-26--hyperbolic-qwen25coder32binstruct
|
|
test_cases: 133
|
|
model: "Hyperbolic: BF16"
|
|
edit_format: diff
|
|
commit_hash: f9ef161, 17aef7b-dirty
|
|
pass_rate_1: 57.9
|
|
pass_rate_2: 69.2
|
|
percent_cases_well_formed: 91.7
|
|
error_outputs: 30
|
|
num_malformed_responses: 29
|
|
num_with_malformed_responses: 11
|
|
user_asks: 9
|
|
lazy_comments: 0
|
|
syntax_errors: 4
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 2
|
|
command: aider --model openai/Qwen/Qwen2.5-Coder-32B-Instruct --openai-api-base https://api.hyperbolic.xyz/v1/
|
|
date: 2024-11-22
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 33.2
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-22-17-53-35--qwen25-coder-32b-Instruct-4bit
|
|
test_cases: 133
|
|
model: "mlx-community: 4bit"
|
|
edit_format: diff
|
|
commit_hash: a16dcab-dirty
|
|
pass_rate_1: 60.2
|
|
pass_rate_2: 72.2
|
|
percent_cases_well_formed: 88.7
|
|
error_outputs: 31
|
|
num_malformed_responses: 30
|
|
num_with_malformed_responses: 15
|
|
user_asks: 6
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 1
|
|
test_timeouts: 0
|
|
command: aider --model openai/mlx-community/Qwen2.5-Coder-32B-Instruct-4bit
|
|
date: 2024-11-23
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 53.4
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-23-15-07-20--qwen25-coder-32b-Instruct-8bit
|
|
test_cases: 133
|
|
model: "mlx-community: 8bit"
|
|
edit_format: diff
|
|
commit_hash: a16dcab-dirty
|
|
pass_rate_1: 59.4
|
|
pass_rate_2: 72.2
|
|
percent_cases_well_formed: 92.5
|
|
error_outputs: 20
|
|
num_malformed_responses: 15
|
|
num_with_malformed_responses: 10
|
|
user_asks: 7
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 5
|
|
test_timeouts: 2
|
|
command: aider --model openai/mlx-community/Qwen2.5-Coder-32B-Instruct-8bit
|
|
date: 2024-11-23
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 98.4
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-24-22-18-18--or-all-or-fixed-blank-messages2
|
|
test_cases: 133
|
|
model: "OpenRouter: multiple"
|
|
edit_format: diff
|
|
commit_hash: 0c59d32
|
|
pass_rate_1: 57.1
|
|
pass_rate_2: 67.7
|
|
percent_cases_well_formed: 95.5
|
|
error_outputs: 56
|
|
num_malformed_responses: 10
|
|
num_with_malformed_responses: 6
|
|
user_asks: 14
|
|
lazy_comments: 0
|
|
syntax_errors: 6
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 3
|
|
test_timeouts: 1
|
|
command: aider --model openrouter/qwen/qwen-2.5-coder-32b-instruct
|
|
date: 2024-11-24
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 21.2
|
|
total_cost: 0.1420
|
|
|
|
- dirname: 2024-11-23-21-08-53--ollama-qwen2.5-coder:32b-instruct-q4_K_M-8kctx
|
|
test_cases: 133
|
|
model: "Ollama: q4_K_M"
|
|
edit_format: diff
|
|
commit_hash: baa1335-dirty, e63df83-dirty, ff8c1aa-dirty
|
|
pass_rate_1: 54.9
|
|
pass_rate_2: 66.9
|
|
percent_cases_well_formed: 94.0
|
|
error_outputs: 21
|
|
num_malformed_responses: 21
|
|
num_with_malformed_responses: 8
|
|
user_asks: 5
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 3
|
|
command: aider --model ollama/qwen2.5-coder:32b-instruct-q4_K_M
|
|
date: 2024-11-23
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 35.7
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-24-02-23-32--deepinfra-qwen-diff
|
|
test_cases: 133
|
|
model: "Deepinfra: BF16"
|
|
edit_format: diff
|
|
commit_hash: bb78e2f
|
|
pass_rate_1: 58.6
|
|
pass_rate_2: 72.2
|
|
percent_cases_well_formed: 94.7
|
|
error_outputs: 15
|
|
num_malformed_responses: 13
|
|
num_with_malformed_responses: 7
|
|
user_asks: 3
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 2
|
|
test_timeouts: 3
|
|
command: aider --model deepinfra/Qwen/Qwen2.5-Coder-32B-Instruct
|
|
date: 2024-11-24
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 17.5
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-24-04-12-58--fireworks-qwen-diff
|
|
test_cases: 133
|
|
model: "Fireworks: unknown"
|
|
edit_format: diff
|
|
commit_hash: 757eac0
|
|
pass_rate_1: 57.9
|
|
pass_rate_2: 72.2
|
|
percent_cases_well_formed: 94.0
|
|
error_outputs: 23
|
|
num_malformed_responses: 19
|
|
num_with_malformed_responses: 8
|
|
user_asks: 8
|
|
lazy_comments: 0
|
|
syntax_errors: 6
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 4
|
|
test_timeouts: 1
|
|
command: aider --model fireworks_ai/accounts/fireworks/models/qwen2p5-coder-32b-instruct
|
|
date: 2024-11-24
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 10.4
|
|
total_cost: 0.5759
|
|
|
|
- dirname: 2024-11-24-02-04-59--ollama-qwen2.5-coder:32b-instruct-q2_K-8kctx
|
|
test_cases: 133
|
|
model: "Ollama: q2_K"
|
|
edit_format: diff
|
|
commit_hash: 757eac0, bb78e2f, 8d0ba40-dirty, 1d09e96
|
|
pass_rate_1: 48.9
|
|
pass_rate_2: 61.7
|
|
percent_cases_well_formed: 91.7
|
|
error_outputs: 32
|
|
num_malformed_responses: 32
|
|
num_with_malformed_responses: 11
|
|
user_asks: 8
|
|
lazy_comments: 0
|
|
syntax_errors: 0
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 1
|
|
command: aider --model ollama/qwen2.5-coder:32b-instruct-q2_K
|
|
date: 2024-11-24
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 97.8
|
|
total_cost: 0.0000
|
|
|
|
- dirname: 2024-11-24-14-56-49--qwen25-32b-or-fireworks
|
|
test_cases: 133
|
|
model: "Fireworks via OpenRouter: unknown"
|
|
edit_format: diff
|
|
commit_hash: c2f184f
|
|
pass_rate_1: 55.6
|
|
pass_rate_2: 67.7
|
|
percent_cases_well_formed: 94.0
|
|
error_outputs: 39
|
|
num_malformed_responses: 24
|
|
num_with_malformed_responses: 8
|
|
user_asks: 13
|
|
lazy_comments: 0
|
|
syntax_errors: 1
|
|
indentation_errors: 1
|
|
exhausted_context_windows: 7
|
|
test_timeouts: 4
|
|
command: aider --model openrouter/qwen/qwen-2.5-coder-32b-instruct
|
|
date: 2024-11-24
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 16.1
|
|
total_cost: 0.1391
|
|
|
|
- dirname: 2024-11-24-22-03-19--or-hyperbolic-or-fixed-blank-messages2
|
|
test_cases: 133
|
|
model: "Hyperbolic via OpenRouter: BF16"
|
|
edit_format: diff
|
|
commit_hash: 0c59d32
|
|
pass_rate_1: 55.6
|
|
pass_rate_2: 68.4
|
|
percent_cases_well_formed: 89.5
|
|
error_outputs: 28
|
|
num_malformed_responses: 24
|
|
num_with_malformed_responses: 14
|
|
user_asks: 29
|
|
lazy_comments: 0
|
|
syntax_errors: 1
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 4
|
|
test_timeouts: 1
|
|
command: aider --model openrouter/qwen/qwen-2.5-coder-32b-instruct
|
|
date: 2024-11-24
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 41.5
|
|
total_cost: 0.1402
|
|
|
|
- dirname: 2024-11-24-15-00-50--qwen25-32b-or-deepinfra
|
|
test_cases: 133
|
|
model: "Deepinfra via OpenRouter: BF16"
|
|
edit_format: diff
|
|
commit_hash: c2f184f
|
|
pass_rate_1: 57.1
|
|
pass_rate_2: 69.9
|
|
percent_cases_well_formed: 89.5
|
|
error_outputs: 35
|
|
num_malformed_responses: 31
|
|
num_with_malformed_responses: 14
|
|
user_asks: 11
|
|
lazy_comments: 0
|
|
syntax_errors: 1
|
|
indentation_errors: 1
|
|
exhausted_context_windows: 4
|
|
test_timeouts: 1
|
|
command: aider --model openrouter/qwen/qwen-2.5-coder-32b-instruct
|
|
date: 2024-11-24
|
|
versions: 0.64.2.dev
|
|
seconds_per_case: 28.5
|
|
total_cost: 0.1390
|
|
|
|
- dirname: 2024-11-26-03-15-06--ollama-qwen2.5-coder:32b-instruct-fp16-2kctx
|
|
test_cases: 132
|
|
model: "Ollama: fp16, 2k ctx"
|
|
edit_format: diff
|
|
commit_hash: 68be6c5-dirty, 554d274, 2ff3a23, 2ff3a23-dirty, 61759f9, dd48b74, 3ebd47d-dirty
|
|
pass_rate_1: 43.2
|
|
pass_rate_2: 51.9
|
|
percent_cases_well_formed: 46.2
|
|
error_outputs: 171
|
|
num_malformed_responses: 165
|
|
num_with_malformed_responses: 71
|
|
user_asks: 97
|
|
lazy_comments: 2
|
|
syntax_errors: 4
|
|
indentation_errors: 0
|
|
exhausted_context_windows: 0
|
|
test_timeouts: 0
|
|
command: "aider --model ollama/qwen2.5-coder:32b-instruct-fp16 # num_ctx: 2048"
|
|
date: 2024-11-26
|
|
versions: 0.64.2.dev,0.65.1.dev
|
|
seconds_per_case: 188.6
|
|
total_cost: 0.0000 |