dearwolf/aider
1
0
Fork 0
mirror of https://github.com/Aider-AI/aider.git synced 2025-05-29 16:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

updated all openai models

Browse source
This commit is contained in:
Paul Gauthier 2024-05-04 07:38:50 -07:00
parent 1981105932
commit f6580fff76
3 changed files with 39 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

19
_data/leaderboard.csv
View file

@ -1,11 +1,16 @@
model,second,first,format,command,version,commit,date
model,second,first,format,command,version,commits,date
claude-3-opus-20240229,68.4,53.4,diff,aider --opus,0.30.1,f4b1797,5/2/24
gpt-4-0613,67.7,46.6,diff,aider -4,0.18.1,3aa17c4,12/16/23
gpt-4-turbo-2024-04-09,64.4,49.2,udiff,aider --gpt-4-turbo,0.30.1,e610e5b,5/1/24
gemini-1.5-pro-latest,57.1,45.9,diff-fenced,aider --model gemini/gemini-1.5-pro-latest,0.32.0,5d32dd7,5/3/24
gpt-3.5-turbo-1106,56.1,45.5,whole,aider --model gpt-3.5-turbo-1106,0.30.1,7b14d77,4/30/24
claude-3-sonnet-20240229,54.9,43.6,whole,aider --sonnet,0.25.0,a5f8076,3/6/24
Command-R+,29.3,22.6,whole,aider --model command-r-plus,0.28.0,a06c927,4/20/24
Deepseek Coder,54.5,47,whole,aider --model openai/deepseek-coder,0.30.1,c07f793,4/29/24
gemini-1.5-pro-latest,57.1,45.9,diff-fenced,aider --model gemini/gemini-1.5-pro-latest,0.32.0,5d32dd7,5/3/24
gpt-3.5-turbo-0125,49.6,39.8,whole,aider --35turbo,0.22.0,da14474,2/2/24
Llama3 70B,49.2,38.6,diff,aider --model groq/llama3-70b-8192,0.32.0,b5bb453,5/3/24
Command-R+,29.3,22.6,whole,aider --model command-r-plus,0.28.0,a06c927,4/20/24
gpt-3.5-turbo-0301,57.9,50.4,whole,aider --model gpt-3.5-turbo-0301,0.16.4-dev,44388db-dirty,11/6/23
gpt-3.5-turbo-0613,50.4,38.3,whole,aider --model gpt-3.5-turbo-0613,0.16.4-dev,93aa497-dirty,11/7/23
gpt-3.5-turbo-1106,56.1,45.5,whole,aider --model gpt-3.5-turbo-1106,0.30.1,7b14d77,4/30/24
gpt-4-0125-preview,66.2,55.6,udiff,aider,0.22.1-dev,edcf9b1,1/25/24
gpt-4-0314,68.4,51.1,diff,aider --model gpt-4-0314,0.7.2,506bfe2 c7c548d,7/1/23
gpt-4-0613,67.7,46.6,diff,aider -4,0.18.1,3aa17c4,12/16/23
gpt-4-1106-preview,63.2,57.1,udiff,aider,0.31.2-dev,1981105-dirty,5/4/24
gpt-4-turbo-2024-04-09,64.4,49.2,diff,aider --gpt-4-turbo,0.30.1,e610e5b,5/1/24
Llama3 70B,49.2,38.6,diff,aider --model groq/llama3-70b-8192,0.32.0,b5bb453,5/3/24
1 model second first format command version commit commits date
2 claude-3-opus-20240229 68.4 53.4 diff aider --opus 0.30.1 f4b1797 f4b1797 5/2/24
gpt-4-0613 67.7 46.6 diff aider -4 0.18.1 3aa17c4 12/16/23
gpt-4-turbo-2024-04-09 64.4 49.2 udiff aider --gpt-4-turbo 0.30.1 e610e5b 5/1/24
gemini-1.5-pro-latest 57.1 45.9 diff-fenced aider --model gemini/gemini-1.5-pro-latest 0.32.0 5d32dd7 5/3/24
gpt-3.5-turbo-1106 56.1 45.5 whole aider --model gpt-3.5-turbo-1106 0.30.1 7b14d77 4/30/24
3 claude-3-sonnet-20240229 54.9 43.6 whole aider --sonnet 0.25.0 a5f8076 a5f8076 3/6/24
4 Command-R+ 29.3 22.6 whole aider --model command-r-plus 0.28.0 a06c927 4/20/24
5 Deepseek Coder 54.5 47 whole aider --model openai/deepseek-coder 0.30.1 c07f793 c07f793 4/29/24
6 gemini-1.5-pro-latest 57.1 45.9 diff-fenced aider --model gemini/gemini-1.5-pro-latest 0.32.0 5d32dd7 5/3/24
7 gpt-3.5-turbo-0125 49.6 39.8 whole aider --35turbo 0.22.0 da14474 da14474 2/2/24
8 Llama3 70B gpt-3.5-turbo-0301 49.2 57.9 38.6 50.4 diff whole aider --model groq/llama3-70b-8192 aider --model gpt-3.5-turbo-0301 0.32.0 0.16.4-dev b5bb453 44388db-dirty 5/3/24 11/6/23
9 Command-R+ gpt-3.5-turbo-0613 29.3 50.4 22.6 38.3 whole aider --model command-r-plus aider --model gpt-3.5-turbo-0613 0.28.0 0.16.4-dev a06c927 93aa497-dirty 4/20/24 11/7/23
10 gpt-3.5-turbo-1106 56.1 45.5 whole aider --model gpt-3.5-turbo-1106 0.30.1 7b14d77 4/30/24
11 gpt-4-0125-preview 66.2 55.6 udiff aider 0.22.1-dev edcf9b1 1/25/24
12 gpt-4-0314 68.4 51.1 diff aider --model gpt-4-0314 0.7.2 506bfe2 c7c548d 7/1/23
13 gpt-4-0613 67.7 46.6 diff aider -4 0.18.1 3aa17c4 12/16/23
14 gpt-4-1106-preview 63.2 57.1 udiff aider 0.31.2-dev 1981105-dirty 5/4/24
15 gpt-4-turbo-2024-04-09 64.4 49.2 diff aider --gpt-4-turbo 0.30.1 e610e5b 5/1/24
16 Llama3 70B 49.2 38.6 diff aider --model groq/llama3-70b-8192 0.32.0 b5bb453 5/3/24

38
benchmark/benchmark.py
View file

@ -870,26 +870,38 @@ def summarize_results(dirname):
)
csv = []
csv.append(' '.join(variants['model']))
csv.append(f"{percents[1]:.1f}")
csv.append(f"{percents[0]:.1f}")
csv.append(' '.join(variants['edit_format']))
csv.append('aider')
csv.append('version')
for hsh in variants['commit_hash']:
csv.append(" ".join(variants["model"]))
second = percents.get(1, 0)
first = percents.get(0, 0)
csv.append(f"{second:.1f}")
csv.append(f"{first:.1f}")
csv.append(" ".join(variants["edit_format"]))
csv.append("aider")
versions = set()
for hsh in variants["commit_hash"]:
if not hsh:
continue
hsh = hsh.split("-")[0]
try:
version = subprocess.check_output(
['git', 'show', f'{hsh}:aider/__init__.py'],
universal_newlines=True
["git", "show", f"{hsh}:aider/__init__.py"], universal_newlines=True
)
version = re.search(r'__version__ = "(.*)"', version).group(1)
csv.append(version)
versions.add(version)
except subprocess.CalledProcessError:
csv.append('unknown')
csv.append(' '.join(variants['commit_hash']))
pass
csv.append(" ".join(sorted(versions)))
commit_hashes = variants.get("commit_hash", [])
if all(commit_hashes):
csv.append(" ".join(commit_hashes))
else:
csv.append("")
csv.append(dirname.name[:10])
csv = ','.join(csv)
csv = ",".join(csv)
print()
print("Add this to _data/leaderboard.csv:")
print(csv)
console.rule()

3
docs/leaderboard.md
View file

@ -26,7 +26,8 @@ it will work best with models that score well on the code editing benchmark.
</tr>
</thead>
<tbody>
{% for row in site.data.leaderboard %}
{% assign sorted = site.data.leaderboard | sort: 'second' | reverse %}
{% for row in sorted %}
<tr style="border-bottom: 1px solid #ddd;">
<td style="padding: 8px;">{{ row.model }}</td>
<td style="padding: 8px;">{{ row.second }}%</td>