From f21c35c42a9a091e5289868ec3a1a5476efb55f3 Mon Sep 17 00:00:00 2001 From: "Paul Gauthier (aider)" Date: Thu, 26 Sep 2024 07:58:26 -0700 Subject: [PATCH] feat: Add benchmark data table --- aider/website/_posts/2024-09-26-senior-junior.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/aider/website/_posts/2024-09-26-senior-junior.md b/aider/website/_posts/2024-09-26-senior-junior.md index e836963b9..7764eaa59 100644 --- a/aider/website/_posts/2024-09-26-senior-junior.md +++ b/aider/website/_posts/2024-09-26-senior-junior.md @@ -11,4 +11,19 @@ nav_exclude: true # Separating code reasoning and editing +Here's a table containing the benchmark data for different model configurations: + +| Model | Junior Model | Junior Edit Format | Pass Rate 2 (%) | Seconds per Case | Total Cost ($) | +|-------|--------------|---------------------|-----------------|-------------------|----------------| +| claude-3.5-sonnet | claude-3.5-sonnet | junior-diff | 80.5 | 25.1 | 4.95 | +| o1-mini | gpt-4o | junior-diff | 70.7 | 23.7 | 9.32 | +| gpt-4o | gpt-4o | junior-diff | 75.2 | 18.2 | 6.09 | +| o1-preview | gpt-4o | junior-diff | 80.5 | 42.3 | 39.38 | +| o1-preview | claude-3.5-sonnet | junior-diff | 82.7 | 44.9 | 37.62 | +| o1-mini | deepseek | junior-diff | 69.2 | 52.2 | 5.79 | +| o1-preview | deepseek | junior-diff | 80.5 | 73.2 | 35.79 | +| o1-preview | deepseek | junior-whole | 85.0 | 67.4 | 35.32 | + +This table provides a comparison of different model configurations, showing their performance in terms of pass rate, processing time, and cost. +