From 1ced72b7286de71f6d8aac97c3f038842ef0b52e Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Wed, 14 Aug 2024 06:31:20 -0700 Subject: [PATCH] update models-over-time --- aider/website/_data/edit_leaderboard.yml | 12 + aider/website/assets/models-over-time.svg | 945 ++++++++++++++++------ 2 files changed, 722 insertions(+), 235 deletions(-) diff --git a/aider/website/_data/edit_leaderboard.yml b/aider/website/_data/edit_leaderboard.yml index 72dd9c96b..426c86336 100644 --- a/aider/website/_data/edit_leaderboard.yml +++ b/aider/website/_data/edit_leaderboard.yml @@ -577,6 +577,7 @@ pass_rate_2: 77.4 percent_cases_well_formed: 99.2 error_outputs: 23 + released: 2024-06-20 num_malformed_responses: 4 num_with_malformed_responses: 1 user_asks: 2 @@ -603,6 +604,7 @@ num_malformed_responses: 0 num_with_malformed_responses: 0 user_asks: 0 + released: 2024-03-13 lazy_comments: 0 syntax_errors: 0 indentation_errors: 0 @@ -644,6 +646,7 @@ commit_hash: d31eef3-dirty pass_rate_1: 40.6 pass_rate_2: 55.6 + released: 2024-07-18 percent_cases_well_formed: 100.0 error_outputs: 1 num_malformed_responses: 0 @@ -668,6 +671,7 @@ pass_rate_1: 60.9 pass_rate_2: 69.9 percent_cases_well_formed: 97.7 + released: 2024-06-28 error_outputs: 58 num_malformed_responses: 13 num_with_malformed_responses: 3 @@ -690,6 +694,7 @@ commit_hash: f7ce78b-dirty pass_rate_1: 46.6 pass_rate_2: 63.9 + released: 2024-07-23 percent_cases_well_formed: 92.5 error_outputs: 84 num_malformed_responses: 19 @@ -716,6 +721,7 @@ percent_cases_well_formed: 100.0 error_outputs: 0 num_malformed_responses: 0 + released: 2024-07-23 num_with_malformed_responses: 0 user_asks: 0 lazy_comments: 0 @@ -738,6 +744,7 @@ pass_rate_2: 72.9 percent_cases_well_formed: 97.7 error_outputs: 13 + released: 2024-07-24 num_malformed_responses: 3 num_with_malformed_responses: 3 user_asks: 1 @@ -763,6 +770,7 @@ error_outputs: 3 num_malformed_responses: 0 num_with_malformed_responses: 0 + released: 2024-07-24 user_asks: 3 lazy_comments: 0 syntax_errors: 1 @@ -785,6 +793,7 @@ percent_cases_well_formed: 100.0 error_outputs: 27 num_malformed_responses: 0 + released: 2024-07-23 num_with_malformed_responses: 0 user_asks: 23 lazy_comments: 8 @@ -810,6 +819,7 @@ num_malformed_responses: 0 num_with_malformed_responses: 0 user_asks: 0 + released: 2024-07-23 lazy_comments: 0 syntax_errors: 0 indentation_errors: 0 @@ -838,6 +848,7 @@ indentation_errors: 2 exhausted_context_windows: 0 test_timeouts: 5 + released: 2024-08-06 command: aider --model openai/gpt-4o-2024-08-06 date: 2024-08-06 versions: 0.48.1-dev @@ -863,6 +874,7 @@ test_timeouts: 0 command: aider --model openai/chatgpt-4o-latest date: 2024-08-14 + released: 2024-08-08 versions: 0.50.2-dev seconds_per_case: 26.3 total_cost: 3.6113 diff --git a/aider/website/assets/models-over-time.svg b/aider/website/assets/models-over-time.svg index 994dab6ae..a4fe87061 100644 --- a/aider/website/assets/models-over-time.svg +++ b/aider/website/assets/models-over-time.svg @@ -1,16 +1,16 @@ - + - 2024-05-15T11:52:34.512395 + 2024-08-14T06:29:51.758534 image/svg+xml - Matplotlib v3.8.4, https://matplotlib.org/ + Matplotlib v3.9.2, https://matplotlib.org/ @@ -21,25 +21,25 @@ - - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -441,12 +474,12 @@ z - + - + @@ -457,9 +490,47 @@ z - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - + + - + - - + - - - - - + + + + - - + + - + - + - - - - + + + + - - + + - + - + - - - - + + + + - - + + - + - + - + - + - - + + - + - + - - - - + + + + + + + - + + + + + + + + + + + + + + + + + + + - + - + + + + + + + + + - - - - - + - + - + - + @@ -1282,28 +1434,9 @@ z - - - - - - - - - - - - - - - - - - - - + - + - + - + @@ -1356,9 +1489,9 @@ z - + - + @@ -1379,9 +1512,9 @@ z - + - + - + - + @@ -1445,9 +1578,9 @@ z - + - + @@ -1460,9 +1593,9 @@ z - + - + @@ -1483,9 +1616,9 @@ z - + - + @@ -1510,50 +1643,9 @@ z - + - - - - + @@ -1571,9 +1663,9 @@ z - + - + @@ -1590,30 +1682,121 @@ z - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -1672,9 +1855,184 @@ z - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1735,8 +2210,8 @@ z - - + +