From f36bcd9b73f539df0627c3c45f3573c88b404e77 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Wed, 15 May 2024 09:45:11 -0700 Subject: [PATCH 1/4] Added gpt-4-turbo-2024-04-09 (udiff) to the leaderboard --- _data/edit_leaderboard.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/_data/edit_leaderboard.yml b/_data/edit_leaderboard.yml index dd0d5695e..dec866afe 100644 --- a/_data/edit_leaderboard.yml +++ b/_data/edit_leaderboard.yml @@ -426,3 +426,24 @@ seconds_per_case: 6.0 total_cost: 0.0000 +- dirname: 2024-04-12-22-11-22--gpt-4-turbo-2024-04-09-plain-udiff + test_cases: 133 + model: gpt-4-turbo-2024-04-09 (udiff) + edit_format: udiff + commit_hash: 9b2e697-dirty + pass_rate_1: 44.4 + pass_rate_2: 60.9 + percent_cases_well_formed: 82.0 + error_outputs: 72 + num_malformed_responses: 24 + user_asks: 0 + lazy_comments: 0 + syntax_errors: 0 + indentation_errors: 4 + exhausted_context_windows: 0 + test_timeouts: 5 + command: aider --model gpt-4-turbo-2024-04-09 + date: 2024-04-12 + versions: 0.28.1-dev + seconds_per_case: 22.0 + total_cost: 6.3924 \ No newline at end of file From efc9e56b23440a60cc5bae6adaa0406ba4021682 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Wed, 15 May 2024 09:46:13 -0700 Subject: [PATCH 2/4] cleanup --- _data/edit_leaderboard.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/_data/edit_leaderboard.yml b/_data/edit_leaderboard.yml index dec866afe..dd0d5695e 100644 --- a/_data/edit_leaderboard.yml +++ b/_data/edit_leaderboard.yml @@ -426,24 +426,3 @@ seconds_per_case: 6.0 total_cost: 0.0000 -- dirname: 2024-04-12-22-11-22--gpt-4-turbo-2024-04-09-plain-udiff - test_cases: 133 - model: gpt-4-turbo-2024-04-09 (udiff) - edit_format: udiff - commit_hash: 9b2e697-dirty - pass_rate_1: 44.4 - pass_rate_2: 60.9 - percent_cases_well_formed: 82.0 - error_outputs: 72 - num_malformed_responses: 24 - user_asks: 0 - lazy_comments: 0 - syntax_errors: 0 - indentation_errors: 4 - exhausted_context_windows: 0 - test_timeouts: 5 - command: aider --model gpt-4-turbo-2024-04-09 - date: 2024-04-12 - versions: 0.28.1-dev - seconds_per_case: 22.0 - total_cost: 6.3924 \ No newline at end of file From 6d2b9d6699ebec73ec2a664bba64567026eb5eec Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Wed, 15 May 2024 09:49:38 -0700 Subject: [PATCH 3/4] Added gpt-4-turbo-2024-04-09 (diff) to the leaderboard --- _data/edit_leaderboard.yml | 22 ++++++++++++++++++++++ _data/refactor_leaderboard.yml | 25 ++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/_data/edit_leaderboard.yml b/_data/edit_leaderboard.yml index dd0d5695e..691214ee8 100644 --- a/_data/edit_leaderboard.yml +++ b/_data/edit_leaderboard.yml @@ -426,3 +426,25 @@ seconds_per_case: 6.0 total_cost: 0.0000 +- dirname: 2024-04-12-22-18-20--gpt-4-turbo-2024-04-09-plain-diff + test_cases: 33 + model: gpt-4-turbo-2024-04-09 (diff) + edit_format: diff + commit_hash: 9b2e697-dirty + pass_rate_1: 48.5 + pass_rate_2: 57.6 + percent_cases_well_formed: 100.0 + error_outputs: 15 + num_malformed_responses: 0 + user_asks: 15 + lazy_comments: 0 + syntax_errors: 0 + indentation_errors: 0 + exhausted_context_windows: 0 + test_timeouts: 0 + command: aider --model gpt-4-turbo-2024-04-09 + date: 2024-04-12 + versions: 0.28.1-dev + seconds_per_case: 17.6 + total_cost: 1.6205 + \ No newline at end of file diff --git a/_data/refactor_leaderboard.yml b/_data/refactor_leaderboard.yml index 8b3816a1d..22d1fb666 100644 --- a/_data/refactor_leaderboard.yml +++ b/_data/refactor_leaderboard.yml @@ -120,4 +120,27 @@ date: 2024-05-13 versions: 0.34.1-dev seconds_per_case: 27.8 - total_cost: 0.0000 \ No newline at end of file + total_cost: 0.0000 + +- dirname: 2024-04-10-13-26-18--refac-gpt-4-turbo-2024-04-09-diff + test_cases: 88 + model: gpt-4-turbo-2024-04-09 (diff) + edit_format: diff + commit_hash: 7875418 + pass_rate_1: 21.4 + percent_cases_well_formed: 6.8 + error_outputs: 247 + num_malformed_responses: 82 + user_asks: 1 + lazy_comments: 2 + syntax_errors: 3 + indentation_errors: 8 + exhausted_context_windows: 0 + test_timeouts: 0 + command: aider --model gpt-4-turbo-2024-04-09 + date: 2024-04-10 + versions: 0.28.1-dev + seconds_per_case: 67.8 + total_cost: 20.4889 + + \ No newline at end of file From 46afd4e0e21f46815ebbf31d35ab825da0a66794 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Wed, 15 May 2024 09:50:47 -0700 Subject: [PATCH 4/4] added (udiff) label --- _data/edit_leaderboard.yml | 2 +- _data/refactor_leaderboard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/edit_leaderboard.yml b/_data/edit_leaderboard.yml index 691214ee8..0be7aaf59 100644 --- a/_data/edit_leaderboard.yml +++ b/_data/edit_leaderboard.yml @@ -256,7 +256,7 @@ - dirname: 2024-05-01-02-09-20--gpt-4-turbo-examples test_cases: 133 - model: gpt-4-turbo-2024-04-09 + model: gpt-4-turbo-2024-04-09 (udiff) edit_format: udiff commit_hash: e610e5b-dirty pass_rate_1: 48.1 diff --git a/_data/refactor_leaderboard.yml b/_data/refactor_leaderboard.yml index 22d1fb666..db4d3483f 100644 --- a/_data/refactor_leaderboard.yml +++ b/_data/refactor_leaderboard.yml @@ -40,7 +40,7 @@ total_cost: 27.9176 - dirname: 2024-04-09-21-49-54--refac-gpt-4-turbo-2024-04-09 test_cases: 88 - model: gpt-4-turbo-2024-04-09 + model: gpt-4-turbo-2024-04-09 (udiff) edit_format: udiff commit_hash: b75fdb9 pass_rate_1: 34.1