From 338c4cfb61fa38abaaed4c867c67b7702a49f4b8 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Mon, 1 Jul 2024 18:51:34 -0300 Subject: [PATCH] added haiku to leaderboard --- website/_data/edit_leaderboard.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/website/_data/edit_leaderboard.yml b/website/_data/edit_leaderboard.yml index c7c4a0efc..b2c1fc469 100644 --- a/website/_data/edit_leaderboard.yml +++ b/website/_data/edit_leaderboard.yml @@ -681,4 +681,27 @@ versions: 0.39.1-dev seconds_per_case: 30.2 total_cost: 0.0857 + +- dirname: 2024-07-01-21-41-48--haiku-whole + test_cases: 133 + model: openrouter/anthropic/claude-3-haiku + edit_format: whole + commit_hash: 75f506d + pass_rate_1: 40.6 + pass_rate_2: 47.4 + percent_cases_well_formed: 100.0 + error_outputs: 6 + num_malformed_responses: 0 + num_with_malformed_responses: 0 + user_asks: 0 + lazy_comments: 0 + syntax_errors: 0 + indentation_errors: 0 + exhausted_context_windows: 0 + test_timeouts: 2 + command: aider --model openrouter/anthropic/claude-3-haiku + date: 2024-07-01 + versions: 0.41.1-dev + seconds_per_case: 7.1 + total_cost: 0.1946 \ No newline at end of file