From 1d0bc3dcb6c28f00ae031f6cc0943404c5cb74ee Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Tue, 14 Nov 2023 16:03:50 -0800 Subject: [PATCH] Updated benchmark reports --- assets/benchmarks-1106.svg | 704 ++++++++++++----------- assets/benchmarks-speed-1106.svg | 932 +++++++++++++++---------------- benchmark/benchmark.py | 4 +- docs/benchmarks-1106.md | 17 +- docs/benchmarks-speed-1106.md | 14 +- 5 files changed, 796 insertions(+), 875 deletions(-) diff --git a/assets/benchmarks-1106.svg b/assets/benchmarks-1106.svg index ab42481da..dbe6bc202 100644 --- a/assets/benchmarks-1106.svg +++ b/assets/benchmarks-1106.svg @@ -6,7 +6,7 @@ - 2023-11-08T11:16:45.721593 + 2023-11-14T15:58:23.818085 image/svg+xml @@ -30,8 +30,8 @@ z - - - + - + - + - + - + @@ -384,7 +384,7 @@ z - + - + - + @@ -449,7 +449,7 @@ z - + @@ -461,12 +461,12 @@ z - + - + - + @@ -507,12 +507,12 @@ z - + - + @@ -520,7 +520,7 @@ z - + @@ -532,12 +532,12 @@ z - + - + @@ -545,7 +545,7 @@ z - + - - - - - - - - - - - - - - - - - - - - - - - - - - + - - + - + - + - + - + - + - + - + @@ -905,18 +726,18 @@ L 404.21745 158.97645 - + - + - + @@ -924,18 +745,18 @@ L 404.21745 118.844675 - + - + - + +" clip-path="url(#p2a9d19ec75)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.2; stroke-linecap: square"/> - + @@ -1004,7 +825,7 @@ L 404.21745 38.581125 - + + + + + @@ -1197,105 +1103,105 @@ z - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - - - @@ -1305,7 +1211,7 @@ L 404.21745 38.581125 - + - + @@ -1370,7 +1276,7 @@ z - + @@ -1378,7 +1284,7 @@ z - + @@ -1386,23 +1292,23 @@ z - + - - + + - + - + @@ -1410,7 +1316,7 @@ z - + @@ -1418,111 +1324,111 @@ z - + - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#p2a9d19ec75)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - - - + + - + @@ -1579,7 +1531,7 @@ z - + @@ -1598,7 +1550,7 @@ z - + @@ -1614,17 +1566,17 @@ z - - - + - + @@ -1700,7 +1652,7 @@ z - + @@ -1842,6 +1794,38 @@ z + + + + @@ -1964,8 +1948,8 @@ z - - + + diff --git a/assets/benchmarks-speed-1106.svg b/assets/benchmarks-speed-1106.svg index 93876b4d7..b66ac0c82 100644 --- a/assets/benchmarks-speed-1106.svg +++ b/assets/benchmarks-speed-1106.svg @@ -6,7 +6,7 @@ - 2023-11-08T11:50:38.052418 + 2023-11-14T16:00:46.511433 image/svg+xml @@ -30,8 +30,8 @@ z - - - + - + - + - + - + @@ -384,7 +384,7 @@ z - + - + - + @@ -449,7 +449,7 @@ z - + @@ -461,12 +461,12 @@ z - + - + - + @@ -507,12 +507,12 @@ z - + - + @@ -520,7 +520,7 @@ z - + @@ -532,12 +532,12 @@ z - + - + @@ -545,7 +545,7 @@ z - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -1182,7 +1103,7 @@ z - + + + @@ -1246,105 +1197,105 @@ z - +" clip-path="url(#pa99eb49187)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3e6a8; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - +" clip-path="url(#pa99eb49187)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#pa99eb49187)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#pa99eb49187)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> +" clip-path="url(#pa99eb49187)" style="fill: #b3d1e6; stroke: #ffffff; stroke-width: 1.5; stroke-linejoin: miter"/> - - - @@ -1354,7 +1305,7 @@ L 404.21745 38.581125 - + @@ -1364,7 +1315,7 @@ L 404.21745 38.581125 - + @@ -1374,7 +1325,7 @@ L 404.21745 38.581125 - + - + @@ -1425,37 +1376,8 @@ z - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1506,7 +1457,7 @@ z - + @@ -1802,6 +1753,13 @@ L 975 0 L 413 0 L 413 4606 z +" transform="scale(0.015625)"/> + @@ -1815,8 +1773,8 @@ z - - + + diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index f8ce04e24..a519990c8 100755 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -77,8 +77,8 @@ def show_stats(dirnames, graphs): elif row.model.startswith(gpt4): row.model = gpt4 + "\n" + row.model[len(gpt4) :] - if row.model == "gpt-4\n-1106-preview": - row.model += "\n(preliminary)" + # if row.model == "gpt-4\n-1106-preview": + # row.model += "\n(preliminary)" if row.completed_tests < 133: print(f"Warning: {row.dir_name} is incomplete: {row.completed_tests}") diff --git a/docs/benchmarks-1106.md b/docs/benchmarks-1106.md index a907aefeb..c99f40fff 100644 --- a/docs/benchmarks-1106.md +++ b/docs/benchmarks-1106.md @@ -2,6 +2,8 @@ [![benchmark results](../assets/benchmarks-1106.svg)](https://aider.chat/assets/benchmarks-1106.svg) +[![benchmark results](../assets/benchmarks-speed-1106.svg)](https://aider.chat/assets/benchmarks-speed-1106.svg) + [OpenAI just released new versions of GPT-3.5 and GPT-4](https://openai.com/blog/new-models-and-developer-products-announced-at-devday), and there's a lot of interest about their ability to code compared to the previous versions. @@ -44,22 +46,11 @@ The benchmark gives aider two tries to complete the task: For now, I have only benchmarked the GPT-4 models using the `diff` edit method. This is the edit format that aider uses by default with gpt-4. -- The new `gpt-4-1106-preview` model seems **much faster** than the earlier GPT-4 models. I won't be able to properly quantify this until the rate limits loosen up. +- The new `gpt-4-1106-preview` model seems **2-2.5X faster** than the June GPT-4 model. - **It seems better at producing correct code on the first try**. It gets 53% of the coding exercises correct, without needing to see errors from the test suite. Previous models only get 46-47% of the exercises correct on the first try. - The new model seems to perform similar -(~62%) to the old models (63-64%) after their second chance to correct bugs by reviewing test suite error output. - -**These are preliminary results.** -OpenAI is enforcing very low -rate limits on the new GPT-4 model. -The rate limiting disrupts the the benchmarking process, -requiring it to be paused and restarted frequently. -It took ~20 partial runs over ~2 days to complete all 133 Exercism problems. -The benchmarking harness is designed to stop/restart in this manner, -but results from a single "clean" run would be more trustworthy. -Once the rate limits are relaxed I will do a clean -run of the entire benchmark. +(~65%) to the old models (63-64%) after their second chance to correct bugs by reviewing test suite error output. ### gpt-3.5-turbo-1106 diff --git a/docs/benchmarks-speed-1106.md b/docs/benchmarks-speed-1106.md index 7641296a5..21b9395b0 100644 --- a/docs/benchmarks-speed-1106.md +++ b/docs/benchmarks-speed-1106.md @@ -37,17 +37,5 @@ generate responses which primarily consist of source code. Some observations: - **GPT-3.5 got 6-11x faster.** The `gpt-3.5-turbo-1106` model is 6-11x faster than the June (0613) version which has been the default `gpt-3.5-turbo` model. -- **GPT-4 Turbo is 4-5x faster.** The new `gpt-4-1106-preview` model is 4-5x faster than the June (0613) version which has been the default `gpt-4` model. +- **GPT-4 Turbo is 2-2.5x faster.** The new `gpt-4-1106-preview` model is 2-2.5x faster than the June (0613) version which has been the default `gpt-4` model. - The old March (0301) version of GPT-3.5 is actually faster than the June (0613) version. This was a surprising discovery. - -**These are preliminary results.** -OpenAI is enforcing very low -rate limits on the new GPT-4 model. -The rate limiting disrupts the benchmarking process, -requiring it to run single threaded, pause and restart frequently. -These anomolous conditions make it slow to -benchmark the new model, and make -it less reliable to compare the results with -benchmark runs against the older model versions. -Once the rate limits are relaxed I will do a clean -run of the entire benchmark suite.