From 3f6ae4b2d9d9036b054b222d858ae67cb40edb25 Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Thu, 8 Aug 2024 14:54:59 -0300 Subject: [PATCH 1/4] Handle retries at a higher level; exceptions come out of the streaming completion object --- aider/coders/base_coder.py | 14 ++++++-- aider/sendchat.py | 67 ++++++++++++-------------------------- 2 files changed, 33 insertions(+), 48 deletions(-) diff --git a/aider/coders/base_coder.py b/aider/coders/base_coder.py index ddf9e85e5..3a1d3d981 100755 --- a/aider/coders/base_coder.py +++ b/aider/coders/base_coder.py @@ -30,7 +30,7 @@ from aider.llm import litellm from aider.mdstream import MarkdownStream from aider.repo import GitRepo from aider.repomap import RepoMap -from aider.sendchat import send_with_retries +from aider.sendchat import retry_exceptions, send_completion from aider.utils import format_content, format_messages, is_image_file from ..dump import dump # noqa: F401 @@ -891,6 +891,8 @@ class Coder: else: self.mdstream = None + retry_delay = 0.125 + self.usage_report = None exhausted = False interrupted = False @@ -899,6 +901,14 @@ class Coder: try: yield from self.send(messages, functions=self.functions) break + except retry_exceptions() as err: + self.io.tool_error(str(err)) + retry_delay *= 2 + if retry_delay > 60: + break + self.io.tool_output(f"Retrying in {retry_delay:.1f} seconds...") + time.sleep(retry_delay) + continue except KeyboardInterrupt: interrupted = True break @@ -1161,7 +1171,7 @@ class Coder: interrupted = False try: - hash_object, completion = send_with_retries( + hash_object, completion = send_completion( model.name, messages, functions, diff --git a/aider/sendchat.py b/aider/sendchat.py index c1ccbbe0e..e767e29ce 100644 --- a/aider/sendchat.py +++ b/aider/sendchat.py @@ -14,53 +14,28 @@ CACHE = None # CACHE = Cache(CACHE_PATH) +def retry_exceptions(): + import httpx + + return ( + httpx.ConnectError, + httpx.RemoteProtocolError, + httpx.ReadTimeout, + litellm.exceptions.APIConnectionError, + litellm.exceptions.APIError, + litellm.exceptions.RateLimitError, + litellm.exceptions.ServiceUnavailableError, + litellm.exceptions.Timeout, + litellm.exceptions.InternalServerError, + litellm.llms.anthropic.AnthropicError, + ) + + def lazy_litellm_retry_decorator(func): def wrapper(*args, **kwargs): - import httpx - - def should_giveup(e): - if not hasattr(e, "status_code"): - return False - - if type(e) in ( - httpx.ConnectError, - httpx.RemoteProtocolError, - httpx.ReadTimeout, - litellm.exceptions.APIConnectionError, - litellm.exceptions.APIError, - litellm.exceptions.RateLimitError, - litellm.exceptions.ServiceUnavailableError, - litellm.exceptions.Timeout, - litellm.exceptions.InternalServerError, - litellm.llms.anthropic.AnthropicError, - ): - return False - - # These seem to return .status_code = "" - # litellm._should_retry() expects an int and throws a TypeError - # - # litellm.llms.anthropic.AnthropicError - # litellm.exceptions.APIError - if not e.status_code: - return False - - return not litellm._should_retry(e.status_code) - decorated_func = backoff.on_exception( backoff.expo, - ( - httpx.ConnectError, - httpx.RemoteProtocolError, - httpx.ReadTimeout, - litellm.exceptions.APIConnectionError, - litellm.exceptions.APIError, - litellm.exceptions.RateLimitError, - litellm.exceptions.ServiceUnavailableError, - litellm.exceptions.Timeout, - litellm.exceptions.InternalServerError, - litellm.llms.anthropic.AnthropicError, - ), - giveup=should_giveup, + retry_exceptions(), max_time=60, on_backoff=lambda details: print( f"{details.get('exception', 'Exception')}\nRetry in {details['wait']:.1f} seconds." @@ -71,8 +46,7 @@ def lazy_litellm_retry_decorator(func): return wrapper -@lazy_litellm_retry_decorator -def send_with_retries( +def send_completion( model_name, messages, functions, stream, temperature=0, extra_headers=None, max_tokens=None ): from aider.llm import litellm @@ -108,9 +82,10 @@ def send_with_retries( return hash_object, res +@lazy_litellm_retry_decorator def simple_send_with_retries(model_name, messages): try: - _hash, response = send_with_retries( + _hash, response = send_completion( model_name=model_name, messages=messages, functions=None, From 608c80404e83d43babfe7bf1e5dac8df05725701 Mon Sep 17 00:00:00 2001 From: "Paul Gauthier (aider)" Date: Thu, 8 Aug 2024 14:58:43 -0300 Subject: [PATCH 2/4] feat: implement countdown for retry in 0.1-second increments --- aider/coders/base_coder.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/aider/coders/base_coder.py b/aider/coders/base_coder.py index 3a1d3d981..a69f23241 100755 --- a/aider/coders/base_coder.py +++ b/aider/coders/base_coder.py @@ -907,7 +907,12 @@ class Coder: if retry_delay > 60: break self.io.tool_output(f"Retrying in {retry_delay:.1f} seconds...") - time.sleep(retry_delay) + countdown = retry_delay + while countdown > 0: + print(f"Retrying in {countdown:.1f} seconds...\r", end="") + time.sleep(0.1) + countdown -= 0.1 + print(" " * 50 + "\r", end="") # Clear the line after countdown continue except KeyboardInterrupt: interrupted = True From 2369489321c10e472c4303c0481fddf18e6cc23c Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Thu, 8 Aug 2024 15:19:24 -0300 Subject: [PATCH 3/4] Clean up countdown --- aider/coders/base_coder.py | 8 ++---- tests/basic/test_sendchat.py | 47 ------------------------------------ 2 files changed, 2 insertions(+), 53 deletions(-) delete mode 100644 tests/basic/test_sendchat.py diff --git a/aider/coders/base_coder.py b/aider/coders/base_coder.py index a69f23241..ae596fd18 100755 --- a/aider/coders/base_coder.py +++ b/aider/coders/base_coder.py @@ -70,6 +70,7 @@ class Coder: lint_outcome = None test_outcome = None multi_response_content = "" + partial_response_content = "" @classmethod def create( @@ -907,12 +908,7 @@ class Coder: if retry_delay > 60: break self.io.tool_output(f"Retrying in {retry_delay:.1f} seconds...") - countdown = retry_delay - while countdown > 0: - print(f"Retrying in {countdown:.1f} seconds...\r", end="") - time.sleep(0.1) - countdown -= 0.1 - print(" " * 50 + "\r", end="") # Clear the line after countdown + time.sleep(retry_delay) continue except KeyboardInterrupt: interrupted = True diff --git a/tests/basic/test_sendchat.py b/tests/basic/test_sendchat.py deleted file mode 100644 index 6ac02a47b..000000000 --- a/tests/basic/test_sendchat.py +++ /dev/null @@ -1,47 +0,0 @@ -import unittest -from unittest.mock import MagicMock, patch - -import httpx - -from aider.llm import litellm -from aider.sendchat import send_with_retries - - -class PrintCalled(Exception): - pass - - -class TestSendChat(unittest.TestCase): - @patch("litellm.completion") - @patch("builtins.print") - def test_send_with_retries_rate_limit_error(self, mock_print, mock_completion): - mock = MagicMock() - mock.status_code = 500 - - # Set up the mock to raise - mock_completion.side_effect = [ - litellm.exceptions.RateLimitError( - "rate limit exceeded", - response=mock, - llm_provider="llm_provider", - model="model", - ), - None, - ] - - # Call the send_with_retries method - send_with_retries("model", ["message"], None, False) - mock_print.assert_called_once() - - @patch("litellm.completion") - @patch("builtins.print") - def test_send_with_retries_connection_error(self, mock_print, mock_completion): - # Set up the mock to raise - mock_completion.side_effect = [ - httpx.ConnectError("Connection error"), - None, - ] - - # Call the send_with_retries method - send_with_retries("model", ["message"], None, False) - mock_print.assert_called_once() From 109f197f527c15c15f4c7129444a1853c7c1a34d Mon Sep 17 00:00:00 2001 From: Paul Gauthier Date: Thu, 8 Aug 2024 15:22:58 -0300 Subject: [PATCH 4/4] feat: Add tests for simple_send_with_retries function --- tests/basic/test_sendchat.py | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/basic/test_sendchat.py diff --git a/tests/basic/test_sendchat.py b/tests/basic/test_sendchat.py new file mode 100644 index 000000000..f77e687ef --- /dev/null +++ b/tests/basic/test_sendchat.py @@ -0,0 +1,47 @@ +import unittest +from unittest.mock import MagicMock, patch + +import httpx + +from aider.llm import litellm +from aider.sendchat import simple_send_with_retries + + +class PrintCalled(Exception): + pass + + +class TestSendChat(unittest.TestCase): + @patch("litellm.completion") + @patch("builtins.print") + def test_simple_send_with_retries_rate_limit_error(self, mock_print, mock_completion): + mock = MagicMock() + mock.status_code = 500 + + # Set up the mock to raise + mock_completion.side_effect = [ + litellm.exceptions.RateLimitError( + "rate limit exceeded", + response=mock, + llm_provider="llm_provider", + model="model", + ), + None, + ] + + # Call the simple_send_with_retries method + simple_send_with_retries("model", ["message"]) + mock_print.assert_called_once() + + @patch("litellm.completion") + @patch("builtins.print") + def test_simple_send_with_retries_connection_error(self, mock_print, mock_completion): + # Set up the mock to raise + mock_completion.side_effect = [ + httpx.ConnectError("Connection error"), + None, + ] + + # Call the simple_send_with_retries method + simple_send_with_retries("model", ["message"]) + mock_print.assert_called_once()