diff --git a/evalscope/models/anthropic_compatible.py b/evalscope/models/anthropic_compatible.py index 69a82108e..b1ca09403 100644 --- a/evalscope/models/anthropic_compatible.py +++ b/evalscope/models/anthropic_compatible.py @@ -1,8 +1,17 @@ import os import time -from typing import Any, Dict, List, Optional, Tuple, Union - -from anthropic import Anthropic, APIStatusError, AsyncAnthropic, BadRequestError, PermissionDeniedError +from typing import Any, Dict, List, Optional, Tuple, Type, Union + +from anthropic import ( + Anthropic, + APIStatusError, + AsyncAnthropic, + AuthenticationError, + BadRequestError, + NotFoundError, + PermissionDeniedError, + UnprocessableEntityError, +) from anthropic.types import Message from evalscope.api.messages import ChatMessage @@ -28,6 +37,18 @@ logger = get_logger() +# Client errors (prompt too long, invalid parameters, bad credentials, ...) are not +# recoverable by retrying. generate()/generate_async() degrade them gracefully via +# handle_bad_request(), so retry_call must let them through immediately instead of burning +# retries * retry_interval on every failing sample. +NON_RETRYABLE_ANTHROPIC_ERRORS: Tuple[Type[Exception], ...] = ( + BadRequestError, + AuthenticationError, + PermissionDeniedError, + NotFoundError, + UnprocessableEntityError, +) + class AnthropicCompatibleAPI(ModelAPI): """Anthropic API compatible model implementation. @@ -155,18 +176,23 @@ def generate( t_start = time.monotonic() ttft: Optional[float] = None - # Generate completion - message = retry_call( - self.client.messages.create, + # A streaming request is not complete when create() returns: the + # stream may still fail while its events are being consumed. Retry + # the whole request so a partial response is discarded and replaced + # by one complete response (mirrors OpenAICompatibleAPI). + def _create_and_collect() -> Tuple[Message, Optional[float]]: + message = self.client.messages.create(**request) + if isinstance(message, Message): + return message, None + return collect_stream_response(message, request_start=t_start) + + message, ttft = retry_call( + _create_and_collect, retries=config.retries, sleep_interval=config.retry_interval, - **request, + no_retry_exceptions=NON_RETRYABLE_ANTHROPIC_ERRORS, ) - # Handle streaming response - if not isinstance(message, Message): - message, ttft = collect_stream_response(message, request_start=t_start) - total_time = time.monotonic() - t_start response = message.model_dump() @@ -185,7 +211,7 @@ def generate( ) return output - except (BadRequestError, PermissionDeniedError) as ex: + except NON_RETRYABLE_ANTHROPIC_ERRORS as ex: return self.handle_bad_request(ex) async def generate_async( @@ -239,18 +265,22 @@ async def generate_async( t_start = time.monotonic() ttft: Optional[float] = None - # Async generation with retry - message = await async_retry_call( - self.async_client.messages.create, + # Keep stream consumption inside the retry boundary. If an async + # stream is interrupted, start a fresh request rather than + # returning or persisting its partial response. + async def _create_and_collect() -> Tuple[Message, Optional[float]]: + message = await self.async_client.messages.create(**request) + if isinstance(message, Message): + return message, None + return await async_collect_stream_response(message, request_start=t_start) + + message, ttft = await async_retry_call( + _create_and_collect, retries=config.retries, sleep_interval=config.retry_interval, - **request, + no_retry_exceptions=NON_RETRYABLE_ANTHROPIC_ERRORS, ) - # Handle streaming response - if not isinstance(message, Message): - message, ttft = await async_collect_stream_response(message, request_start=t_start) - total_time = time.monotonic() - t_start response = message.model_dump() @@ -268,7 +298,7 @@ async def generate_async( ) return output - except (BadRequestError, PermissionDeniedError) as ex: + except NON_RETRYABLE_ANTHROPIC_ERRORS as ex: return self.handle_bad_request(ex) def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice, diff --git a/evalscope/models/litellm_compatible.py b/evalscope/models/litellm_compatible.py index 784458172..5015a9bb1 100644 --- a/evalscope/models/litellm_compatible.py +++ b/evalscope/models/litellm_compatible.py @@ -9,7 +9,7 @@ """ import time -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Type from openai.types.chat import ChatCompletion @@ -35,6 +35,26 @@ logger = get_logger() +def _litellm_no_retry_exceptions() -> Tuple[Type[Exception], ...]: + """LiteLLM 4xx client-error types that must not be retried. + + LiteLLM maps provider client errors (context length exceeded, bad credentials, unknown + model, invalid parameters, ...) onto its own exception types. Retrying them only burns + retries * retry_interval per failing sample, so pass them as no-retry exceptions. + Transient errors (timeouts, connection failures, 429/5xx) stay retryable. + The litellm import stays lazy to keep this module importable without litellm. + """ + from litellm.exceptions import ( + AuthenticationError, + BadRequestError, + NotFoundError, + PermissionDeniedError, + UnprocessableEntityError, + ) + + return (AuthenticationError, BadRequestError, NotFoundError, PermissionDeniedError, UnprocessableEntityError) + + class LiteLLMAPI(ModelAPI): """LiteLLM model API provider. @@ -96,6 +116,7 @@ def generate( litellm.completion, retries=config.retries, sleep_interval=config.retry_interval, + no_retry_exceptions=_litellm_no_retry_exceptions(), **request, ) @@ -167,6 +188,7 @@ async def generate_async( litellm.acompletion, retries=config.retries, sleep_interval=config.retry_interval, + no_retry_exceptions=_litellm_no_retry_exceptions(), **request, ) diff --git a/evalscope/models/openai_compatible.py b/evalscope/models/openai_compatible.py index e470e0d53..97a611a2f 100644 --- a/evalscope/models/openai_compatible.py +++ b/evalscope/models/openai_compatible.py @@ -1,8 +1,17 @@ import os import time -from typing import Any, Dict, List, Optional, Tuple, Union - -from openai import APIStatusError, AsyncOpenAI, BadRequestError, OpenAI, PermissionDeniedError, UnprocessableEntityError +from typing import Any, Dict, List, Optional, Tuple, Type, Union + +from openai import ( + APIStatusError, + AsyncOpenAI, + AuthenticationError, + BadRequestError, + NotFoundError, + OpenAI, + PermissionDeniedError, + UnprocessableEntityError, +) from openai._types import NOT_GIVEN from openai.types.chat import ChatCompletion @@ -29,6 +38,18 @@ logger = get_logger() +# Client errors (context length exceeded, invalid parameters, bad credentials, ...) are not +# recoverable by retrying. generate()/generate_async() degrade them gracefully via +# handle_bad_request(), so retry_call must let them through immediately instead of burning +# retries * retry_interval on every failing sample. +NON_RETRYABLE_OPENAI_ERRORS: Tuple[Type[Exception], ...] = ( + BadRequestError, + AuthenticationError, + PermissionDeniedError, + NotFoundError, + UnprocessableEntityError, +) + class OpenAICompatibleAPI(ModelAPI): @@ -136,6 +157,7 @@ def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: _create_and_collect, retries=config.retries, sleep_interval=config.retry_interval, + no_retry_exceptions=NON_RETRYABLE_OPENAI_ERRORS, ) total_time = time.monotonic() - t_start @@ -158,7 +180,7 @@ def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: ) return output - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + except NON_RETRYABLE_OPENAI_ERRORS as ex: return self.handle_bad_request(ex) except ValueError as ex: logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') @@ -214,6 +236,7 @@ async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: _create_and_collect, retries=config.retries, sleep_interval=config.retry_interval, + no_retry_exceptions=NON_RETRYABLE_OPENAI_ERRORS, ) total_time = time.monotonic() - t_start @@ -235,7 +258,7 @@ async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: ) return output - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + except NON_RETRYABLE_OPENAI_ERRORS as ex: return self.handle_bad_request(ex) except ValueError as ex: logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') diff --git a/evalscope/models/openai_responses.py b/evalscope/models/openai_responses.py index cab538e93..527ecd819 100644 --- a/evalscope/models/openai_responses.py +++ b/evalscope/models/openai_responses.py @@ -2,7 +2,7 @@ import time from typing import Any, Dict, List, Optional, Tuple, Union -from openai import APIStatusError, BadRequestError, PermissionDeniedError, UnprocessableEntityError +from openai import APIStatusError from openai._types import NOT_GIVEN from evalscope.api.messages import ChatMessage @@ -13,7 +13,7 @@ from evalscope.utils.argument_utils import get_supported_params from evalscope.utils.function_utils import async_retry_call, retry_call -from .openai_compatible import OpenAICompatibleAPI +from .openai_compatible import NON_RETRYABLE_OPENAI_ERRORS, OpenAICompatibleAPI from .utils.openai import openai_handle_bad_request from .utils.openai_responses import ( async_collect_response_stream, @@ -84,6 +84,7 @@ def generate( self.client.responses.create, retries=config.retries, sleep_interval=config.retry_interval, + no_retry_exceptions=NON_RETRYABLE_OPENAI_ERRORS, **request, ) if not self._is_response_object(response): @@ -92,7 +93,7 @@ def generate( total_time = time.monotonic() - t_start return self._build_output(response, tools, total_time, ttft) - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + except NON_RETRYABLE_OPENAI_ERRORS as ex: return self.handle_bad_request(ex) except ValueError as ex: logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') @@ -115,6 +116,7 @@ async def generate_async( self.async_client.responses.create, retries=config.retries, sleep_interval=config.retry_interval, + no_retry_exceptions=NON_RETRYABLE_OPENAI_ERRORS, **request, ) if not self._is_response_object(response): @@ -123,7 +125,7 @@ async def generate_async( total_time = time.monotonic() - t_start return self._build_output(response, tools, total_time, ttft) - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + except NON_RETRYABLE_OPENAI_ERRORS as ex: return self.handle_bad_request(ex) except ValueError as ex: logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') diff --git a/evalscope/utils/function_utils.py b/evalscope/utils/function_utils.py index 1f8c5c3bf..a42e3a41d 100644 --- a/evalscope/utils/function_utils.py +++ b/evalscope/utils/function_utils.py @@ -3,7 +3,7 @@ import time from concurrent.futures import Future, ThreadPoolExecutor, wait from functools import wraps -from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, TypeVar, Union +from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple, Type, TypeVar, Union from evalscope.utils import asyncio_runtime from evalscope.utils.logger import get_logger @@ -68,31 +68,65 @@ def wrapper(*args, **kwargs): return wrapper -def retry_call(func, *args, retries=3, sleep_interval=0, **kwargs): - """Function that retries a function call up to `retries` times if an exception occurs.""" - for attempt in range(retries): +def retry_call( + func: Callable[..., T], + *args, + retries: Optional[int] = 3, + sleep_interval: float = 0, + no_retry_exceptions: Tuple[Type[Exception], ...] = (), + **kwargs, +) -> T: + """Function that retries a function call up to `retries` times if an exception occurs. + + `retries` is the total number of attempts: `None` falls back to the default and values + below 1 are clamped to 1, so the wrapped call is always made at least once. + + Exceptions matching `no_retry_exceptions` are re-raised immediately instead of being + retried (e.g. non-retryable 4xx client errors handled by the caller itself). + """ + total_attempts = 3 if retries is None else max(1, retries) + for attempt in range(total_attempts): try: return func(*args, **kwargs) + except no_retry_exceptions: + # Non-retryable errors must reach the caller's own handler without delay. + raise except Exception as e: - if attempt < retries - 1: + if attempt < total_attempts - 1: if sleep_interval > 0: - logger.warning(f'Attempt {attempt + 1} / {retries} failed: {e}. Retrying...') + logger.warning(f'Attempt {attempt + 1} / {total_attempts} failed: {e}. Retrying...') time.sleep(sleep_interval) else: raise async def async_retry_call( - func: Callable[..., Awaitable[T]], *args, retries: int = 3, sleep_interval: float = 0, **kwargs + func: Callable[..., Awaitable[T]], + *args, + retries: Optional[int] = 3, + sleep_interval: float = 0, + no_retry_exceptions: Tuple[Type[Exception], ...] = (), + **kwargs, ) -> T: - """Async version of retry_call. Retries an async function call up to `retries` times if an exception occurs.""" - for attempt in range(retries): + """Async version of retry_call. Retries an async function call up to `retries` times if an exception occurs. + + `retries` is the total number of attempts: `None` falls back to the default and values + below 1 are clamped to 1, so the wrapped call is always made at least once. + + Exceptions matching `no_retry_exceptions` are re-raised immediately instead of being + retried (e.g. non-retryable 4xx client errors handled by the caller itself). + """ + total_attempts = 3 if retries is None else max(1, retries) + for attempt in range(total_attempts): try: return await func(*args, **kwargs) + except no_retry_exceptions: + # Non-retryable errors must reach the caller's own handler without delay. + raise except Exception as e: - if attempt < retries - 1: + if attempt < total_attempts - 1: if sleep_interval > 0: - logger.warning(f'Attempt {attempt + 1} / {retries} failed: {e}. Retrying...') + logger.warning(f'Attempt {attempt + 1} / {total_attempts} failed: {e}. Retrying...') await asyncio.sleep(sleep_interval) else: raise diff --git a/tests/models/test_anthropic_stream_retry.py b/tests/models/test_anthropic_stream_retry.py new file mode 100644 index 000000000..ff58e88b6 --- /dev/null +++ b/tests/models/test_anthropic_stream_retry.py @@ -0,0 +1,124 @@ +import asyncio +from types import SimpleNamespace +from typing import Iterator, List, Optional + +import pytest +from anthropic.types import ( + ContentBlockDeltaEvent, + ContentBlockStartEvent, + Message, + MessageDeltaEvent, + MessageDeltaUsage, + MessageStartEvent, + TextBlock, + TextDelta, + Usage, +) +from anthropic.types.raw_message_delta_event import Delta + +from evalscope.api.model import GenerateConfig +from evalscope.models.anthropic_compatible import AnthropicCompatibleAPI + + +def _events(text: str) -> List: + """Build a minimal but valid Anthropic streaming event sequence yielding `text`.""" + return [ + MessageStartEvent( + type='message_start', + message=Message( + id='message-id', + type='message', + role='assistant', + model='test-model', + content=[], + stop_reason=None, + stop_sequence=None, + usage=Usage(input_tokens=3, output_tokens=0), + ), + ), + ContentBlockStartEvent(type='content_block_start', index=0, content_block=TextBlock(type='text', text='')), + ContentBlockDeltaEvent(type='content_block_delta', index=0, delta=TextDelta(type='text_delta', text=text)), + MessageDeltaEvent( + type='message_delta', + delta=Delta(stop_reason='end_turn', stop_sequence=None), + usage=MessageDeltaUsage(output_tokens=5), + ), + ] + + +def _prepare_api(monkeypatch: pytest.MonkeyPatch) -> AnthropicCompatibleAPI: + api = object.__new__(AnthropicCompatibleAPI) + api.model_name = 'test-model' + api.resolve_tools = lambda tools, tool_choice, config: (tools, tool_choice, config) + api.completion_params = lambda config: {'model': 'test-model', 'stream': True} + api.explicit_cache_control_params = lambda config: None + api.validate_request_params = lambda request: None + api.on_response = lambda response: None + api.chat_choices_from_message = lambda message, tools: [] + + monkeypatch.setattr('evalscope.models.anthropic_compatible.anthropic_chat_messages', lambda *a, **kw: (None, [])) + + def model_output(message, choices): + return SimpleNamespace( + content=message.content[0].text, + usage=None, + message=SimpleNamespace(), + time=None, + ) + + monkeypatch.setattr('evalscope.models.anthropic_compatible.model_output_from_anthropic', model_output) + return api + + +def test_generate_retries_when_stream_consumption_fails(monkeypatch: pytest.MonkeyPatch) -> None: + api = _prepare_api(monkeypatch) + attempts = 0 + + def create(**request): + nonlocal attempts + attempts += 1 + attempt = attempts + + def stream() -> Iterator: + if attempt == 1: + yield _events('discarded partial response')[2] + raise ConnectionError('stream interrupted by upstream gateway') + yield from _events('complete response') + + return stream() + + api.client = SimpleNamespace(messages=SimpleNamespace(create=create)) + + result = api.generate([], [], None, GenerateConfig(retries=2, retry_interval=0, stream=True)) + + assert attempts == 2 + assert result.content == 'complete response' + + +def test_generate_async_retries_when_stream_consumption_fails(monkeypatch: pytest.MonkeyPatch) -> None: + api = _prepare_api(monkeypatch) + attempts = 0 + + async def create(**request): + nonlocal attempts + attempts += 1 + attempt = attempts + + async def stream(): + if attempt == 1: + yield _events('discarded partial response')[2] + raise ConnectionError('stream interrupted by upstream gateway') + for event in _events('complete response'): + yield event + + return stream() + + async_client = SimpleNamespace(messages=SimpleNamespace(create=create)) + monkeypatch.setattr(AnthropicCompatibleAPI, 'async_client', property(lambda self: async_client)) + + result: Optional[SimpleNamespace] = asyncio.run( + api.generate_async([], [], None, GenerateConfig(retries=2, retry_interval=0, stream=True)) + ) + + assert attempts == 2 + assert result.content == 'complete response' diff --git a/tests/models/test_no_retry_exceptions.py b/tests/models/test_no_retry_exceptions.py new file mode 100644 index 000000000..f244b24d5 --- /dev/null +++ b/tests/models/test_no_retry_exceptions.py @@ -0,0 +1,308 @@ +"""Non-retryable 4xx client errors must escape retry_call immediately. + +Each model backend degrades client errors (context length exceeded, invalid +parameters, bad credentials) in its own exception handler instead of failing the +run, so retrying them inside retry_call only burns ``retries * retry_interval`` +per failing sample. These tests exercise the backend call sites with mock SDK +clients; no network access is performed. +""" + +import asyncio +from types import SimpleNamespace +from typing import Type + +import anthropic +import httpx +import openai +import pytest + +from evalscope.api.model import GenerateConfig +from evalscope.models.anthropic_compatible import AnthropicCompatibleAPI +from evalscope.models.openai_compatible import OpenAICompatibleAPI +from evalscope.models.openai_responses import OpenAIResponsesAPI + +OPENAI_CLIENT_ERROR_CASES = ( + pytest.param(openai.BadRequestError, 400, id='bad-request'), + pytest.param(openai.AuthenticationError, 401, id='authentication'), + pytest.param(openai.PermissionDeniedError, 403, id='permission-denied'), + pytest.param(openai.NotFoundError, 404, id='not-found'), + pytest.param(openai.UnprocessableEntityError, 422, id='unprocessable-entity'), +) + +ANTHROPIC_CLIENT_ERROR_CASES = ( + pytest.param(anthropic.BadRequestError, 400, id='bad-request'), + pytest.param(anthropic.AuthenticationError, 401, id='authentication'), + pytest.param(anthropic.PermissionDeniedError, 403, id='permission-denied'), + pytest.param(anthropic.NotFoundError, 404, id='not-found'), + pytest.param(anthropic.UnprocessableEntityError, 422, id='unprocessable-entity'), +) + + +def _openai_client_error( + error_type: Type[openai.APIStatusError], + status_code: int, + message: str = 'non-retryable client error', +) -> openai.APIStatusError: + response = httpx.Response( + status_code, + request=httpx.Request('POST', 'https://example.test/v1/chat/completions'), + json={'error': {'message': message}}, + ) + return error_type( + f'Error code: {status_code} - {message}', + response=response, + body=response.json(), + ) + + +def _anthropic_client_error( + error_type: Type[anthropic.APIStatusError], + status_code: int, + message: str = 'non-retryable client error', +) -> anthropic.APIStatusError: + response = httpx.Response( + status_code, + request=httpx.Request('POST', 'https://example.test/v1/messages'), + json={'type': 'error', 'error': {'type': 'invalid_request_error', 'message': message}}, + ) + return error_type(f'Error code: {status_code} - {message}', response=response, body=response.json()) + + +def _prepare_openai_api(monkeypatch: pytest.MonkeyPatch) -> OpenAICompatibleAPI: + api = object.__new__(OpenAICompatibleAPI) + api.base_url = 'https://example.test/v1' + api.model_name = 'test-model' + api.resolve_tools = lambda tools, tool_choice, config: (tools, tool_choice, config) + api.completion_params = lambda config, tools: {'model': 'test-model'} + api.validate_request_params = lambda request: None + + monkeypatch.setattr('evalscope.models.openai_compatible.openai_chat_messages', lambda *args, **kwargs: []) + return api + + +def _prepare_anthropic_api(monkeypatch: pytest.MonkeyPatch) -> AnthropicCompatibleAPI: + api = object.__new__(AnthropicCompatibleAPI) + api.model_name = 'test-model' + api.resolve_tools = lambda tools, tool_choice, config: (tools, tool_choice, config) + api.completion_params = lambda config: {'model': 'test-model'} + api.explicit_cache_control_params = lambda config: None + api.validate_request_params = lambda request: None + + monkeypatch.setattr( + 'evalscope.models.anthropic_compatible.anthropic_chat_messages', lambda *args, **kwargs: (None, []) + ) + return api + + +def _prepare_openai_responses_api() -> OpenAIResponsesAPI: + api = object.__new__(OpenAIResponsesAPI) + api.model_name = 'test-model' + api._build_request = lambda input, tools, tool_choice, config: ({}, tools, config) + return api + + +@pytest.mark.parametrize(('error_type', 'status_code'), OPENAI_CLIENT_ERROR_CASES) +def test_openai_generate_does_not_retry_client_errors(monkeypatch, error_type, status_code) -> None: + api = _prepare_openai_api(monkeypatch) + attempts = 0 + + def create(**request): + nonlocal attempts + attempts += 1 + raise _openai_client_error(error_type, status_code) + + api.client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + + with pytest.raises(error_type): + api.generate([], [], None, GenerateConfig(retries=5, retry_interval=0)) + + assert attempts == 1 + + +@pytest.mark.parametrize(('error_type', 'status_code'), OPENAI_CLIENT_ERROR_CASES) +def test_openai_generate_async_does_not_retry_client_errors(monkeypatch, error_type, status_code) -> None: + api = _prepare_openai_api(monkeypatch) + attempts = 0 + + async def create(**request): + nonlocal attempts + attempts += 1 + raise _openai_client_error(error_type, status_code) + + async_client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + monkeypatch.setattr(OpenAICompatibleAPI, 'async_client', property(lambda self: async_client)) + + with pytest.raises(error_type): + asyncio.run(api.generate_async([], [], None, GenerateConfig(retries=5, retry_interval=0))) + + assert attempts == 1 + + +def test_openai_bad_request_still_degrades_gracefully(monkeypatch) -> None: + api = _prepare_openai_api(monkeypatch) + attempts = 0 + + def create(**request): + nonlocal attempts + attempts += 1 + raise _openai_client_error(openai.BadRequestError, 400, 'maximum context length is 4096 tokens') + + api.client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + + result = api.generate([], [], None, GenerateConfig(retries=5, retry_interval=0)) + + assert attempts == 1 + assert '400' in result.choices[0].message.content + + +def test_openai_generate_still_retries_connection_errors(monkeypatch) -> None: + api = _prepare_openai_api(monkeypatch) + attempts = 0 + + def create(**request): + nonlocal attempts + attempts += 1 + raise ConnectionError('stream interrupted by upstream gateway') + + api.client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))) + + with pytest.raises(ConnectionError): + api.generate([], [], None, GenerateConfig(retries=3, retry_interval=0)) + + assert attempts == 3 + + +@pytest.mark.parametrize(('error_type', 'status_code'), ANTHROPIC_CLIENT_ERROR_CASES) +def test_anthropic_generate_does_not_retry_client_errors(monkeypatch, error_type, status_code) -> None: + api = _prepare_anthropic_api(monkeypatch) + attempts = 0 + + def create(**request): + nonlocal attempts + attempts += 1 + raise _anthropic_client_error(error_type, status_code) + + api.client = SimpleNamespace(messages=SimpleNamespace(create=create)) + + with pytest.raises(error_type): + api.generate([], [], None, GenerateConfig(retries=5, retry_interval=0)) + + assert attempts == 1 + + +@pytest.mark.parametrize(('error_type', 'status_code'), ANTHROPIC_CLIENT_ERROR_CASES) +def test_anthropic_generate_async_does_not_retry_client_errors(monkeypatch, error_type, status_code) -> None: + api = _prepare_anthropic_api(monkeypatch) + attempts = 0 + + async def create(**request): + nonlocal attempts + attempts += 1 + raise _anthropic_client_error(error_type, status_code) + + async_client = SimpleNamespace(messages=SimpleNamespace(create=create)) + monkeypatch.setattr(AnthropicCompatibleAPI, 'async_client', property(lambda self: async_client)) + + with pytest.raises(error_type): + asyncio.run(api.generate_async([], [], None, GenerateConfig(retries=5, retry_interval=0))) + + assert attempts == 1 + + +def test_anthropic_bad_request_still_degrades_gracefully(monkeypatch) -> None: + api = _prepare_anthropic_api(monkeypatch) + attempts = 0 + + def create(**request): + nonlocal attempts + attempts += 1 + raise _anthropic_client_error(anthropic.BadRequestError, 400, 'prompt is too long') + + api.client = SimpleNamespace(messages=SimpleNamespace(create=create)) + + result = api.generate([], [], None, GenerateConfig(retries=5, retry_interval=0)) + + assert attempts == 1 + assert 'prompt is too long' in result.choices[0].message.content + + +def test_openai_responses_generate_does_not_retry_bad_request() -> None: + api = _prepare_openai_responses_api() + attempts = 0 + + def create(**request): + nonlocal attempts + attempts += 1 + raise _openai_client_error(openai.BadRequestError, 400) + + api.client = SimpleNamespace(responses=SimpleNamespace(create=create)) + + with pytest.raises(openai.BadRequestError): + api.generate([], [], None, GenerateConfig(retries=5, retry_interval=0)) + + assert attempts == 1 + + +def test_openai_responses_generate_async_does_not_retry_bad_request(monkeypatch) -> None: + api = _prepare_openai_responses_api() + attempts = 0 + + async def create(**request): + nonlocal attempts + attempts += 1 + raise _openai_client_error(openai.BadRequestError, 400) + + async_client = SimpleNamespace(responses=SimpleNamespace(create=create)) + monkeypatch.setattr(OpenAIResponsesAPI, 'async_client', property(lambda self: async_client)) + + with pytest.raises(openai.BadRequestError): + asyncio.run(api.generate_async([], [], None, GenerateConfig(retries=5, retry_interval=0))) + + assert attempts == 1 + + +def test_litellm_generate_does_not_retry_bad_request(monkeypatch) -> None: + litellm = pytest.importorskip('litellm') + from evalscope.models.litellm_compatible import LiteLLMAPI + + api = object.__new__(LiteLLMAPI) + api.model_name = 'test-model' + api.api_key = None + api.base_url = None + attempts = 0 + + def completion(**request): + nonlocal attempts + attempts += 1 + raise litellm.exceptions.BadRequestError('maximum context length is 4096 tokens', 'test-model', 'openai') + + monkeypatch.setattr(litellm, 'completion', completion) + + # LiteLLM's outer handler logs and re-raises; the point is exactly one attempt. + with pytest.raises(litellm.exceptions.BadRequestError): + api.generate([], [], None, GenerateConfig(retries=5, retry_interval=0)) + + assert attempts == 1 + + +def test_litellm_generate_async_does_not_retry_bad_request(monkeypatch) -> None: + litellm = pytest.importorskip('litellm') + from evalscope.models.litellm_compatible import LiteLLMAPI + + api = object.__new__(LiteLLMAPI) + api.model_name = 'test-model' + api.api_key = None + api.base_url = None + attempts = 0 + + async def completion(**request): + nonlocal attempts + attempts += 1 + raise litellm.exceptions.BadRequestError('maximum context length is 4096 tokens', 'test-model', 'openai') + + monkeypatch.setattr(litellm, 'acompletion', completion) + + with pytest.raises(litellm.exceptions.BadRequestError): + asyncio.run(api.generate_async([], [], None, GenerateConfig(retries=5, retry_interval=0))) + + assert attempts == 1 diff --git a/tests/test_function_utils.py b/tests/test_function_utils.py index 9d9721458..4c9c7591f 100644 --- a/tests/test_function_utils.py +++ b/tests/test_function_utils.py @@ -7,6 +7,152 @@ from evalscope.utils import asyncio_runtime from evalscope.utils.asyncio_runtime import AsyncioLoopRunner, AsyncioLoopThread, cancel_and_wait +from evalscope.utils.function_utils import async_retry_call, retry_call + + +class _CountingError(Exception): + """Test exception whose raises are counted via the shared `calls` list.""" + + def __init__(self, calls: list) -> None: + super().__init__('boom') + calls.append(1) + + +def test_retry_call_clamps_zero_retries_to_one_attempt() -> None: + calls: list = [] + + def _ok() -> str: + calls.append(1) + return 'ok' + + assert retry_call(_ok, retries=0) == 'ok' + assert len(calls) == 1 + + +def test_retry_call_clamps_negative_retries_to_one_attempt() -> None: + calls: list = [] + + def _fail() -> str: + raise _CountingError(calls) + + with pytest.raises(_CountingError): + retry_call(_fail, retries=-2, sleep_interval=0) + assert len(calls) == 1 + + +def test_retry_call_treats_none_retries_as_default_attempts() -> None: + calls: list = [] + + def _fail() -> str: + raise _CountingError(calls) + + with pytest.raises(_CountingError): + retry_call(_fail, retries=None, sleep_interval=0) + assert len(calls) == 3 + + +def test_retry_call_returns_value_on_success() -> None: + calls: list = [] + + def _ok() -> str: + calls.append(1) + return 'ok' + + assert retry_call(_ok, retries=3) == 'ok' + assert len(calls) == 1 + + +def test_retry_call_exhausts_attempts_and_reraises_last_error() -> None: + calls: list = [] + + def _fail() -> str: + raise _CountingError(calls) + + with pytest.raises(_CountingError): + retry_call(_fail, retries=3, sleep_interval=0) + assert len(calls) == 3 + + +def test_async_retry_call_clamps_zero_retries_to_one_attempt() -> None: + calls: list = [] + + async def _ok() -> str: + calls.append(1) + return 'ok' + + assert asyncio.run(async_retry_call(_ok, retries=0)) == 'ok' + assert len(calls) == 1 + + +def test_async_retry_call_treats_none_retries_as_default_attempts() -> None: + calls: list = [] + + async def _fail() -> str: + raise _CountingError(calls) + + with pytest.raises(_CountingError): + asyncio.run(async_retry_call(_fail, retries=None, sleep_interval=0)) + assert len(calls) == 3 + + +def test_async_retry_call_exhausts_attempts_and_reraises_last_error() -> None: + calls: list = [] + + async def _fail() -> str: + raise _CountingError(calls) + + with pytest.raises(_CountingError): + asyncio.run(async_retry_call(_fail, retries=2, sleep_interval=0)) + assert len(calls) == 2 + + +def test_retry_call_reraises_no_retry_exceptions_immediately() -> None: + calls: list = [] + + def _fail() -> str: + raise _CountingError(calls) + + with pytest.raises(_CountingError): + retry_call(_fail, retries=5, sleep_interval=0, no_retry_exceptions=(_CountingError, )) + assert len(calls) == 1 + + +def test_retry_call_no_retry_exceptions_match_subclasses() -> None: + + class _ChildError(_CountingError): + pass + + calls: list = [] + + def _fail() -> str: + raise _ChildError(calls) + + with pytest.raises(_ChildError): + retry_call(_fail, retries=5, sleep_interval=0, no_retry_exceptions=(_CountingError, )) + assert len(calls) == 1 + + +def test_retry_call_still_retries_unrelated_exceptions_with_no_retry_set() -> None: + calls: list = [] + + def _fail() -> str: + calls.append(1) + raise ValueError('transient') + + with pytest.raises(ValueError): + retry_call(_fail, retries=3, sleep_interval=0, no_retry_exceptions=(_CountingError, )) + assert len(calls) == 3 + + +def test_async_retry_call_reraises_no_retry_exceptions_immediately() -> None: + calls: list = [] + + async def _fail() -> str: + raise _CountingError(calls) + + with pytest.raises(_CountingError): + asyncio.run(async_retry_call(_fail, retries=5, sleep_interval=0, no_retry_exceptions=(_CountingError, ))) + assert len(calls) == 1 async def _current_loop() -> asyncio.AbstractEventLoop: