diff --git a/hindsight-api-slim/hindsight_api/engine/cross_encoder.py b/hindsight-api-slim/hindsight_api/engine/cross_encoder.py index 735da895b..3c0eb0f7c 100644 --- a/hindsight-api-slim/hindsight_api/engine/cross_encoder.py +++ b/hindsight-api-slim/hindsight_api/engine/cross_encoder.py @@ -41,7 +41,7 @@ resolve_model_device_type, select_local_device, ) -from .tei_retry import tei_retry_delay +from .tei_retry import TEI_KEEPALIVE_EXPIRY_SECONDS, is_retryable_tei_transport_error, tei_retry_delay logger = logging.getLogger(__name__) @@ -382,7 +382,29 @@ async def _async_request_with_retry( response = await client.post(url, **kwargs) response.raise_for_status() return response - except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e: + except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.WriteTimeout) as e: + last_error = e + if attempt < self.max_retries: + logger.warning( + f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. " + f"Retrying in {delay}s..." + ) + await asyncio.sleep(delay) + delay *= 2 # Exponential backoff + except httpx.RequestError as e: + if not is_retryable_tei_transport_error(e): + raise + last_error = e + if attempt < self.max_retries: + logger.warning( + f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. " + f"Retrying in {delay}s..." + ) + await asyncio.sleep(delay) + delay *= 2 # Exponential backoff + except OSError as e: + if not is_retryable_tei_transport_error(e): + raise last_error = e if attempt < self.max_retries: logger.warning( @@ -421,7 +443,10 @@ async def initialize(self) -> None: f"Reranker: initializing TEI provider at {self.base_url} " f"(batch_size={self.batch_size}, max_concurrent={self.max_concurrent})" ) - self._async_client = httpx.AsyncClient(timeout=self.timeout) + self._async_client = httpx.AsyncClient( + timeout=self.timeout, + limits=httpx.Limits(keepalive_expiry=min(self.timeout, TEI_KEEPALIVE_EXPIRY_SECONDS)), + ) # Verify server is reachable and get model info # Use a temporary semaphore for initialization diff --git a/hindsight-api-slim/hindsight_api/engine/embeddings.py b/hindsight-api-slim/hindsight_api/engine/embeddings.py index ddc8a38f6..4c669d042 100644 --- a/hindsight-api-slim/hindsight_api/engine/embeddings.py +++ b/hindsight-api-slim/hindsight_api/engine/embeddings.py @@ -53,7 +53,7 @@ resolve_model_device_type, select_local_device, ) -from .tei_retry import tei_retry_delay +from .tei_retry import TEI_KEEPALIVE_EXPIRY_SECONDS, is_retryable_tei_transport_error, tei_retry_delay logger = logging.getLogger(__name__) @@ -522,6 +522,26 @@ def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response ) time.sleep(delay) delay *= 2 # Exponential backoff + except httpx.RequestError as e: + if not is_retryable_tei_transport_error(e): + raise + last_error = e + if attempt < self.max_retries: + logger.warning( + f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..." + ) + time.sleep(delay) + delay *= 2 # Exponential backoff + except OSError as e: + if not is_retryable_tei_transport_error(e): + raise + last_error = e + if attempt < self.max_retries: + logger.warning( + f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..." + ) + time.sleep(delay) + delay *= 2 # Exponential backoff except httpx.HTTPStatusError as e: # TEI uses 429 as normal overload backpressure. Retry it with # the same bounded budget as transient server errors. @@ -549,7 +569,10 @@ async def initialize(self) -> None: return logger.info(f"Embeddings: initializing TEI provider at {self.base_url}") - self._client = httpx.Client(timeout=self.timeout) + self._client = httpx.Client( + timeout=self.timeout, + limits=httpx.Limits(keepalive_expiry=min(self.timeout, TEI_KEEPALIVE_EXPIRY_SECONDS)), + ) # Verify server is reachable and get model info try: diff --git a/hindsight-api-slim/hindsight_api/engine/tei_retry.py b/hindsight-api-slim/hindsight_api/engine/tei_retry.py index 4c6b25a64..cf119b6e5 100644 --- a/hindsight-api-slim/hindsight_api/engine/tei_retry.py +++ b/hindsight-api-slim/hindsight_api/engine/tei_retry.py @@ -1,5 +1,6 @@ """Shared retry timing for Text Embeddings Inference HTTP clients.""" +import errno import math import random from datetime import datetime, timezone @@ -13,6 +14,7 @@ # capping well below the request timeout keeps a slow proxy from converting a # transient overload into a minutes-long recall stall. MAX_RETRY_DELAY_SECONDS = 5.0 +TEI_KEEPALIVE_EXPIRY_SECONDS = 15.0 # Fraction of the delay used as the jitter window, matching the "equal jitter" # policy of ``db_utils._backoff_delay``. TEI overload is self-synchronising -- a @@ -22,6 +24,23 @@ # lockstep and re-colliding on the same exhausted pool. JITTER_RATIO = 0.5 +# TEI requests are usually against a pooled HTTP client. If the underlying +# socket dies while idle, the first failure often shows up as a bare OSError +# instead of a structured httpx timeout/connect error. +RETRYABLE_OS_ERRNOS = { + errno.EBADF, + errno.ECONNABORTED, + errno.ECONNREFUSED, + errno.ECONNRESET, + errno.EHOSTDOWN, + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.ENETRESET, + errno.ENETUNREACH, + errno.EPIPE, + errno.ETIMEDOUT, +} + def _retry_after_seconds(value: str | None) -> float | None: if not isinstance(value, str) or not value: @@ -71,3 +90,21 @@ def tei_retry_delay( return delay - random.uniform(0.0, spread) # Retry-After is a minimum, so spread upward -- but never past the ceiling. return delay + random.uniform(0.0, min(spread, limit - delay)) + + +def is_retryable_tei_transport_error(exc: BaseException) -> bool: + """Return True for transport failures that should be retried once. + + This keeps stale pooled sockets from bubbling up as hard failures when the + underlying descriptor died while idle. + """ + + if isinstance(exc, OSError): + return exc.errno in RETRYABLE_OS_ERRNOS or exc.errno is None + + for attr in ("__cause__", "__context__"): + nested = getattr(exc, attr, None) + if nested is not None and nested is not exc and is_retryable_tei_transport_error(nested): + return True + + return False diff --git a/hindsight-api-slim/tests/test_tei_embeddings.py b/hindsight-api-slim/tests/test_tei_embeddings.py index 68b937c79..78ef9b801 100644 --- a/hindsight-api-slim/tests/test_tei_embeddings.py +++ b/hindsight-api-slim/tests/test_tei_embeddings.py @@ -1,5 +1,7 @@ """Regression tests for transient HTTP handling in the remote TEI embeddings client.""" +import errno + import httpx import pytest @@ -27,6 +29,27 @@ def handler(request: httpx.Request) -> httpx.Response: assert attempts == 2 +def test_bad_file_descriptor_retries_then_succeeds() -> None: + attempts = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError(errno.EBADF, "Bad file descriptor") + return httpx.Response(200, json=[[0.3, 0.4]]) + + embeddings = RemoteTEIEmbeddings( + base_url="http://localhost:8080", + max_retries=3, + retry_delay=0, + ) + embeddings._client = httpx.Client(transport=httpx.MockTransport(handler)) + + assert embeddings.encode(["text"]) == [[0.3, 0.4]] + assert attempts == 2 + + def test_persistent_connect_timeout_exhausts_retry_budget() -> None: attempts = 0