Fix TEI stale socket retries

hindsight-api-slim/hindsight_api/engine/cross_encoder.py

@@ -41,7 +41,7 @@
     resolve_model_device_type,
     select_local_device,
 )
-from .tei_retry import tei_retry_delay
+from .tei_retry import TEI_KEEPALIVE_EXPIRY_SECONDS, is_retryable_tei_transport_error, tei_retry_delay
 
 logger = logging.getLogger(__name__)
 
@@ -382,7 +382,29 @@ async def _async_request_with_retry(
                         response = await client.post(url, **kwargs)
                     response.raise_for_status()
                     return response
-                except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout) as e:
+                except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout, httpx.WriteTimeout) as e:
+                    last_error = e
+                    if attempt < self.max_retries:
+                        logger.warning(
+                            f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2  # Exponential backoff
+                except httpx.RequestError as e:
+                    if not is_retryable_tei_transport_error(e):
+                        raise
+                    last_error = e
+                    if attempt < self.max_retries:
+                        logger.warning(
+                            f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                            f"Retrying in {delay}s..."
+                        )
+                        await asyncio.sleep(delay)
+                        delay *= 2  # Exponential backoff
+                except OSError as e:
+                    if not is_retryable_tei_transport_error(e):
+                        raise
                     last_error = e
                     if attempt < self.max_retries:
                         logger.warning(
@@ -421,7 +443,10 @@ async def initialize(self) -> None:
             f"Reranker: initializing TEI provider at {self.base_url} "
             f"(batch_size={self.batch_size}, max_concurrent={self.max_concurrent})"
         )
-        self._async_client = httpx.AsyncClient(timeout=self.timeout)
+        self._async_client = httpx.AsyncClient(
+            timeout=self.timeout,
+            limits=httpx.Limits(keepalive_expiry=min(self.timeout, TEI_KEEPALIVE_EXPIRY_SECONDS)),
+        )
 
         # Verify server is reachable and get model info
         # Use a temporary semaphore for initialization

hindsight-api-slim/hindsight_api/engine/embeddings.py

@@ -53,7 +53,7 @@
     resolve_model_device_type,
     select_local_device,
 )
-from .tei_retry import tei_retry_delay
+from .tei_retry import TEI_KEEPALIVE_EXPIRY_SECONDS, is_retryable_tei_transport_error, tei_retry_delay
 
 logger = logging.getLogger(__name__)
 
@@ -522,6 +522,26 @@ def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response
                     )
                     time.sleep(delay)
                     delay *= 2  # Exponential backoff
+            except httpx.RequestError as e:
+                if not is_retryable_tei_transport_error(e):
+                    raise
+                last_error = e
+                if attempt < self.max_retries:
+                    logger.warning(
+                        f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
+                    )
+                    time.sleep(delay)
+                    delay *= 2  # Exponential backoff
+            except OSError as e:
+                if not is_retryable_tei_transport_error(e):
+                    raise
+                last_error = e
+                if attempt < self.max_retries:
+                    logger.warning(
+                        f"TEI request failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. Retrying in {delay}s..."
+                    )
+                    time.sleep(delay)
+                    delay *= 2  # Exponential backoff
             except httpx.HTTPStatusError as e:
                 # TEI uses 429 as normal overload backpressure. Retry it with
                 # the same bounded budget as transient server errors.
@@ -549,7 +569,10 @@ async def initialize(self) -> None:
             return
 
         logger.info(f"Embeddings: initializing TEI provider at {self.base_url}")
-        self._client = httpx.Client(timeout=self.timeout)
+        self._client = httpx.Client(
+            timeout=self.timeout,
+            limits=httpx.Limits(keepalive_expiry=min(self.timeout, TEI_KEEPALIVE_EXPIRY_SECONDS)),
+        )
 
         # Verify server is reachable and get model info
         try:

hindsight-api-slim/hindsight_api/engine/tei_retry.py

@@ -1,5 +1,6 @@
 """Shared retry timing for Text Embeddings Inference HTTP clients."""
 
+import errno
 import math
 import random
 from datetime import datetime, timezone
@@ -13,6 +14,7 @@
 # capping well below the request timeout keeps a slow proxy from converting a
 # transient overload into a minutes-long recall stall.
 MAX_RETRY_DELAY_SECONDS = 5.0
+TEI_KEEPALIVE_EXPIRY_SECONDS = 15.0
 
 # Fraction of the delay used as the jitter window, matching the "equal jitter"
 # policy of ``db_utils._backoff_delay``. TEI overload is self-synchronising -- a
@@ -22,6 +24,23 @@
 # lockstep and re-colliding on the same exhausted pool.
 JITTER_RATIO = 0.5
 
+# TEI requests are usually against a pooled HTTP client. If the underlying
+# socket dies while idle, the first failure often shows up as a bare OSError
+# instead of a structured httpx timeout/connect error.
+RETRYABLE_OS_ERRNOS = {
+    errno.EBADF,
+    errno.ECONNABORTED,
+    errno.ECONNREFUSED,
+    errno.ECONNRESET,
+    errno.EHOSTDOWN,
+    errno.EHOSTUNREACH,
+    errno.ENETDOWN,
+    errno.ENETRESET,
+    errno.ENETUNREACH,
+    errno.EPIPE,
+    errno.ETIMEDOUT,
+}
+
 
 def _retry_after_seconds(value: str | None) -> float | None:
     if not isinstance(value, str) or not value:
@@ -71,3 +90,21 @@ def tei_retry_delay(
         return delay - random.uniform(0.0, spread)
     # Retry-After is a minimum, so spread upward -- but never past the ceiling.
     return delay + random.uniform(0.0, min(spread, limit - delay))
+
+
+def is_retryable_tei_transport_error(exc: BaseException) -> bool:
+    """Return True for transport failures that should be retried once.
+
+    This keeps stale pooled sockets from bubbling up as hard failures when the
+    underlying descriptor died while idle.
+    """
+
+    if isinstance(exc, OSError):
+        return exc.errno in RETRYABLE_OS_ERRNOS or exc.errno is None
+
+    for attr in ("__cause__", "__context__"):
+        nested = getattr(exc, attr, None)
+        if nested is not None and nested is not exc and is_retryable_tei_transport_error(nested):
+            return True
+
+    return False

hindsight-api-slim/tests/test_tei_embeddings.py

@@ -1,5 +1,7 @@
 """Regression tests for transient HTTP handling in the remote TEI embeddings client."""
 
+import errno
+
 import httpx
 import pytest
 
@@ -27,6 +29,27 @@ def handler(request: httpx.Request) -> httpx.Response:
     assert attempts == 2
 
 
+def test_bad_file_descriptor_retries_then_succeeds() -> None:
+    attempts = 0
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        nonlocal attempts
+        attempts += 1
+        if attempts == 1:
+            raise OSError(errno.EBADF, "Bad file descriptor")
+        return httpx.Response(200, json=[[0.3, 0.4]])
+
+    embeddings = RemoteTEIEmbeddings(
+        base_url="http://localhost:8080",
+        max_retries=3,
+        retry_delay=0,
+    )
+    embeddings._client = httpx.Client(transport=httpx.MockTransport(handler))
+
+    assert embeddings.encode(["text"]) == [[0.3, 0.4]]
+    assert attempts == 2
+
+
 def test_persistent_connect_timeout_exhausts_retry_budget() -> None:
     attempts = 0