feat: support keyless search providers

ms_agent/tools/base.py

@@ -1,10 +1,20 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
+import math
 from abc import abstractmethod
 from omegaconf import DictConfig
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from ms_agent.utils.constants import DEFAULT_OUTPUT_DIR
 
+#: Return this from :attr:`ToolBase.max_output_chars` to declare that a tool
+#: bounds its own output and must never be cut by the generic truncator.
+SELF_MANAGED_OUTPUT = math.inf
+
+#: Where to keep text from when an oversized output IS cut generically.
+TRUNCATE_KEEP_HEAD = 'head'
+TRUNCATE_KEEP_TAIL = 'tail'
+TRUNCATE_KEEP_BOTH = 'both'
+
 
 class ToolBase:
     """The base class for all tools.
@@ -19,6 +29,30 @@ def __init__(self, config):
         self.output_dir = getattr(self.config, 'output_dir',
                                   DEFAULT_OUTPUT_DIR)
 
+    @property
+    def max_output_chars(self) -> Optional[float]:
+        """Model-facing character budget for this tool's output.
+
+        * ``None`` (default) — use the global ``MAX_TOOL_OUTPUT_LEN``.
+        * :data:`SELF_MANAGED_OUTPUT` — the tool guarantees its own bound
+          (paging, spilling to disk, …); never truncate it generically.
+        * a number — this tool's own budget, used instead of the global one.
+
+        Override in a subclass to opt in. Declaring a budget is a promise about
+        SHAPE as much as size: a tool that returns structured data should keep
+        itself under budget so the generic cut never has to run.
+        """
+        return None
+
+    @property
+    def truncate_keep(self) -> str:
+        """Which end survives when this tool's output IS cut generically.
+
+        ``'head'`` (a command's first output is the useful part), ``'tail'``
+        (a long run whose verdict is last), or ``'both'`` (default).
+        """
+        return TRUNCATE_KEEP_BOTH
+
     def exclude_func(self, tool_config: DictConfig):
         if tool_config is not None:
             self.exclude_functions = getattr(tool_config, 'exclude', [])

ms_agent/tools/search/tavily/fetcher.py

@@ -32,11 +32,9 @@ def __init__(
         include_favicon: bool = False,
         include_usage: bool = False,
     ):
-        key = api_key or os.getenv('TAVILY_API_KEY')
-        if not key:
-            raise ValueError(
-                'TAVILY_API_KEY required for tavily_extract fetcher')
-        self._api_key = key
+        # Keyless is a supported mode here too (Tavily serves /extract without
+        # credentials under the same header as /search); see tavily/search.py.
+        self._api_key = api_key or os.getenv('TAVILY_API_KEY') or ''
         self._extract_depth = extract_depth
         self._format = format
         self._timeout = max(1.0, min(60.0, float(timeout)))
@@ -52,7 +50,6 @@ def fetch(self,
         Extract one URL. Optional ``query`` enables chunk reranking (more relevant raw_content).
         """
         body: Dict[str, Any] = {
-            'api_key': self._api_key,
             'urls': [url],
             'extract_depth': self._extract_depth,
             'format': self._format,
@@ -64,10 +61,18 @@ def fetch(self,
         if query:
             body['query'] = query
             body['chunks_per_source'] = self._chunks_per_source
+        # Omitted when empty: any api_key in the body overrides the keyless
+        # header (see TavilySearchRequest.to_api_body).
+        if self._api_key:
+            body['api_key'] = self._api_key
 
         try:
+            from ms_agent.tools.search.tavily.search import KEYLESS_HEADER
             data = post_json(
-                TAVILY_EXTRACT_URL, body, timeout=self._timeout + 30.0)
+                TAVILY_EXTRACT_URL,
+                body,
+                timeout=self._timeout + 30.0,
+                headers=(dict(KEYLESS_HEADER) if not self._api_key else {}))
         except Exception as e:
             logger.warning(f'Tavily extract failed for {url[:80]}: {e}')
             return '', {

ms_agent/tools/search/tavily/http.py

@@ -1,35 +1,124 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
 """Minimal HTTP JSON client for Tavily REST API (stdlib only)."""
 import json
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
 
 
+class TavilyHTTPError(RuntimeError):
+    """A Tavily call that failed, with the pieces a caller can act on.
+
+    Plain ``RuntimeError`` forced every caller to re-parse the message to tell
+    "you are out of quota, ask the user for a key" from "that host is down".
+    Keyless mode makes that distinction routine rather than exceptional — the
+    free tier is a small hourly bucket — so the parts travel as fields:
+    ``status`` (HTTP code, None for transport failures), ``code`` (Tavily's own
+    machine-readable ``error.code``, e.g. ``hourly_cap_reached``) and
+    ``retry_after`` seconds when the response carried one.
+    """
+
+    def __init__(self,
+                 message: str,
+                 *,
+                 status: Optional[int] = None,
+                 code: str = '',
+                 retry_after: Optional[int] = None,
+                 detail: Any = None):
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.retry_after = retry_after
+        self.detail = detail
+
+    @property
+    def is_quota(self) -> bool:
+        """Out of quota — retryable later, and fixable now with an API key."""
+        return self.status == 429 or self.code in ('hourly_cap_reached',
+                                                   'rate_limit_exceeded')
+
+    @property
+    def is_auth(self) -> bool:
+        return self.status in (401, 403)
+
+
+def _ssl_context():
+    """A verifying TLS context that works on interpreters with no CA store.
+
+    ``urlopen`` uses the interpreter's default store, which is empty in some
+    virtualenvs (``ssl.get_default_verify_paths().cafile is None`` — measured on
+    the WebUI backend's venv, where every Tavily call died with
+    CERTIFICATE_VERIFY_FAILED). certifi is already an indirect dependency there;
+    when it is missing we hand back None so urlopen behaves exactly as before.
+    Never disables verification.
+    """
+    try:
+        import certifi
+        import ssl
+        return ssl.create_default_context(cafile=certifi.where())
+    except Exception:
+        return None
+
+
+def _parse_error_body(raw: str) -> Any:
+    try:
+        return json.loads(raw) if raw else {}
+    except json.JSONDecodeError:
+        return {'raw': raw}
+
+
+def _dig_error(detail: Any) -> tuple:
+    """``(code, message, retry_after)`` out of Tavily's error envelope.
+
+    Two shapes are in the wild: ``{"error": {"code", "message",
+    "retry_after_seconds"}}`` (keyless quota) and ``{"detail": {"error": ...}}``
+    (auth). Anything else degrades to empty strings rather than raising while
+    already handling an error.
+    """
+    code = message = ''
+    retry_after = None
+    node = detail
+    if isinstance(node, dict) and isinstance(node.get('detail'), dict):
+        node = node['detail']
+    if isinstance(node, dict):
+        err = node.get('error')
+        if isinstance(err, dict):
+            code = str(err.get('code') or '')
+            message = str(err.get('message') or '')
+            ra = err.get('retry_after_seconds')
+            if isinstance(ra, (int, float)):
+                retry_after = int(ra)
+        elif isinstance(err, str):
+            message = err
+    return code, message, retry_after
+
+
 def post_json(
     url: str,
     body: Dict[str, Any],
     *,
     timeout: float = 120.0,
+    headers: Optional[Dict[str, str]] = None,
 ) -> Dict[str, Any]:
     """
     POST JSON and parse JSON response.
 
+    ``headers`` is merged over the defaults — that is how keyless mode is
+    selected (``X-Tavily-Access-Mode: keyless``).
+
     Raises:
-        RuntimeError: on HTTP errors or invalid JSON (includes Tavily error body).
+        TavilyHTTPError: on HTTP errors or invalid JSON (carries Tavily's own
+            error code / retry-after so callers can tell quota from outage).
     """
     data = json.dumps(body, ensure_ascii=False).encode('utf-8')
-    req = Request(
-        url,
-        data=data,
-        method='POST',
-        headers={
-            'Content-Type': 'application/json',
-            'Accept': 'application/json',
-        },
-    )
+    merged = {
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+    }
+    merged.update(headers or {})
+    req = Request(url, data=data, method='POST', headers=merged)
     try:
-        with urlopen(req, timeout=timeout) as resp:
+        with urlopen(req, timeout=timeout, context=_ssl_context()) as resp:
             raw = resp.read().decode('utf-8', errors='replace')
             if not raw.strip():
                 return {}
@@ -40,10 +129,24 @@ def post_json(
             err_body = e.read().decode('utf-8', errors='replace')
         except Exception:
             pass
-        try:
-            detail = json.loads(err_body) if err_body else {}
-        except json.JSONDecodeError:
-            detail = {'raw': err_body}
-        raise RuntimeError(f'Tavily HTTP {e.code}: {detail}') from e
+        detail = _parse_error_body(err_body)
+        code, message, retry_after = _dig_error(detail)
+        if retry_after is None:
+            header_value = None
+            try:
+                header_value = e.headers.get('retry-after')
+            except Exception:
+                pass
+            if header_value:
+                try:
+                    retry_after = int(float(header_value))
+                except (TypeError, ValueError):
+                    retry_after = None
+        raise TavilyHTTPError(
+            f'Tavily HTTP {e.code}: {message or detail}',
+            status=e.code,
+            code=code,
+            retry_after=retry_after,
+            detail=detail) from e
     except URLError as e:
-        raise RuntimeError(f'Tavily network error: {e}') from e
+        raise TavilyHTTPError(f'Tavily network error: {e}') from e

ms_agent/tools/search/tavily/schema.py

@@ -33,7 +33,6 @@ class TavilySearchRequest:
     def to_api_body(self, api_key: str) -> Dict[str, Any]:
         n = max(0, min(20, int(self.max_results)))
         body: Dict[str, Any] = {
-            'api_key': api_key,
             'query': self.query,
             'max_results': n,
             'search_depth': self.search_depth,
@@ -64,6 +63,13 @@ def to_api_body(self, api_key: str) -> Dict[str, Any]:
             body['exclude_domains'] = list(self.exclude_domains)[:150]
         if self.country:
             body['country'] = self.country
+        # Sent LAST and only when non-empty. Keyless mode (the
+        # X-Tavily-Access-Mode header) is overridden by any api_key present in
+        # the body: measured 2026-08-20, an empty string is tolerated but a
+        # non-empty one is validated and a bogus value 401s. Omitting the field
+        # entirely is the only shape that is unambiguous in both modes.
+        if api_key:
+            body['api_key'] = api_key
         return body
 
 

ms_agent/tools/search/tavily/search.py

@@ -15,6 +15,16 @@
 
 TAVILY_SEARCH_URL = 'https://api.tavily.com/search'
 
+#: Tavily serves /search and /extract without credentials when this header is
+#: present (https://docs.tavily.com/documentation/keyless). Responses are
+#: identical to keyed ones — same parameters, same result shape — but the quota
+#: is a small sliding hourly bucket rather than the free tier's monthly credits.
+#: Measured 2026-08-20: refill is roughly one request per 60-90s, and exhaustion
+#: is a clean HTTP 429 (`error.code: hourly_cap_reached`) carrying Retry-After.
+#: It exists so the framework works on first run with nothing configured; a key
+#: always takes precedence when one is available.
+KEYLESS_HEADER = {'X-Tavily-Access-Mode': 'keyless'}
+
 
 class TavilySearch(SearchEngine):
     """
@@ -32,24 +42,36 @@ def __init__(
         api_key: Optional[str] = None,
         request_timeout: float = 120.0,
     ):
-        key = api_key or os.getenv('TAVILY_API_KEY')
-        if not key:
-            raise ValueError(
-                'TAVILY_API_KEY must be set in environment or web_search.tavily_api_key'
-            )
-        self._api_key = key
+        # No key is a supported mode, not an error: without one we fall back to
+        # Tavily's keyless tier so a fresh install can search out of the box.
+        # Constructing this used to raise, which WebSearchTool.connect() caught
+        # and turned into "engine unavailable" — the reason an unconfigured
+        # framework silently had no web search at all.
+        self._api_key = api_key or os.getenv('TAVILY_API_KEY') or ''
         self._request_timeout = float(request_timeout)
 
+    @property
+    def keyless(self) -> bool:
+        return not self._api_key
+
+    def _headers(self) -> dict:
+        return dict(KEYLESS_HEADER) if self.keyless else {}
+
     def search(self,
                search_request: TavilySearchRequest) -> TavilySearchResult:
         body = search_request.to_api_body(self._api_key)
-        try:
-            data = post_json(
-                TAVILY_SEARCH_URL, body, timeout=self._request_timeout)
-        except Exception as e:
-            raise RuntimeError(f'Tavily search failed: {e}') from e
+        # Deliberately unguarded: TavilyHTTPError carries the quota/auth fields
+        # the tool layer needs to tell the agent WHY a search failed. This used
+        # to be wrapped in a bare RuntimeError, which erased them.
+        data = post_json(
+            TAVILY_SEARCH_URL,
+            body,
+            timeout=self._request_timeout,
+            headers=self._headers())
         safe_args = {k: v for k, v in body.items() if k != 'api_key'}
-        safe_args['api_key'] = '<redacted>'
+        if self._api_key:
+            safe_args['api_key'] = '<redacted>'
+        safe_args['access_mode'] = 'keyless' if self.keyless else 'api_key'
         return TavilySearchResult(
             query=search_request.query,
             arguments=safe_args,