fix(api-keys)!: use salted, peppered scrypt digests

CHANGELOG.md

@@ -4,6 +4,10 @@ This repository starts with a fresh history. Releases are listed here once tagge
 
 ## [Unreleased]
 
+- API keys: new key digests are HMAC-SHA256 with a server pepper
+  (`API_KEY_HASH_PEPPER`). Legacy SHA-256 rows still authenticate and are
+  rehashed on their next successful use. Production-like environments refuse
+  to boot without the pepper. See docs/api-key-hashing.md.
 - Upstream MCP: optional multi-tool discovery and exact-name allowlist with
   stable public IDs, per-tool permit scope, and unchanged per-call credits.
 - Initial import of the gateway (Fair Source, FSL-1.1-ALv2), the Python SDK and

gateway/.env.example

@@ -158,6 +158,13 @@ JEV_RISK_GUARD_TIERS=medium,high
 TYPESAFE_API_KEY=
 TYPESAFE_BASE_URL=https://api.typesafe.ai
 
+# Server pepper for HMAC-SHA256 API-key digests. REQUIRED in production-like
+# environments: startup refuses to boot and keys are rejected without it.
+# Generate a long random value (e.g. 32+ random bytes) in your secret store;
+# never commit it. Changing it invalidates every peppered key.
+# See docs/api-key-hashing.md.
+API_KEY_HASH_PEPPER=
+
 # --- IoT Protocol Bridge ---
 MQTT_BROKER_URL=mqtt://localhost:1883
 MQTT_DEFAULT_QOS=1

gateway/app/core/config.py

@@ -248,6 +248,10 @@ def demo_allowed_tools_list(self) -> list[str]:
     JEV_RISK_GUARD_TIMEOUT_SECONDS: float = 1.5
     JEV_RISK_GUARD_TIERS: str = "medium,high"
     TYPESAFE_API_KEY: SecretStr = SecretStr("")
+    # Server pepper for HMAC-SHA256 API-key digests. Required in
+    # production-like environments (startup refuses to boot without it).
+    # See docs/api-key-hashing.md. Never log it.
+    API_KEY_HASH_PEPPER: SecretStr = SecretStr("")
     TYPESAFE_BASE_URL: str = "https://api.typesafe.ai"
     # Default repeat detection window in seconds. Duplicates outside this window
     # are not detected.

gateway/app/main.py

@@ -25,6 +25,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import HTMLResponse, JSONResponse
 
+from app.services.api_key_service import validate_api_key_hashing_config
 from .core.auth import AuthContext, get_auth_context
 from .core.api_key_usage import APIKeyUsageMiddleware
 from .core.build_metadata import get_build_commit_sha, get_build_provenance
@@ -176,6 +177,7 @@
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     validate_trust_mode_guardrails(settings)
+    validate_api_key_hashing_config()
     warn_if_trust_mode_permissive(settings)
     # Operator-facing posture record. The unauthenticated /health/dependencies
     # payload no longer publishes per-service simulation modes when proof

gateway/app/services/api_key_service.py

@@ -5,7 +5,9 @@
 Supports automatic rotation on suspicious activity detection.
 
 Architecture:
-1. Keys are stored hashed (SHA-256) in the database
+1. Keys are stored as HMAC-SHA256 digests keyed by a server pepper
+   (API_KEY_HASH_PEPPER); legacy unkeyed SHA-256 rows still authenticate and
+   are rehashed on their next successful use
 2. Only the key prefix and masked key are ever shown
 3. Rotation creates new key and optionally revokes old ones
 4. Emergency revocation immediately invalidates all keys
@@ -14,6 +16,7 @@
 import hashlib
 import hmac
 import json
+import os
 import secrets
 import logging
 from datetime import datetime, timedelta
@@ -24,7 +27,9 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlmodel import col
 
+from ..core.config import get_settings
 from ..core.time import to_naive_utc, utc_now
+from ..core.trust_mode import is_production_like_environment
 from ..db.database import get_session_factory
 from ..db.models import (
     APIKeyModel,
@@ -92,6 +97,71 @@
     pass
 
 
+API_KEY_HASH_PEPPER_ENV = "API_KEY_HASH_PEPPER"
+
+
+class APIKeyHashingUnavailable(RuntimeError):
+    """Raised when a production-like environment has no API-key pepper."""
+
+
+def _api_key_hash_pepper() -> bytes | None:
+    """The server pepper: process env first, then Settings (``.env``).
+
+    Settings holds it as a SecretStr so it is masked in any settings dump.
+    It is never logged.
+    """
+    value = os.environ.get(API_KEY_HASH_PEPPER_ENV, "").strip()
+    if not value:
+        value = get_settings().API_KEY_HASH_PEPPER.get_secret_value().strip()
+    return value.encode("utf-8") if value else None
+
+
+def _legacy_api_key_hashing_allowed() -> bool:
+    """Unpeppered hashing of NEW keys is allowed only in local/test envs."""
+    return not is_production_like_environment(get_settings().ENVIRONMENT)
+
+
+def api_key_hashing_configured() -> bool:
+    return _api_key_hash_pepper() is not None or _legacy_api_key_hashing_allowed()
+
+
+def validate_api_key_hashing_config() -> None:
+    """Startup guard: production-like environments refuse to boot unpeppered."""
+    if not api_key_hashing_configured():
+        raise APIKeyHashingUnavailable(
+            f"{API_KEY_HASH_PEPPER_ENV} must be set in production-like "
+            "environments. See docs/api-key-hashing.md."
+        )
+
+
+def _peppered_digest(pepper: bytes, presented_key: str) -> str:
+    return hmac.new(pepper, presented_key.encode("utf-8"), hashlib.sha256).hexdigest()
+
+
+def _legacy_digest(presented_key: str) -> str:
+    # Pre-pepper storage format, kept only so existing rows still match and
+    # can be rehashed. API keys are 256-bit random tokens, not passwords;
+    # nothing new is stored in this format outside local/test environments.
+    return hashlib.sha256(presented_key.encode("utf-8")).hexdigest()
+
+
+def hash_api_key(full_key: str) -> str:
+    """Digest to store for a newly issued key.
+
+    HMAC-SHA256 with the server pepper. Without a pepper, local/test
+    environments fall back to the legacy digest; production-like
+    environments fail closed.
+    """
+    pepper = _api_key_hash_pepper()
+    if pepper is not None:
+        return _peppered_digest(pepper, full_key)
+    if _legacy_api_key_hashing_allowed():
+        return _legacy_digest(full_key)
+    raise APIKeyHashingUnavailable(
+        f"{API_KEY_HASH_PEPPER_ENV} is not set; refusing to issue API keys."
+    )
+
+
 def generate_api_key() -> tuple[str, str, str]:
     """
     Generate a new API key.
@@ -100,7 +170,7 @@
         tuple: (full_key, key_hash, key_prefix)
     """
     full_key = f"b2a_{secrets.token_urlsafe(API_KEY_LENGTH)}"
-    key_hash = hashlib.sha256(full_key.encode()).hexdigest()
+    key_hash = hash_api_key(full_key)
     key_prefix = full_key[:API_KEY_PREFIX_LENGTH]
     return full_key, key_hash, key_prefix
 
@@ -452,7 +522,17 @@
             return None
 
         key_prefix = api_key[:API_KEY_PREFIX_LENGTH]
-        key_hash = hashlib.sha256(api_key.encode()).hexdigest()
+        pepper = _api_key_hash_pepper()
+        if pepper is None and not _legacy_api_key_hashing_allowed():
+            # Fail closed: a production-like process without a pepper
+            # authenticates nothing (startup also refuses to boot).
+            logger.error("API key hashing is not configured; rejecting key")
+            return None
+        peppered_hash = (
+            _peppered_digest(pepper, api_key) if pepper is not None else None
+        )
+        legacy_hash = _legacy_digest(api_key)
+        candidate_hashes = [h for h in (peppered_hash, legacy_hash) if h]
 
         async with self._session_factory()() as session:
             # Look up by the indexed full digest, not the prefix alone: the
@@ -463,7 +543,7 @@
             # failed every request for both keys with a 500.
             result = await session.execute(
                 select(APIKeyModel).where(
-                    col(APIKeyModel.key_hash) == key_hash,
+                    col(APIKeyModel.key_hash).in_(candidate_hashes),
                     col(APIKeyModel.key_prefix) == key_prefix,
                     col(APIKeyModel.status) == APIKeyStatus.ACTIVE.value,
                 )
@@ -473,7 +553,14 @@
             if not key:
                 return None
 
-            if not hmac.compare_digest(key.key_hash, key_hash):
+            # HMAC first, legacy second; both compared in constant time.
+            matched_peppered = peppered_hash is not None and hmac.compare_digest(
+                key.key_hash, peppered_hash
+            )
+            matched_legacy = not matched_peppered and hmac.compare_digest(
+                key.key_hash, legacy_hash
+            )
+            if not (matched_peppered or matched_legacy):
                 return None
 
             if not consume:
@@ -539,8 +626,48 @@
                 if (cast(Any, used).rowcount or 0) == 0:
                     return None
 
+            if matched_legacy and peppered_hash is not None:
+                await self._rehash_legacy_key(
+                    session, key, legacy_hash=legacy_hash, peppered_hash=peppered_hash
+                )
+
         return key
 
+    @staticmethod
+    async def _rehash_legacy_key(
+        session: AsyncSession,
+        key: APIKeyModel,
+        *,
+        legacy_hash: str,
+        peppered_hash: str,
+    ) -> None:
+        """Upgrade a legacy SHA-256 row to HMAC after a successful auth.
+
+        Guarded on the legacy digest so concurrent requests upgrade once.
+        Best effort: a failure leaves the legacy row, which still works.
+        """
+        try:
+            result = await session.execute(
+                update(APIKeyModel)
+                .where(
+                    col(APIKeyModel.key_id) == key.key_id,
+                    col(APIKeyModel.key_hash) == legacy_hash,
+                )
+                .values(key_hash=peppered_hash)
+                .execution_options(synchronize_session=False)
+            )
+            await session.commit()
+            if (cast(Any, result).rowcount or 0) == 1:
+                key.key_hash = peppered_hash
+                logger.info("Rehashed legacy API key digest key_id=%s", key.key_id)
+        except Exception as exc:  # noqa: BLE001 - must not fail the request
+            await session.rollback()
+            logger.warning(
+                "Legacy API key rehash failed key_id=%s (%s)",
+                key.key_id,
+                type(exc).__name__,
+            )
+
     async def rotate_key(
         self,
         wallet_id: str,

gateway/docs/README.md

@@ -18,6 +18,7 @@ consequential MCP tool behind a governed boundary. Start with the
 | Run fourteen injected failures and get a signed evidence bundle | [Failure lab suite](failure-lab-suite.md) |
 | Review security claims and limits | [Security limitations](../SECURITY_LIMITATIONS.md) and [security review kit](security-review-kit.md) |
 | Rotate or revoke API keys | [API key rotation](api-key-rotation.md) |
+| Configure API-key hashing (pepper) | [API key hashing](api-key-hashing.md) |
 | Plan a design-partner evaluation | [Design partner guide](../DESIGN_PARTNER_GUIDE.md) |
 | Tell plain pytest, Postgres, and live runs apart | [Test tiers](TESTING_TIERS.md) |
 | Contribute to the repository | [Contributing guide](../../CONTRIBUTING.md) |

gateway/docs/api-key-hashing.md

@@ -0,0 +1,49 @@
+# API key hashing
+
+API keys are stored only as digests. New digests are **HMAC-SHA256 keyed by a
+server pepper** (`API_KEY_HASH_PEPPER`), so a leaked `api_keys` table cannot be
+checked offline against candidate keys without also holding the pepper.
+
+## Configuration
+
+| Variable | Required | Notes |
+|---|---|---|
+| `API_KEY_HASH_PEPPER` | Yes in production-like environments | Long random secret (32+ random bytes). Read from the process environment, then `.env` via Settings (stored as a `SecretStr`). Never logged. |
+
+## Lookup and migration
+
+`APIKeyService.validate_key` computes both candidate digests and looks the row
+up by digest and prefix:
+
+1. **HMAC-SHA256(pepper, key)**: the current format.
+2. **Legacy SHA-256(key)**: rows written before the pepper existed.
+
+Both digests are compared in constant time. When a key matches on its legacy
+digest and a pepper is configured, the row is rewritten to the HMAC digest after
+the authentication succeeds. The update is guarded on the old digest, so
+concurrent requests upgrade it only once. A failed rehash is logged by key id
+and exception type only, and the legacy row keeps working.
+
+No schema change is needed, because both digests are 64 hex characters.
+
+## When the pepper is unset
+
+| Environment | Issue new keys | Validate keys | Startup |
+|---|---|---|---|
+| Local-compatible (`local`, `test`, ...) | Legacy SHA-256 digest | Legacy digest only | Boots |
+| Production-like | Refused (`APIKeyHashingUnavailable`) | Every key rejected | **Refuses to boot** |
+
+Production fails closed so that a missing secret can never quietly put the
+fleet back on unkeyed hashing.
+
+## Deploy steps
+
+1. Generate the pepper and store it in the secret manager for every
+   production-like environment **before** deploying this version. Without it the
+   gateway will not start.
+2. Deploy. Existing keys keep working through the legacy fallback and are
+   rehashed on first use.
+3. Keys that are never used stay in the legacy format. Rotate or revoke them
+   if they should be retired.
+4. Do not change the pepper. Changing it invalidates every rehashed or newly
+   issued key. Rotating the pepper needs its own dual-pepper plan.

gateway/tests/conftest.py

@@ -185,6 +185,9 @@ def require_disposable_test_database() -> None:
 # aiosqlite connection whose worker thread is not a daemon, and an unclosed one
 # blocks interpreter shutdown. See the teardown in setup_database.
 os.environ.setdefault("STATE_BACKEND", "memory")
+# Fixed, non-secret test pepper so tests that switch to a production-like
+# ENVIRONMENT still hash API keys (production fails closed without one).
+os.environ.setdefault("API_KEY_HASH_PEPPER", "amw-test-suite-pepper")
 # Ephemeral SQLite create_all even when a job sets ENVIRONMENT=production
 # (production_trust CI). Never applies to Postgres — see allows_metadata_create_all.
 os.environ.setdefault("ALLOW_METADATA_CREATE_ALL", "true")

gateway/tests/test_api_key_constant_time.py

@@ -20,8 +20,10 @@
 async def test_db_key_digest_comparison_is_constant_time_and_near_miss_is_rejected(
     clean_database, monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    valid_hash = hashlib.sha256(VALID_API_KEY.encode()).hexdigest()
-    near_miss_hash = hashlib.sha256(NEAR_MISS_API_KEY.encode()).hexdigest()
+    # Current (peppered HMAC) storage format, so the lookup matches on the
+    # first constant-time comparison.
+    valid_hash = api_key_service_module.hash_api_key(VALID_API_KEY)
+    near_miss_hash = api_key_service_module.hash_api_key(NEAR_MISS_API_KEY)
     assert (
         VALID_API_KEY[: api_key_service_module.API_KEY_PREFIX_LENGTH]
         == NEAR_MISS_API_KEY[: api_key_service_module.API_KEY_PREFIX_LENGTH]