diff --git a/apps/backend/local-stack/.env.example b/apps/backend/local-stack/.env.example index ae90952c..86018080 100644 --- a/apps/backend/local-stack/.env.example +++ b/apps/backend/local-stack/.env.example @@ -11,12 +11,15 @@ # text — llama.cpp server (OpenAI-compatible /v1) on port 11434 # image — stable-diffusion.cpp sd-server on port 8188 # voice — sherpa-onnx Kokoro TTS on port 8089 -# stt — sherpa-onnx Moonshine STT on port 8087 (same voice container) +# stt — sherpa-onnx Moonshine STT + whisper.cpp on port 8087 (same voice container) # web — opt-in web client container on port 5274 # ollama / comfyui — advanced alternatives on the same ports (advanced) # The model fetcher is profile-scoped: it downloads only the models for the # modalities you enable. -COMPOSE_PROFILES=text,image,voice,stt +# +# STT is OFF by default (C-393 AC-7): a microphone-adjacent service must be +# opt-in. Add `stt` here and set ENABLE_STT=true below to enable it. +COMPOSE_PROFILES=text,image,voice # ── Hardware backend ────────────────────────────────────────────────────── # One of: cpu, cuda, rocm, vulkan, intel, musa. @@ -30,6 +33,22 @@ COMPOSE_FILE=compose.yaml:compose.cpu.yaml TEXT_MODEL=qwen2.5-1.5b-instruct-q4_k_m.gguf IMAGE_MODEL=flux1-schnell-q4_k.gguf +# ── STT model tiers (C-393) ─────────────────────────────────────────────── +# Manifest targetPaths under the models volume; the fetcher downloads ONLY +# the selected entries plus the Silero VAD model. Tiers: +# minimal (shipped default): stt/sherpa-onnx-moonshine-tiny-en-int8 + stt/whisper-tiny/ggml-tiny.bin +# default: stt/sherpa-onnx-moonshine-base-en-int8 + stt/whisper-base/ggml-base.bin +# accuracy: stt/sherpa-onnx-moonshine-base-en-int8 + stt/whisper-small/ggml-small.bin +# STT_STREAM_MODEL=stt/sherpa-onnx-moonshine-tiny-en-int8 +# STT_BATCH_MODEL=stt/whisper-tiny/ggml-tiny.bin +# STT_STREAM_ENGINE=moonshine +# STT_BATCH_ENGINE=whisper-cpp +# STT_ALLOWED_ORIGINS=http://localhost:5274,http://127.0.0.1:5274,tauri://localhost,http://tauri.localhost,https://tauri.localhost +# STT_VAD_THRESHOLD=0.5 +# STT_VAD_MIN_SPEECH_MS=250 +# STT_VAD_MIN_SILENCE_MS=500 +# STT_VAD_MAX_SPEECH_MS=30000 + # ── Licences ───────────────────────────────────────────────────────────── # SD 1.5 is CreativeML OpenRAIL-M (use-restricted). Leave empty to skip it; # the fetcher prints the licence and skips the download. Accept with the @@ -45,9 +64,15 @@ AIKAMI_ACCEPT_LICENSES= # WEB_PORT=5274 # ── Voice extras ───────────────────────────────────────────────────────── -# Set true to also start the STT websocket server inside the voice container -# (required when you enable the `stt` profile). -ENABLE_STT=true +# Enable the C-393 STT service inside the voice container (WS streaming on +# 8087, OpenAI-compatible batch proxy, capabilities + health). Requires ALL +# THREE of: the `stt` profile above, ENABLE_STT=true, and compose.stt.yaml in +# COMPOSE_FILE (the base file never publishes the STT port — AC-7). +# +# COMPOSE_PROFILES=text,image,voice,stt +# COMPOSE_FILE=compose.yaml:compose.cpu.yaml:compose.stt.yaml +# ENABLE_STT=true +ENABLE_STT=false # ── Existing model trees (AC-13) ───────────────────────────────────────── # Point at a pre-existing models/ directory from the old stack to bind-mount diff --git a/apps/backend/local-stack/README.md b/apps/backend/local-stack/README.md index 9a8cf140..1e121a04 100644 --- a/apps/backend/local-stack/README.md +++ b/apps/backend/local-stack/README.md @@ -129,7 +129,7 @@ edit `.env` directly. | `text` | llama.cpp server (`/v1`) | Qwen2.5-1.5B Q4_K_M (CPU default) | | `image` | sd-server | FLUX.1-schnell Q4_K | | `voice` | sherpa-onnx Kokoro TTS | Kokoro-82M | -| `stt` | sherpa-onnx Moonshine STT (same container) | Moonshine tiny | +| `stt` | sherpa-onnx Moonshine streaming + whisper.cpp batch (same container) | Moonshine tiny + whisper tiny + Silero VAD (minimal tier) | | `web` | the web client container | — | The model fetcher is **profile-scoped**: `COMPOSE_PROFILES=text` downloads @@ -140,6 +140,72 @@ only the text model. Enable what you need: COMPOSE_PROFILES=text,image,voice,stt,web ``` +> ⚠️ **STT is opt-in (C-393 AC-7).** The shipped defaults do **not** start +> the STT service: `.env.example` lists `COMPOSE_PROFILES=text,image,voice` +> and `ENABLE_STT=false`. To enable speech-to-text, add `stt` to +> `COMPOSE_PROFILES`, set `ENABLE_STT=true`, **and** include +> `compose.stt.yaml` in `COMPOSE_FILE` (the base file never publishes the +> STT port, so without the override port 8087 stays unbound). A +> microphone-adjacent service that starts unasked would be a privacy +> problem. + +### Speech-to-text (STT) + +The `stt` profile runs two engines inside the same voice container: + +| Protocol | Endpoint | Engine | Use case | +|---|---|---|---| +| Streaming | `WS 127.0.0.1:8087/v1/stream` | sherpa-onnx Moonshine + Silero VAD | Push-to-talk, hands-free; partial hypotheses while speaking | +| Batch | `POST 127.0.0.1:8087/v1/audio/transcriptions` | whisper.cpp | Recorded clips, imported audio; OpenAI-compatible | +| Introspection | `GET 127.0.0.1:8087/v1/capabilities` | — | Engines, models, languages, VAD, `wordTimestamps` | +| Readiness | `GET 127.0.0.1:8087/health` | — | 200 healthy / 503 naming the missing model file | + +The streaming protocol is defined in +`packages/shared/schemas/src/lib/local_ai/stt.ts` (the wire contract C-359 +codes against): + +- **Audio format is fixed**: 16 kHz mono 16-bit PCM (`pcm_s16le`), 32000 + bytes/sec. Resampling is the client's job; the server rejects anything + else with `error: bad-audio-format`. +- Client → server: `{"type":"start","protocolVersion":1,"audio":{...}}` + (JSON text frame — `audio` declares the fixed 16 kHz mono 16-bit PCM + format), then binary frames of raw PCM, then `{"type":"stop"}`. +- Server → client: `ready`, `speech-start`, `partial`*, `final`, + `speech-end`, `error`. VAD runs **server-side** — the client never infers + endpointing. +- **Moonshine is English-only.** Requesting another language returns + `error: unsupported-language` pointing at the batch endpoint; the service + never transcribes non-English audio as garbled English. whisper.cpp + covers ~99 languages for batch. +- The engine behind each protocol is env-selected + (`STT_STREAM_ENGINE`, `STT_BATCH_ENGINE`) — the seam for a future + licensed CrisperWhisper provider with word-level timestamps. + +**Privacy posture**: audio is processed in memory only — it is **never +written to disk, never logged, and never leaves the machine**. The service +binds `127.0.0.1` on the host, and websocket connections are rejected when +the `Origin` header is not on the allowlist (`STT_ALLOWED_ORIGINS`) so a +random web page cannot open a socket to your local transcription service. +There is no debug audio dump, not even behind a flag. + +**Model tiers** (C-393): the fetcher downloads exactly the selected tier +plus the Silero VAD model — not every entry of the modality: + +| Tier | `STT_STREAM_MODEL` | `STT_BATCH_MODEL` | +|---|---|---| +| minimal (shipped default) | `stt/sherpa-onnx-moonshine-tiny-en-int8` | `stt/whisper-tiny/ggml-tiny.bin` | +| default | `stt/sherpa-onnx-moonshine-base-en-int8` | `stt/whisper-base/ggml-base.bin` | +| accuracy | `stt/sherpa-onnx-moonshine-base-en-int8` | `stt/whisper-small/ggml-small.bin` | + +Set the envs in `.env` to select a tier. The minimal tier is shipped as the +default because it is the only one that reliably meets the 300 ms +first-partial latency budget on CPU. + +VAD tuning (all optional): `STT_VAD_THRESHOLD` (default 0.5), +`STT_VAD_MIN_SPEECH_MS` (250), `STT_VAD_MIN_SILENCE_MS` (500), +`STT_VAD_MAX_SPEECH_MS` (30000 — a too-long utterance is capped with a +`final`, then a new segment starts). + ### Advanced: Ollama and ComfyUI Both are available as drop-in alternatives on the **same ports**: @@ -239,10 +305,19 @@ engine on a Mac is CPU-only and slow. On Darwin the supported setup is: ```bash ./bin/run-native-llm.sh # llama-server on 11434 (or shimmy if present) ./bin/run-native-tts.sh # sherpa-onnx Kokoro TTS on 8089 - ./bin/run-native-stt.sh # sherpa-onnx Moonshine STT on 8087 + ./bin/run-native-stt.sh # C-393 STT service on 8087 (Moonshine streaming + whisper.cpp batch) ``` - Each downloads its default model on first run. The native path and the - containerised path expose **identical endpoints** (same ports). + The LLM and TTS launchers download their default models on first run; + `run-native-stt.sh` does **not** — STT models are provisioned by the + model fetcher (like the containerised path), so run the fetcher first or + the script exits with a fetch hint. The native path and the + containerised path expose **identical endpoints** (same ports, same + protocol). `run-native-stt.sh` needs `pip install sherpa-onnx` on the + host; batch transcription additionally needs the whisper.cpp + `whisper-server` binary (`brew` provides `whisper-cli`, but the server + is a source build with `-DWHISPER_BUILD_SERVER=ON` — see the script + header). Without it the streaming service still runs and reports batch + unavailable via `/v1/capabilities`. 2. Only the optional web client is containerised: ```bash COMPOSE_PROFILES=web docker compose up -d @@ -262,7 +337,12 @@ provides no Metal passthrough for the engines. other's health. `docker compose ps` shows meaningful per-service state. - **Missing model**: a service whose model file is absent starts and then fails its own health check — the health message names the missing file, and - the other engines are unaffected. + the other engines are unaffected. With the `stt` profile enabled, the voice + health check covers BOTH the TTS port (8089) and the STT port (8087) — a + dead batch process cannot hide behind a healthy TTS. +- **STT observability**: logs cover connection lifecycle, model load, + language, and decode duration. **Transcript text is never logged** — it is + user speech content (AC-8). - **Offline**: once images and models are cached, the whole stack starts with networking disabled. A missing model disables only its own service. - **Warm start**: an already-provisioned stack reaches all-healthy in well @@ -293,6 +373,11 @@ bun moon run local-stack:lint | Ports match `development_ports.ts`, loopback binds, no 8080 | AC-11 | | Native launchers present, executable, port-defaulted (explicit Darwin branch) | AC-12 | | `MODELS_PATH` bind mount render + health | AC-13 | +| STT off by default (`.env.example` defaults, no STT port render) | C-393 AC-7 | +| STT manifest tiers + no weights COPYed into the image | C-393 AC-11 | +| STT live wire contract (`stt_service.test.ts`: partials+final, VAD, batch, capabilities, language, format, origin) | C-393 AC-1..AC-9 (live) | +| Audio never persisted / transcript never logged (container fs + log grep) | C-393 AC-8 (live) | +| Missing STT model → unhealthy naming the file (throwaway container) | C-393 AC-10 (live) | ## Container security diff --git a/apps/backend/local-stack/bin/run-native-stt.sh b/apps/backend/local-stack/bin/run-native-stt.sh index 9e2b0c3b..bc08fc5a 100755 --- a/apps/backend/local-stack/bin/run-native-stt.sh +++ b/apps/backend/local-stack/bin/run-native-stt.sh @@ -1,42 +1,104 @@ #!/usr/bin/env bash # apps/backend/local-stack/bin/run-native-stt.sh -# Native host launcher for local speech-to-text (STT) without Docker. +# Native host launcher for local speech-to-text (STT) without Docker — the +# macOS path (Docker Desktop has no Metal passthrough; this is a +# latency-sensitive service, C-393 AC-12). # -# Runs the sherpa-onnx C++ offline websocket STT server with a Moonshine -# int8-quantized ONNX model. whisper.cpp users can swap the binary below for -# `whisper-server` (whisper.cpp example server) — the websocket protocol the -# client speaks is what matters. +# Starts the SAME service the container runs, on the same port and protocol: +# - python3 docker/voice/stt_server.py on $STT_PORT (8087) — the C-393 +# streaming websocket (WS /v1/stream, Moonshine + Silero VAD), plus +# GET /v1/capabilities, GET /health, and the OpenAI-compatible batch +# proxy. +# - whisper-server (whisper.cpp) on the internal WHISPER_PORT when the +# binary is present — batch transcription (POST /v1/audio/transcriptions). +# +# Host requirements: +# - python3 (any modern 3.x) with the sherpa-onnx wheel: +# pip install sherpa-onnx +# (and a C compiler + cmake if you build sherpa-onnx from source) +# - whisper.cpp server for batch (optional but recommended): +# brew install whisper-cpp # provides whisper-cli +# # whisper-server needs a source build with WHISPER_BUILD_SERVER=ON: +# # git clone https://github.com/ggml-org/whisper.cpp +# # cmake -B build -DWHISPER_BUILD_SERVER=ON && cmake --build build --target whisper-server +# # ln -s "$PWD/build/bin/whisper-server" /usr/local/bin/ +# +# Models live in ./models/stt and are provisioned by the stack/model +# fetcher — this script never downloads them; it verifies the files exist +# and exits with a fetch hint when a model is missing. Model selection +# mirrors the container: +# STT_STREAM_MODEL / STT_BATCH_MODEL / STT_VAD_MODEL (manifest targetPaths). set -euo pipefail -MODEL_DIR="$(pwd)/models/stt" -MODEL_NAME="sherpa-onnx-moonshine-tiny-en-int8" -MODEL_PATH="$MODEL_DIR/$MODEL_NAME" -# Port from packages/shared/constants development_ports.ts (C-390 AC-11). +# Ports from packages/shared/constants development_ports.ts (C-390 AC-11). PORT="${STT_PORT:-8087}" +# Export so stt_server.py's batch proxy reads the SAME internal port the +# whisper-server was launched on (it defaults to 8091 on its own). +export WHISPER_PORT="${WHISPER_PORT:-8091}" +BIND="${STT_BIND_ADDRESS:-127.0.0.1}" + +# C-393 model selection (manifest targetPaths, mirror the container defaults). +STT_STREAM_MODEL="${STT_STREAM_MODEL:-stt/sherpa-onnx-moonshine-tiny-en-int8}" +STT_BATCH_MODEL="${STT_BATCH_MODEL:-stt/whisper-tiny/ggml-tiny.bin}" +STT_VAD_MODEL="${STT_VAD_MODEL:-stt/silero_vad.onnx}" + +MODELS_DIR="$(cd "$(dirname "$0")/.." && pwd)/models" +STREAM_DIR="$MODELS_DIR/$STT_STREAM_MODEL" +BATCH_FILE="$MODELS_DIR/$STT_BATCH_MODEL" +VAD_FILE="$MODELS_DIR/$STT_VAD_MODEL" -# Verify the sherpa-onnx binary is installed on the host BEFORE downloading -# any model — don't pull gigabytes of weights for a server that can't run. -if ! command -v sherpa-onnx-offline-websocket-server >/dev/null 2>&1; then - echo "❌ sherpa-onnx is not installed on the host." - echo " Install it with: pip install sherpa-onnx" - echo " or download the prebuilt C++ binaries from the k2-fsa GitHub releases." +# Verify the streaming model files exist BEFORE fetching anything the server +# cannot run — the service must not claim readiness without its model. +if [ ! -d "$STREAM_DIR" ] || [ ! -f "$STREAM_DIR/encode.int8.onnx" ]; then + echo "❌ Moonshine STT model missing in $MODELS_DIR/$STT_STREAM_MODEL." + echo " Run the model fetcher: bun stack/fetch_models.ts --entry stt-moonshine-tiny-en-int8 --entry stt-whisper-tiny" + echo " (or download the tarball from the k2-fsa sherpa-onnx releases and" + echo " extract it to $STREAM_DIR)" + exit 1 +fi +if [ ! -f "$VAD_FILE" ]; then + echo "❌ Silero VAD model missing at $VAD_FILE — fetch it with the model fetcher." exit 1 fi -if [ ! -d "$MODEL_PATH" ]; then - echo "Moonshine STT model missing in $MODEL_DIR. Downloading..." - mkdir -p "$MODEL_DIR" - curl -fSL -o "$MODEL_DIR/moonshine.tar.bz2" \ - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2" - tar xjf "$MODEL_DIR/moonshine.tar.bz2" -C "$MODEL_DIR" - rm -f "$MODEL_DIR/moonshine.tar.bz2" +# Export the model paths for stt_server.py (it resolves defaults itself, but +# the explicit exports keep this script the single source of truth). +export MODELS_DIR STT_STREAM_MODEL STT_BATCH_MODEL STT_VAD_MODEL STT_BIND_ADDRESS="$BIND" + +# Batch engine (optional on the host): whisper-server must be installed +# separately; without it the service still streams and reports batch +# unavailable via /v1/capabilities. +if command -v whisper-server >/dev/null 2>&1; then + if [ ! -f "$BATCH_FILE" ]; then + echo "⚠ whisper batch model missing at $BATCH_FILE — batch will be unavailable" + echo " (fetch it with: bun stack/fetch_models.ts --entry stt-whisper-tiny)" + else + echo "Starting whisper.cpp batch server on 127.0.0.1:$WHISPER_PORT ..." + WHISPER_LOG="$(mktemp "${TMPDIR:-/tmp}/whisper-server.XXXXXX.log")" + whisper-server \ + --host 127.0.0.1 \ + --port "$WHISPER_PORT" \ + --model "$BATCH_FILE" \ + --threads "${STT_WHISPER_THREADS:-4}" \ + --no-gpu \ + > "$WHISPER_LOG" 2>&1 & + WHISPER_PID=$! + fi +else + echo "⚠ whisper-server not found on the host — batch endpoint unavailable (streaming still works)" fi -echo "Starting native sherpa-onnx WebSocket STT server on port $PORT..." -exec sherpa-onnx-offline-websocket-server \ - --port="$PORT" \ - --moonshine-preprocessor="$MODEL_PATH/preprocess.onnx" \ - --moonshine-encoder="$MODEL_PATH/encode.int8.onnx" \ - --moonshine-uncached-decoder="$MODEL_PATH/uncached_decode.int8.onnx" \ - --moonshine-cached-decoder="$MODEL_PATH/cached_decode.int8.onnx" \ - --tokens="$MODEL_PATH/tokens.txt" +# Keep the background whisper-server alive while stt_server.py runs and +# clean it up (plus its unique temp log) when the script exits — an exec +# handoff would orphan the batch process once the STT server stopped. +cleanup() { + if [ -n "${WHISPER_PID:-}" ]; then + kill "$WHISPER_PID" 2>/dev/null || true + wait "$WHISPER_PID" 2>/dev/null || true + fi + rm -f "${WHISPER_LOG:-}" +} +trap cleanup EXIT + +echo "Starting native STT server on $BIND:$PORT ..." +python3 "$(dirname "$0")/../docker/voice/stt_server.py" "$PORT" diff --git a/apps/backend/local-stack/compose.stt.yaml b/apps/backend/local-stack/compose.stt.yaml new file mode 100644 index 00000000..d7cafdaf --- /dev/null +++ b/apps/backend/local-stack/compose.stt.yaml @@ -0,0 +1,23 @@ +# apps/backend/local-stack/compose.stt.yaml +# +# C-393 STT port override — adds the STT port publish to the voice service. +# +# The base compose.yaml deliberately does NOT publish the STT port: AC-7 +# requires that the default stack binds no STT port. Docker Compose cannot +# conditionally publish a port per-profile, so enabling STT means adding this +# override to COMPOSE_FILE (documented in .env.example): +# +# COMPOSE_FILE=compose.yaml:compose.cpu.yaml:compose.stt.yaml +# +# Everything else about the STT service (the streaming server, whisper.cpp +# batch engine, health check) is defined in the base file and activated by +# the `stt` profile + ENABLE_STT=true. +# +# Port: 8087 (emulator) / 8086 (staging) / 8090 (production) — from +# packages/shared/constants development_ports.ts (C-390 AC-11). The host +# publish is loopback-only, like every engine. + +services: + voice: + ports: + - "127.0.0.1:${STT_PORT:-8087}:8087" diff --git a/apps/backend/local-stack/compose.yaml b/apps/backend/local-stack/compose.yaml index 07ca4dd5..59dd4295 100644 --- a/apps/backend/local-stack/compose.yaml +++ b/apps/backend/local-stack/compose.yaml @@ -60,6 +60,11 @@ services: AIKAMI_ACCEPT_LICENSES: "${AIKAMI_ACCEPT_LICENSES:-}" TEXT_MODEL: "${TEXT_MODEL:-}" IMAGE_MODEL: "${IMAGE_MODEL:-}" + # C-393: STT entries are tier-selected — only the chosen streaming + + # batch models (plus the Silero VAD model) are fetched, not every + # entry of the modality. + STT_STREAM_MODEL: "${STT_STREAM_MODEL:-}" + STT_BATCH_MODEL: "${STT_BATCH_MODEL:-}" # Named-volume path: hand the populated tree to the engine uid (voice # runs as 1000) so non-root engines can write /models. The # compose.models-path.yaml override sets this to 0 to keep the user's @@ -142,7 +147,12 @@ services: start_period: 30s restart: unless-stopped - # ── Voice engine — sherpa-onnx (TTS /v1/audio/speech, optional STT ws) ─ + # ── Voice engine — sherpa-onnx (TTS /v1/audio/speech, optional STT) ─── + # C-393: with the `stt` profile, the same container hosts the streaming + # websocket + capabilities + health + batch proxy (stt_server.py) on + # $STT_PORT and the whisper.cpp batch engine on the internal $WHISPER_PORT + # (never published). STT is opt-in: ENABLE_STT defaults false (AC-7) and + # the `stt` profile is not part of the shipped COMPOSE_PROFILES. voice: profiles: ["voice", "stt"] build: @@ -150,16 +160,29 @@ services: dockerfile: Dockerfile.sherpa ports: - "127.0.0.1:${TTS_PORT:-8089}:8089" - - "127.0.0.1:${STT_PORT:-8087}:8087" environment: TTS_PORT: "8089" STT_PORT: "8087" + WHISPER_PORT: "8091" ENABLE_STT: "${ENABLE_STT:-false}" # The voice container reads its models from the shared volume (the # fetcher pre-populates it); the entrypoint skips its own download when # the model directory is already present. KOKORO_DIR: "/models/tts/kokoro-multi-lang-v1_0" - STT_DIR: "/models/stt/sherpa-onnx-moonshine-tiny-en-int8" + # C-393 engine + tier selection (manifest targetPaths under /models). + STT_STREAM_ENGINE: "${STT_STREAM_ENGINE:-moonshine}" + STT_BATCH_ENGINE: "${STT_BATCH_ENGINE:-whisper-cpp}" + STT_STREAM_MODEL: "${STT_STREAM_MODEL:-stt/sherpa-onnx-moonshine-tiny-en-int8}" + STT_BATCH_MODEL: "${STT_BATCH_MODEL:-stt/whisper-tiny/ggml-tiny.bin}" + STT_VAD_MODEL: "${STT_VAD_MODEL:-stt/silero_vad.onnx}" + STT_ALLOWED_ORIGINS: "${STT_ALLOWED_ORIGINS:-http://localhost:5274,http://127.0.0.1:5274,tauri://localhost,http://tauri.localhost,https://tauri.localhost}" + # The container must accept WS from the compose network; the host + # publish binding above is loopback-only (AC-11). + STT_BIND_ADDRESS: "0.0.0.0" + STT_VAD_THRESHOLD: "${STT_VAD_THRESHOLD:-0.5}" + STT_VAD_MIN_SPEECH_MS: "${STT_VAD_MIN_SPEECH_MS:-250}" + STT_VAD_MIN_SILENCE_MS: "${STT_VAD_MIN_SILENCE_MS:-500}" + STT_VAD_MAX_SPEECH_MS: "${STT_VAD_MAX_SPEECH_MS:-30000}" volumes: - *model-volume depends_on: @@ -172,6 +195,9 @@ services: - -c - | curl -fsS http://127.0.0.1:8089/health >/dev/null 2>&1 || exit 1 + if [ "${ENABLE_STT:-false}" = "true" ]; then + curl -fsS http://127.0.0.1:8087/health >/dev/null 2>&1 || exit 1 + fi interval: 10s timeout: 5s retries: 12 diff --git a/apps/backend/local-stack/docker/voice/Dockerfile.sherpa b/apps/backend/local-stack/docker/voice/Dockerfile.sherpa index 559cd5ac..74f0bc47 100644 --- a/apps/backend/local-stack/docker/voice/Dockerfile.sherpa +++ b/apps/backend/local-stack/docker/voice/Dockerfile.sherpa @@ -5,9 +5,16 @@ # sherpa-onnx-offline-websocket-server) — no PyTorch, no CUDA, no heavy # Python ML stack at runtime. # +# C-393: the same container also builds whisper.cpp's `whisper-server` +# (batch STT) from source at pinned tag v1.9.2 — the OpenAI-compatible +# batch endpoint (POST /v1/audio/transcriptions) is proxied from +# stt_server.py to it on an internal port. Build-time only: cmake/g++/git +# are removed after the build so the runtime image stays lean. +# # Models are bind-mounted from the host at /models/tts and /models/stt # (see docker-compose.yml). docker/voice/entrypoint.sh auto-downloads the -# default Kokoro TTS model on first start. +# default Kokoro TTS model on first start; STT models come from the C-390 +# model fetcher into the shared volume (AC-11 — no weights in any layer). # # The container runs as a dedicated non-root user. The model store is made # writable through a configurable UID/GID: default 1000:1000 (the typical @@ -21,23 +28,46 @@ FROM python:3.11-slim ARG SHERPA_ONNX_VERSION=1.13.4 +ARG WHISPER_CPP_VERSION=v1.9.2 ARG VOICE_UID=1000 ARG VOICE_GID=1000 # curl/tar/bzip2 are needed by entrypoint.sh (model downloads + STT archive # extraction). sherpa-onnx is manylinux — prebuilt wheels, no compilation. +# whisper.cpp is compiled from source at the pinned tag (C-393 AC-3). RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ tar \ bzip2 \ + make \ + cmake \ + g++ \ + git \ + libgomp1 \ && rm -rf /var/lib/apt/lists/* \ && pip install --no-cache-dir "sherpa-onnx==${SHERPA_ONNX_VERSION}" \ + && git clone --depth 1 --branch "${WHISPER_CPP_VERSION}" https://github.com/ggml-org/whisper.cpp /tmp/whisper.cpp \ + && cmake -S /tmp/whisper.cpp -B /tmp/whisper.cpp/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DWHISPER_BUILD_EXAMPLES=ON \ + -DWHISPER_BUILD_SERVER=ON \ + -DWHISPER_BUILD_TESTS=OFF \ + -DWHISPER_SDL2=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_NATIVE=OFF \ + && cmake --build /tmp/whisper.cpp/build --target whisper-server --config Release -j"$(nproc)" \ + && install -m 0755 /tmp/whisper.cpp/build/bin/whisper-server /usr/local/bin/whisper-server \ + && rm -rf /tmp/whisper.cpp \ + && apt-get purge -y cmake g++ git make \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* \ && groupadd --gid "$VOICE_GID" aikami-voice \ && useradd --uid "$VOICE_UID" --gid "$VOICE_GID" --create-home --shell /usr/sbin/nologin aikami-voice # Entrypoint: model auto-download + TTS/STT server supervisor COPY entrypoint.sh /entrypoint.sh COPY tts_server.py /tts_server.py +COPY stt_server.py /stt_server.py RUN chmod +x /entrypoint.sh # Model store — grant the runtime user ownership so the shared model volume @@ -47,6 +77,7 @@ RUN mkdir -p /models/tts /models/stt \ ENV TTS_PORT=8089 \ STT_PORT=8087 \ + WHISPER_PORT=8091 \ ENABLE_STT=false EXPOSE 8089 8087 diff --git a/apps/backend/local-stack/docker/voice/entrypoint.sh b/apps/backend/local-stack/docker/voice/entrypoint.sh index 8fbef404..c8c0bc47 100755 --- a/apps/backend/local-stack/docker/voice/entrypoint.sh +++ b/apps/backend/local-stack/docker/voice/entrypoint.sh @@ -3,18 +3,25 @@ # Entrypoint for the sherpa-onnx voice container. # # Starts the Kokoro TTS OpenAI-compatible server (/v1/audio/speech) on -# $TTS_PORT (8089). If ENABLE_STT=true, additionally starts an offline STT -# websocket server on $STT_PORT (8087). +# $TTS_PORT (8089). If ENABLE_STT=true, additionally starts the C-393 STT +# service on $STT_PORT (8087): +# - stt_server.py — WS /v1/stream streaming (Moonshine + Silero VAD), +# GET /v1/capabilities, GET /health, and the OpenAI-compatible batch +# proxy POST /v1/audio/transcriptions +# - whisper-server — whisper.cpp batch engine on the INTERNAL $WHISPER_PORT +# (8091, never published) that the proxy forwards to # # Models live under /models (the shared aikami-models volume or the MODELS_PATH # bind mount, populated by the model fetcher): -# /models/tts/kokoro-multi-lang-v1_0 (TTS) -# /models/stt/sherpa-onnx-moonshine-tiny-en-int8 (STT) +# /models/tts/kokoro-multi-lang-v1_0 (TTS) +# /models/stt/ Moonshine streaming (STT) +# /models/stt/ whisper.cpp batch model (STT) +# /models/stt/ Silero VAD (STT) # -# The fetcher pre-populates the volume with checksum-verified downloads; the -# auto-download branch below remains as a fallback for the standalone -# container and the native path, and is skipped when the model directory is -# already present. +# STT models are fetched ONLY by the C-390 model fetcher (AC-11) — the +# auto-download branch below is TTS-only. A missing STT model is not fatal: +# stt_server.py reports unhealthy on /health naming the missing file (AC-10) +# and streaming sessions get error model-not-loaded. set -euo pipefail MODELS_DIR="/models" @@ -24,8 +31,20 @@ TTS_MODEL="${TTS_MODEL:-$KOKORO_DIR/model.onnx}" TTS_VOICES="${TTS_VOICES:-$KOKORO_DIR/voices.bin}" TTS_PORT="${TTS_PORT:-8089}" STT_PORT="${STT_PORT:-8087}" +WHISPER_PORT="${WHISPER_PORT:-8091}" ENABLE_STT="${ENABLE_STT:-false}" +# C-393 engine/model selection — values are manifest targetPaths (relative +# to the models volume). STT_BATCH_MODEL is only used when STT_BATCH_ENGINE +# is whisper-cpp (the seam for a future licensed provider). +STT_STREAM_ENGINE="${STT_STREAM_ENGINE:-moonshine}" +STT_BATCH_ENGINE="${STT_BATCH_ENGINE:-whisper-cpp}" +STT_STREAM_MODEL="${STT_STREAM_MODEL:-stt/sherpa-onnx-moonshine-tiny-en-int8}" +STT_BATCH_MODEL="${STT_BATCH_MODEL:-stt/whisper-tiny/ggml-tiny.bin}" +STT_VAD_MODEL="${STT_VAD_MODEL:-stt/silero_vad.onnx}" + +export STT_STREAM_ENGINE STT_BATCH_ENGINE STT_STREAM_MODEL STT_BATCH_MODEL STT_VAD_MODEL + # The fetcher pre-populates the shared volume with checksum-verified # downloads (and, on the named-volume path, chowns it to the engine uid); the # mkdirs below are defensive only — they must not hard-fail when a directory @@ -54,25 +73,31 @@ fi echo "[voice] Starting Kokoro TTS server on port $TTS_PORT ..." python3 /tts_server.py "$TTS_PORT" & -# ── STT (optional): Moonshine offline websocket server ──────────────────── +# ── STT (optional): C-393 streaming + batch servers ───────────────────── if [ "$ENABLE_STT" = "true" ]; then - STT_DIR="$MODELS_DIR/stt/sherpa-onnx-moonshine-tiny-en-int8" - if [ ! -d "$STT_DIR" ]; then - echo "[voice] Moonshine STT model missing — downloading ..." - curl -fSL -o /tmp/moonshine.tar.bz2 \ - "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-tiny-en-int8.tar.bz2" - tar xjf /tmp/moonshine.tar.bz2 -C "$MODELS_DIR/stt" - rm -f /tmp/moonshine.tar.bz2 + echo "[voice] STT enabled: stream=$STT_STREAM_ENGINE ($STT_STREAM_MODEL), batch=$STT_BATCH_ENGINE ($STT_BATCH_MODEL)" + + # Batch engine: whisper.cpp whisper-server on the internal port. Not + # published to the host; stt_server.py proxies /v1/audio/transcriptions + # to it. Missing binary (native host without whisper.cpp) is tolerated — + # capabilities report batch unavailable. + if [ "$STT_BATCH_ENGINE" = "whisper-cpp" ] && command -v whisper-server >/dev/null 2>&1; then + echo "[voice] Starting whisper.cpp batch server on 127.0.0.1:$WHISPER_PORT ..." + # Print flags default off in whisper-server, so transcript text never + # reaches any log (AC-8); the log file carries model-load lines only. + whisper-server \ + --host 127.0.0.1 \ + --port "$WHISPER_PORT" \ + --model "/models/$STT_BATCH_MODEL" \ + --threads "${STT_WHISPER_THREADS:-4}" \ + --no-gpu \ + > /tmp/whisper-server.log 2>&1 & + elif [ "$STT_BATCH_ENGINE" = "whisper-cpp" ]; then + echo "[voice] whisper-server binary not found — batch endpoint will report unavailable" fi - echo "[voice] Starting Moonshine STT websocket server on port $STT_PORT ..." - sherpa-onnx-offline-websocket-server \ - --port="$STT_PORT" \ - --moonshine-preprocessor="$STT_DIR/preprocess.onnx" \ - --moonshine-encoder="$STT_DIR/encode.int8.onnx" \ - --moonshine-uncached-decoder="$STT_DIR/uncached_decode.int8.onnx" \ - --moonshine-cached-decoder="$STT_DIR/cached_decode.int8.onnx" \ - --tokens="$STT_DIR/tokens.txt" & + echo "[voice] Starting STT streaming server on port $STT_PORT ..." + python3 /stt_server.py "$STT_PORT" & fi # Keep the container alive and surface logs diff --git a/apps/backend/local-stack/docker/voice/stt_server.py b/apps/backend/local-stack/docker/voice/stt_server.py new file mode 100644 index 00000000..ff5257ef --- /dev/null +++ b/apps/backend/local-stack/docker/voice/stt_server.py @@ -0,0 +1,881 @@ +#!/usr/bin/env python3 +"""C-393 STT service — streaming websocket + capabilities + health + batch proxy. + +One stdlib-only Python process on the STT port (8087) inside the sherpa +voice container (docker/voice/entrypoint.sh), serving: + + GET /health readiness — 503 with the missing model file + named when models are absent (AC-10) + GET /v1/capabilities introspection (AC-4) + WS /v1/stream streaming protocol (AC-1/AC-2/AC-5/AC-6/AC-9) + POST /v1/audio/transcriptions OpenAI-compatible batch, proxied to the + whisper.cpp whisper-server on an internal + port (AC-3) + +Wire contract (shared schemas in packages/shared/schemas/src/lib/local_ai/ +stt.ts — the service emits exactly those JSON shapes): + + Client → server (text frames): {"type":"start","protocolVersion":1, + "audio":{"sampleRate":16000,"channels":1, + "encoding":"pcm_s16le"},"language"?} + {"type":"stop"} + Client → server (binary frames): raw 16 kHz mono 16-bit PCM (pcm_s16le) + Server → client (text frames): ready | speech-start | partial | final | + speech-end | error + +Audio is 16 kHz mono 16-bit PCM only (AC-6). VAD runs server-side with Silero +(AC-2) and the client never infers endpointing. Transcript text is NEVER +logged (AC-8): logs cover connection lifecycle, model load, language, and +decode duration only. No audio is ever written to disk. + +Env: + STT_BIND_ADDRESS bind address (default 127.0.0.1; the container overrides + to 0.0.0.0 so the compose network can reach it) + STT_STREAM_MODEL streaming model dir under MODELS_DIR (targetPath from + models.manifest.json, default stt/sherpa-onnx-moonshine-tiny-en-int8) + STT_BATCH_MODEL batch model file under MODELS_DIR (default stt/whisper-tiny/ggml-tiny.bin) + STT_VAD_MODEL Silero VAD model file (default stt/silero_vad.onnx) + STT_STREAM_ENGINE engine selector, seam for licensed providers (default moonshine) + STT_BATCH_ENGINE engine selector (default whisper-cpp) + STT_ALLOWED_ORIGINS comma-separated Origin allowlist for WS (AC-9); an + absent Origin is always allowed (native/Tauri clients) + STT_VAD_THRESHOLD / STT_VAD_MIN_SPEECH_MS / STT_VAD_MIN_SILENCE_MS / + STT_VAD_MAX_SPEECH_MS Silero VAD tuning knobs (see README) + STT_MAX_SESSIONS concurrent streaming sessions (default 1; beyond that + a client gets error: overloaded) + STT_PARTIAL_INTERVAL_MS minimum new-audio between partial decodes (default 300) + STT_IDLE_TIMEOUT_MS socket idle timeout (default 30000) + WHISPER_PORT internal port of the whisper.cpp whisper-server (default 8091) +""" +import base64 +import hashlib +import json +import os +import socket +import struct +import sys +import threading +import time +import urllib.error +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +# ── Constants ───────────────────────────────────────────────────────────── +WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" +SAMPLE_RATE = 16000 +BYTES_PER_SECOND = SAMPLE_RATE * 2 # 16-bit mono +# Bounded trailing window for partial decodes (~2 s of audio): partial +# hypotheses stay cheap as the utterance grows (long-utterance Edge Case). +PARTIAL_WINDOW_SAMPLES = SAMPLE_RATE * 2 + +# whisper.cpp's language list (~99 languages, ISO 639-1 codes). Moonshine is +# English-only (AC-5); the batch engine covers the rest. +WHISPER_LANGUAGES = [ + "en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl", "ca", + "nl", "ar", "sv", "it", "id", "hi", "fi", "vi", "he", "uk", "el", "ms", + "cs", "ro", "da", "hu", "ta", "no", "th", "ur", "hr", "bg", "lt", "la", + "mi", "ml", "cy", "sk", "te", "fa", "lv", "bn", "sr", "az", "sl", "kn", + "et", "mk", "br", "eu", "is", "hy", "ne", "mn", "bs", "kk", "sq", "sw", + "gl", "mr", "pa", "si", "km", "sn", "yo", "so", "af", "oc", "ka", "be", + "tg", "sd", "gu", "am", "yi", "lo", "uz", "fo", "ht", "ps", "tk", "nn", + "mt", "sa", "lb", "my", "bo", "tl", "mg", "as", "tt", "haw", "ln", "ha", + "ba", "jw", "su", +] + +DEFAULT_STREAM_MODEL = "stt/sherpa-onnx-moonshine-tiny-en-int8" +DEFAULT_BATCH_MODEL = "stt/whisper-tiny/ggml-tiny.bin" +DEFAULT_VAD_MODEL = "stt/silero_vad.onnx" + + +def _env_str(name: str, default: str) -> str: + """Read a string env var with a default.""" + return os.environ.get(name, default) + + +def _env_int(name: str, default: int) -> int: + """Read an int env var with a default; invalid values fall back.""" + try: + return int(os.environ.get(name, str(default))) + except ValueError: + return default + + +def _env_float(name: str, default: float) -> float: + """Read a float env var with a default; invalid values fall back.""" + try: + return float(os.environ.get(name, str(default))) + except ValueError: + return default + + +def _now_ms() -> int: + """Wall-clock milliseconds (epoch).""" + return int(time.time() * 1000) + + +# ── Minimal RFC6455 websocket (server side) ─────────────────────────────── +class WebSocket: + """A server-side websocket connection over a raw socket. + + Handles handshake (performed by the HTTP handler), frame parsing with + client-mask unmasking, fragmentation, and ping/pong/close. Server frames + are never masked. Stdlib-only per the container convention. + """ + + def __init__(self, rfile, wfile): + self._rfile = rfile + self._wfile = wfile + + @staticmethod + def accept_key(key: str) -> str: + """RFC6455 Sec-WebSocket-Accept for a client key.""" + digest = hashlib.sha1((key + WS_GUID).encode("ascii")).digest() + return base64.b64encode(digest).decode("ascii") + + def _read_exact(self, n: int) -> bytes: + """Read exactly n bytes from the buffered stream.""" + chunks = [] + remaining = n + while remaining > 0: + chunk = self._rfile.read(remaining) + if not chunk: + raise ConnectionError("websocket stream closed") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + def read_frame(self): + """Read one frame. Returns (fin, opcode, payload) or None on close.""" + try: + header = self._read_exact(2) + except (ConnectionError, OSError): + return None + b0, b1 = header[0], header[1] + fin = bool(b0 & 0x80) + opcode = b0 & 0x0F + masked = bool(b1 & 0x80) + length = b1 & 0x7F + if length == 126: + length = struct.unpack(">H", self._read_exact(2))[0] + elif length == 127: + length = struct.unpack(">Q", self._read_exact(8))[0] + if length > 4 * 1024 * 1024: + raise ValueError("websocket frame too large") + mask_key = self._read_exact(4) if masked else None + payload = self._read_exact(length) if length else b"" + if masked: + payload = bytes(byte ^ mask_key[i % 4] for i, byte in enumerate(payload)) + return fin, opcode, payload + + def send_frame(self, opcode: int, payload: bytes) -> None: + """Send one unmasked server frame.""" + header = bytearray([0x80 | opcode]) + length = len(payload) + if length < 126: + header.append(length) + elif length < 65536: + header.append(126) + header.extend(struct.pack(">H", length)) + else: + header.append(127) + header.extend(struct.pack(">Q", length)) + self._wfile.write(bytes(header) + payload) + self._wfile.flush() + + def send_text(self, text: str) -> None: + """Send a text frame.""" + self.send_frame(0x1, text.encode("utf-8")) + + def send_binary(self, data: bytes) -> None: + """Send a binary frame.""" + self.send_frame(0x2, data) + + def send_close(self, code: int = 1000, reason: str = "") -> None: + """Send a close frame.""" + payload = struct.pack(">H", code) + reason.encode("utf-8") + try: + self.send_frame(0x8, payload) + except (OSError, BrokenPipeError): + pass + + def read_message(self): + """Read one complete message, handling fragmentation + control frames. + + Returns (opcode, payload) or None when the peer closed. + """ + opcode = None + data = b"" + while True: + try: + frame = self.read_frame() + except (ValueError, ConnectionError, OSError): + return None + if frame is None: + return None + fin, op, payload = frame + if op == 0x8: # close + self.send_close(1000) + return None + if op == 0x9: # ping → pong + self.send_frame(0xA, payload) + continue + if op == 0xA: # pong + continue + if op == 0x0: # continuation + if opcode is None: + return None + data += payload + if fin: + return opcode, data + elif op in (0x1, 0x2): + opcode = op + data = payload + if fin: + return opcode, data + else: + return None + + +# ── VAD + incremental decode pipeline ───────────────────────────────────── +class VadPipeline: + """Feeds PCM to Silero VAD and emits speech events + partials/final. + + The sherpa-onnx VoiceActivityDetector exposes completed segments via + front()/pop(); in-progress speech is tracked with our own sample buffer + so partial hypotheses can be produced while the user is still talking. + """ + + def __init__(self, detector, recognizer, on_event, partial_interval_samples: int, decode_lock): + self._detector = detector + self._recognizer = recognizer + self._on_event = on_event + self._partial_interval = partial_interval_samples + self._decode_lock = decode_lock + self._speech = False + self._buffer: list[float] = [] + self._speech_start_ms = 0 + self._last_decode_ms = 0 + self._ever_finalized = False + + def accept(self, samples: list[float]) -> None: + """Accept a chunk of 16 kHz mono float samples from the wire.""" + was_speech = self._speech + self._detector.accept_waveform(samples) + now_speech = self._detector.is_speech_detected() + if not was_speech and now_speech: + self._speech = True + self._speech_start_ms = _now_ms() + self._on_event({"type": "speech-start", "atMs": self._speech_start_ms}) + if now_speech: + self._buffer.extend(samples) + self._maybe_partial() + # Completed utterances (VAD detected silence long enough). + while not self._detector.empty(): + # Finalize BEFORE pop(): sherpa-onnx's VoiceActivityDetector + # invalidates the front() reference on pop(), so read the + # segment (samples/start) while it is still valid. + segment = self._detector.front + self._finalize(segment) + self._detector.pop() + + def _decode(self, samples: list[float]) -> str: + """Run the offline recognizer over the given samples (thread-safe). + + sherpa-onnx 1.13 Python binding: decode_stream() returns None, but + the decoded text is available on the stream's `result` attribute + (verified against the shipped wheel). + """ + if not samples: + return "" + started = _now_ms() + with self._decode_lock: + stream = self._recognizer.create_stream() + stream.accept_waveform(SAMPLE_RATE, samples) + self._recognizer.decode_stream(stream) + text = stream.result.text + elapsed = _now_ms() - started + print(f"[stt] decode {len(samples) / SAMPLE_RATE:.2f}s audio in {elapsed}ms", flush=True) + return text.strip() + + def _maybe_partial(self) -> None: + """Emit a partial when enough NEW audio has accumulated since last.""" + now = _now_ms() + if now - self._last_decode_ms < 150: + return + if len(self._buffer) - self._last_partial_len() < self._partial_interval: + return + self._last_decode_ms = now + self._last_partial_len_marker = len(self._buffer) + # Decode only a bounded trailing window — partial hypotheses concern + # the most recent audio, and decoding the whole utterance grows + # unboundedly with utterance length (long-utterance Edge Case). + text = self._decode(self._buffer[-PARTIAL_WINDOW_SAMPLES:]) + if text: + self._on_event({"type": "partial", "text": text, "atMs": _now_ms()}) + + def _last_partial_len(self) -> int: + return getattr(self, "_last_partial_len_marker", 0) + + def _finalize(self, segment) -> None: + """Emit the final transcript for a completed VAD segment.""" + samples = list(segment.samples) if hasattr(segment, "samples") else list(self._buffer) + # sherpa-onnx SpeechSegment.start is the START SAMPLE INDEX at 16 kHz + # (verified against the 1.13.4 wheel) — convert to ms. + start_ms = int(getattr(segment, "start", 0) / SAMPLE_RATE * 1000) + text = self._decode(samples) + end_ms = _now_ms() + self._on_event({"type": "final", "text": text, "startMs": start_ms, "endMs": end_ms}) + self._on_event({"type": "speech-end", "atMs": end_ms}) + self._speech = False + self._buffer = [] + self._last_partial_len_marker = 0 + self._ever_finalized = True + + def flush(self) -> None: + """Finalize the in-progress utterance on client `stop` (AC-1). + + Exactly one final per session: an in-progress utterance is decoded; + a session that never produced speech gets a single empty final + (silence-only input, Edge Cases). A session whose utterance was + already finalized by VAD emits nothing further. + """ + if self._speech or self._buffer: + start_ms = self._speech_start_ms or _now_ms() + text = self._decode(self._buffer) + end_ms = _now_ms() + self._on_event({"type": "final", "text": text, "startMs": start_ms, "endMs": end_ms}) + self._on_event({"type": "speech-end", "atMs": end_ms}) + self._speech = False + self._buffer = [] + self._last_partial_len_marker = 0 + self._ever_finalized = True + elif not self._ever_finalized: + # Silence-only input: an empty final, never an error (Edge Cases). + now = _now_ms() + self._on_event({"type": "final", "text": "", "startMs": now, "endMs": now}) + self._ever_finalized = True + + +# ── Model loading ───────────────────────────────────────────────────────── +class SttModels: + """Loads and exposes the streaming recognizer + VAD + batch model paths. + + A missing model must not crash the server — /health reports the exact + missing file (AC-10) and streaming sessions get `model-not-loaded`. + """ + + def __init__(self, models_dir: str): + self.models_dir = models_dir + self.stream_model = _env_str("STT_STREAM_MODEL", DEFAULT_STREAM_MODEL) + self.batch_model = _env_str("STT_BATCH_MODEL", DEFAULT_BATCH_MODEL) + self.vad_model = _env_str("STT_VAD_MODEL", DEFAULT_VAD_MODEL) + self.stream_engine = _env_str("STT_STREAM_ENGINE", "moonshine") + self.batch_engine = _env_str("STT_BATCH_ENGINE", "whisper-cpp") + self.whisper_port = _env_int("WHISPER_PORT", 8091) + self.recognizer = None + self.vad = None + self.missing: list[str] = [] + self._load() + + def _path(self, rel: str) -> str: + """Resolve a manifest targetPath under the models dir.""" + return str(Path(self.models_dir) / rel) + + def stream_dir(self) -> str: + """Directory containing the streaming model files.""" + return self._path(self.stream_model) + + def _load(self) -> None: + """Attempt to load the streaming recognizer and the VAD.""" + if self.stream_engine != "moonshine": + self.missing.append(f"streaming engine '{self.stream_engine}' is not available") + print(f"[stt] unsupported STT_STREAM_ENGINE={self.stream_engine}", flush=True) + return + import sherpa_onnx # deferred so /health still works without the wheel + + model_dir = self.stream_dir() + required = { + "preprocess.onnx": Path(model_dir, "preprocess.onnx"), + "encode.int8.onnx": Path(model_dir, "encode.int8.onnx"), + "uncached_decode.int8.onnx": Path(model_dir, "uncached_decode.int8.onnx"), + "cached_decode.int8.onnx": Path(model_dir, "cached_decode.int8.onnx"), + "tokens.txt": Path(model_dir, "tokens.txt"), + } + missing_model = [str(p) for p in required.values() if not p.exists()] + if missing_model: + self.missing.extend(missing_model) + print(f"[stt] streaming model missing: {missing_model}", flush=True) + return + vad_path = self._path(self.vad_model) + if not Path(vad_path).exists(): + self.missing.append(vad_path) + print(f"[stt] VAD model missing: {vad_path}", flush=True) + return + + try: + # sherpa-onnx 1.13 Python API: Moonshine has a dedicated + # constructor (verified against the shipped wheel). + self.recognizer = sherpa_onnx.OfflineRecognizer.from_moonshine( + preprocessor=str(required["preprocess.onnx"]), + encoder=str(required["encode.int8.onnx"]), + uncached_decoder=str(required["uncached_decode.int8.onnx"]), + cached_decoder=str(required["cached_decode.int8.onnx"]), + tokens=str(required["tokens.txt"]), + num_threads=_env_int("STT_NUM_THREADS", 2), + decoding_method="greedy_search", + ) + except Exception as exc: # noqa: BLE001 — surface the load failure + self.missing.append(model_dir) + print(f"[stt] recognizer load failed: {exc}", flush=True) + return + + try: + vad_config = sherpa_onnx.VadModelConfig() + vad_config.sample_rate = SAMPLE_RATE + vad_config.silero_vad.model = vad_path + vad_config.silero_vad.threshold = _env_float("STT_VAD_THRESHOLD", 0.5) + vad_config.silero_vad.min_silence_duration = _env_int( + "STT_VAD_MIN_SILENCE_MS", 500 + ) / 1000.0 + vad_config.silero_vad.min_speech_duration = _env_int( + "STT_VAD_MIN_SPEECH_MS", 250 + ) / 1000.0 + vad_config.silero_vad.max_speech_duration = _env_int( + "STT_VAD_MAX_SPEECH_MS", 30000 + ) / 1000.0 + # 0.5 s internal ring buffer — avoids the circular-buffer overflow + # churn seen with smaller buffers on 1600-sample chunks. + self.vad = sherpa_onnx.VoiceActivityDetector(vad_config, 0.5) + except Exception as exc: # noqa: BLE001 + self.missing.append(vad_path) + print(f"[stt] VAD load failed: {exc}", flush=True) + + if self.recognizer is not None and self.vad is not None: + print(f"[stt] Moonshine model loaded from {model_dir}", flush=True) + print(f"[stt] VAD loaded from {vad_path}", flush=True) + + def batch_available(self) -> bool: + """True only when the whisper.cpp batch server is reachable. + + Probes the internal WHISPER_PORT so /v1/capabilities and /health + reflect process liveness, not just the model file's presence — a + stopped or missing whisper-server must report batch unavailable. + """ + if not Path(self._path(self.batch_model)).exists(): + return False + try: + with socket.create_connection(("127.0.0.1", self.whisper_port), timeout=0.5): + return True + except OSError: + return False + + def batch_model_name(self) -> str: + return Path(self.batch_model).name + + def stream_model_name(self) -> str: + return Path(self.stream_model).name + + def capabilities(self) -> dict: + """The GET /v1/capabilities document (AC-4).""" + return { + "streaming": { + "available": self.recognizer is not None and self.vad is not None, + "engine": self.stream_engine, + "model": self.stream_model_name(), + "languages": ["en"], + "vad": True, + "wordTimestamps": False, + }, + "batch": { + "available": self.batch_available(), + "engine": self.batch_engine, + "model": self.batch_model_name(), + "languages": list(WHISPER_LANGUAGES), + }, + "audio": {"sampleRate": SAMPLE_RATE, "channels": 1, "encoding": "pcm_s16le"}, + "protocolVersion": 1, + } + + def health(self) -> tuple[int, dict]: + """(status_code, body) for GET /health (AC-10).""" + if not self.missing and self.recognizer is not None and self.vad is not None: + return 200, {"status": "ok"} + missing = list(self.missing) + if not self.batch_available(): + missing.append(self._path(self.batch_model)) + return 503, {"status": "unhealthy", "missing": missing} + + +# ── HTTP + websocket handler ────────────────────────────────────────────── +class SttHandler(BaseHTTPRequestHandler): + """Threaded handler: HTTP endpoints + raw websocket upgrade for /v1/stream.""" + + server_version = "aikami-stt/1" + + def log_message(self, *args): # keep container logs quiet (AC-8: no transcripts) + pass + + # ── helpers ── + def _send_json(self, code: int, payload: dict) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_text(self, code: int, text: str, ctype: str = "text/plain") -> None: + body = text.encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + # ── routes ── + def do_GET(self): # noqa: N802 — http.server API + if self.path == "/health": + code, body = self.server.models.health() # type: ignore[attr-defined] + self._send_json(code, body) + return + if self.path == "/v1/capabilities": + self._send_json(200, self.server.models.capabilities()) # type: ignore[attr-defined] + return + if self.path == "/v1/stream": + self._handle_websocket() + return + self._send_text(404, "not found") + + def do_POST(self): # noqa: N802 — http.server API + if self.path == "/v1/audio/transcriptions": + self._proxy_batch() + return + self._send_text(404, "not found") + + # ── websocket (AC-1/AC-2/AC-5/AC-6/AC-9) ── + def _handle_websocket(self) -> None: + # AC-9: reject cross-origin connections before any audio is accepted. + # An absent Origin is deliberately allowed — Tauri webviews and + # non-browser callers may send none at all. + origin = self.headers.get("Origin") + allowed = self.server.allowed_origins # type: ignore[attr-defined] + if origin and origin not in allowed: + print(f"[stt] rejected websocket origin: {origin}", flush=True) + self._send_text(403, "origin not allowed") + return + key = self.headers.get("Sec-WebSocket-Key", "") + if not key: + self._send_text(400, "missing Sec-WebSocket-Key") + return + try: + self.wfile.write(b"HTTP/1.1 101 Switching Protocols\r\n") + self.wfile.write(b"Upgrade: websocket\r\n") + self.wfile.write(b"Connection: Upgrade\r\n") + self.wfile.write(f"Sec-WebSocket-Accept: {WebSocket.accept_key(key)}\r\n".encode()) + self.wfile.write(b"\r\n") + self.wfile.flush() + except OSError: + return + self._run_stream_session(WebSocket(self.rfile, self.wfile)) + + def _run_stream_session(self, ws: WebSocket) -> None: + """The WS /v1/stream session loop (AC-1).""" + models: SttModels = self.server.models # type: ignore[attr-defined] + sessions = self.server.active_sessions # type: ignore[attr-defined] + max_sessions = self.server.max_sessions # type: ignore[attr-defined] + session_start = _now_ms() + + if not sessions.acquire(blocking=False): + print("[stt] session rejected: overloaded", flush=True) + ws.send_text(json.dumps({"type": "error", "code": "overloaded", "message": "server busy"})) + ws.send_close(1013) + return + + try: + idle_timeout = self.server.idle_timeout_ms / 1000.0 # type: ignore[attr-defined] + self.connection.settimeout(idle_timeout) + first = ws.read_message() + if first is None: + return + opcode, payload = first + if opcode != 0x1: + ws.send_text( + json.dumps( + {"type": "error", "code": "internal", "message": "first message must be a JSON start"} + ) + ) + ws.send_close(1002) + return + try: + start_msg = json.loads(payload.decode("utf-8")) + except (ValueError, UnicodeDecodeError): + ws.send_text( + json.dumps({"type": "error", "code": "internal", "message": "invalid JSON start"}) + ) + ws.send_close(1002) + return + + if start_msg.get("type") != "start": + ws.send_text( + json.dumps({"type": "error", "code": "internal", "message": "expected a start message"}) + ) + ws.send_close(1002) + return + if start_msg.get("protocolVersion") != 1: + ws.send_text( + json.dumps( + { + "type": "error", + "code": "protocol-version-mismatch", + "message": f"unsupported protocolVersion {start_msg.get('protocolVersion')}", + } + ) + ) + ws.send_close(1002) + return + + # AC-6: the client must declare the audio format it will stream + # (wire contract). Reject a missing or mismatched declaration + # before any audio is accepted — the format is fixed, so only + # 16 kHz mono 16-bit PCM passes. + audio = start_msg.get("audio") + if not isinstance(audio, dict) or audio != { + "sampleRate": SAMPLE_RATE, + "channels": 1, + "encoding": "pcm_s16le", + }: + ws.send_text( + json.dumps( + { + "type": "error", + "code": "bad-audio-format", + "message": ( + "start message must declare the audio format " + '{"sampleRate": 16000, "channels": 1, "encoding": "pcm_s16le"}' + ), + } + ) + ) + ws.send_close(1002) + return + + # AC-5: Moonshine is English-only — report, never mis-transcribe. + language = start_msg.get("language") + if language and language.lower() not in ("en", "english"): + ws.send_text( + json.dumps( + { + "type": "error", + "code": "unsupported-language", + "message": ( + f"language '{language}' is not supported by the streaming engine; " + "use POST /v1/audio/transcriptions (whisper.cpp) for multilingual audio" + ), + } + ) + ) + ws.send_close(1002) + return + + if models.recognizer is None or models.vad is None: + ws.send_text( + json.dumps( + { + "type": "error", + "code": "model-not-loaded", + "message": f"model not loaded; missing: {models.missing}", + } + ) + ) + ws.send_close(1011) + return + + # Session accepted. + ws.send_text( + json.dumps({"type": "ready", "capabilities": models.capabilities()}) + ) + print( + f"[stt] session start language={language or 'en'} " + f"(after {_now_ms() - session_start}ms)", + flush=True, + ) + + pipeline = VadPipeline( + detector=models.vad, + recognizer=models.recognizer, + on_event=lambda event: ws.send_text(json.dumps(event)), + partial_interval_samples=max( + 1, int(self.server.partial_interval_ms * SAMPLE_RATE / 1000.0) + ), + decode_lock=self.server.decode_lock, # type: ignore[attr-defined] + ) + + # AC-6: validate the audio format on the fly (16k mono s16le). + # Sliding-window byte-rate guard: only sustained OVER-delivery is + # rejected. Pauses, muted clients, and stalls are valid — VAD + # handles endpointing — so there is no under-delivery floor. + rate_window_ms = 1000 + rate_window_start = _now_ms() + rate_window_bytes = 0 + + while True: + message = ws.read_message() + if message is None: + # Client closed — cancel and free buffers immediately. + print("[stt] session closed by client", flush=True) + return + op, payload = message + if op == 0x1: + try: + control = json.loads(payload.decode("utf-8")) + except (ValueError, UnicodeDecodeError): + continue + if control.get("type") == "stop": + pipeline.flush() + ws.send_close(1000) + print( + f"[stt] session end after {_now_ms() - session_start}ms", + flush=True, + ) + return + continue + if op != 0x2 or len(payload) == 0: + continue + if len(payload) % 2 != 0: + self._stream_error(ws, "bad-audio-format", "expected 16 kHz mono 16-bit PCM (even byte frames)") + return + + now = _now_ms() + rate_window_bytes += len(payload) + # Reject only when a full window delivered more than 16 kHz + # mono 16-bit PCM allows (~32000 B/s) plus the ±30% jitter + # tolerance — e.g. 44.1k stereo is ~5.5× the rate. + if now - rate_window_start >= rate_window_ms: + window_elapsed = (now - rate_window_start) / 1000.0 + measured = rate_window_bytes / window_elapsed + if measured > BYTES_PER_SECOND * 1.3: + self._stream_error( + ws, + "bad-audio-format", + ( + f"expected 16 kHz mono 16-bit PCM " + f"(32000 bytes/sec), measured ~{int(measured)} bytes/sec" + ), + ) + return + rate_window_start = now + rate_window_bytes = 0 + + samples = [ + value / 32768.0 + for value in struct.unpack(f"<{len(payload) // 2}h", payload) + ] + pipeline.accept(samples) + + except (ConnectionError, OSError, TimeoutError): + print(f"[stt] session dropped after {_now_ms() - session_start}ms", flush=True) + finally: + sessions.release() + try: + self.connection.settimeout(None) + except OSError: + pass + + def _stream_error(self, ws: WebSocket, code: str, message: str) -> None: + """Send an error frame and close (AC-5/AC-6).""" + ws.send_text(json.dumps({"type": "error", "code": code, "message": message})) + ws.send_close(1002) + + # ── batch proxy (AC-3) ── + def _proxy_batch(self) -> None: + """Forward POST /v1/audio/transcriptions to the internal whisper server.""" + models: SttModels = self.server.models # type: ignore[attr-defined] + whisper_port = self.server.whisper_port # type: ignore[attr-defined] + if models.batch_engine != "whisper-cpp" or not models.batch_available(): + self._send_json( + 503, + { + "error": { + "message": f"batch model not available: {models._path(models.batch_model)}" + } + }, + ) + return + try: + length = int(self.headers.get("Content-Length", 0)) + except ValueError: + length = 0 + body = self.rfile.read(length) if length > 0 else b"" + # whisper.cpp's server exposes the multipart endpoint at /inference; + # its JSON response ({'text': ...}) is OpenAI-shaped, so a path + # rewrite is the whole translation (AC-3). + upstream = f"http://127.0.0.1:{whisper_port}/inference" + request = urllib.request.Request(upstream, data=body, method="POST") + for header in ("Content-Type", "Accept", "Authorization"): + value = self.headers.get(header) + if value: + request.add_header(header, value) + try: + with urllib.request.urlopen(request, timeout=300) as response: # noqa: S310 — local only + data = response.read() + self.send_response(response.status) + self.send_header( + "Content-Type", response.headers.get("Content-Type", "application/json") + ) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + print(f"[stt] batch transcription ok ({len(data)} bytes)", flush=True) + except urllib.error.HTTPError as error: + data = error.read() + self.send_response(error.code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + except Exception as exc: # noqa: BLE001 + self._send_json(502, {"error": {"message": f"batch backend unavailable: {exc}"}}) + + +# ── Server ──────────────────────────────────────────────────────────────── +class SttServer(ThreadingHTTPServer): + """Threaded HTTP/WS server carrying shared STT state.""" + + daemon_threads = True + allow_reuse_address = True + + def __init__(self, address, handler, models: SttModels): + super().__init__(address, handler) + self.models = models + self.allowed_origins = { + origin.strip() + for origin in _env_str( + "STT_ALLOWED_ORIGINS", + "http://localhost:5274,http://127.0.0.1:5274," + "tauri://localhost,http://tauri.localhost,https://tauri.localhost", + ).split(",") + if origin.strip() + } + self.max_sessions = max(1, _env_int("STT_MAX_SESSIONS", 1)) + self.partial_interval_ms = _env_int("STT_PARTIAL_INTERVAL_MS", 300) + self.idle_timeout_ms = _env_int("STT_IDLE_TIMEOUT_MS", 30000) + self.whisper_port = _env_int("WHISPER_PORT", 8091) + self.active_sessions = threading.Semaphore(self.max_sessions) + self.decode_lock = threading.Lock() + + +def main() -> None: + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8087 + bind = _env_str("STT_BIND_ADDRESS", "127.0.0.1") + models_dir = _env_str("MODELS_DIR", "/models") + models = SttModels(models_dir) + server = SttServer((bind, port), SttHandler, models) + print(f"[stt] listening on {bind}:{port}", flush=True) + print(f"[stt] streaming engine={models.stream_engine} model={models.stream_model}", flush=True) + print(f"[stt] batch engine={models.batch_engine} model={models.batch_model}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/apps/backend/local-stack/scripts/check.sh b/apps/backend/local-stack/scripts/check.sh index abe1baff4..4c36bb44 100755 --- a/apps/backend/local-stack/scripts/check.sh +++ b/apps/backend/local-stack/scripts/check.sh @@ -16,6 +16,7 @@ cd "$(dirname "$0")/.." fail=0 pass=0 +skip_count=0 ok() { pass=$((pass + 1)) @@ -27,6 +28,13 @@ bad() { echo "FAIL - $1" } +# A check that could not run (missing image, container could not start) is +# NOT a pass — reserve ok for a completed assertion. +skip() { + skip_count=$((skip_count + 1)) + echo "skip - $1" +} + check() { local desc="$1" shift @@ -46,13 +54,53 @@ else tail -30 /tmp/aikami-local-stack-unit.log >&2 fi -# ── Bash syntax ────────────────────────────────────────────────────────── -echo "== bash syntax ==" +# ── Bash / Python syntax ───────────────────────────────────────────────── +echo "== bash / python syntax ==" check "bash syntax: bin/run-native-llm.sh" bash -n bin/run-native-llm.sh check "bash syntax: bin/run-native-tts.sh" bash -n bin/run-native-tts.sh check "bash syntax: bin/run-native-stt.sh" bash -n bin/run-native-stt.sh check "bash syntax: docker/voice/entrypoint.sh" bash -n docker/voice/entrypoint.sh check "bash syntax: scripts/emit_config.sh" bash -n scripts/emit_config.sh +check "python syntax: docker/voice/stt_server.py" python3 -m py_compile docker/voice/stt_server.py +check "python syntax: docker/voice/tts_server.py" python3 -m py_compile docker/voice/tts_server.py + +# ── C-393 AC-7: STT is off by default ─────────────────────────────────── +echo "== C-393 AC-7: STT off by default ==" +# The shipped default must not enable the stt profile — assert the absence +# of `stt` rather than an exact full profile list so unrelated default +# profiles can change without breaking this check. +if grep -q '^COMPOSE_PROFILES=' .env.example \ + && ! grep -qE '^COMPOSE_PROFILES=.*\bstt\b' .env.example; then + ok "AC-7: .env.example does not enable the stt profile" +else + bad "AC-7: .env.example must not list stt in COMPOSE_PROFILES" +fi +if grep -q '^ENABLE_STT=false' .env.example; then + ok "AC-7: .env.example sets ENABLE_STT=false" +else + bad "AC-7: .env.example must set ENABLE_STT=false" +fi +if grep -q '^ENABLE_STT=true' .env.example; then + bad "AC-7: .env.example must not set ENABLE_STT=true" +else + ok "AC-7: .env.example has no ENABLE_STT=true" +fi +# The stt profile must not bind any port when it is not enabled. Pin +# COMPOSE_FILE explicitly so ambient .env values (e.g. a developer's +# COMPOSE_FILE that includes compose.stt.yaml) cannot leak the STT port +# into the no-STT render. +render_no_stt=$(COMPOSE_FILE="compose.yaml:compose.cpu.yaml" COMPOSE_PROFILES=text,image,voice docker compose config 2>/dev/null) +if printf '%s' "$render_no_stt" | grep -qE "published: *[\"']?8087[\"']?"; then + bad "AC-7: no-STT render must not publish the STT port" +else + ok "AC-7: no-STT render publishes no STT port" +fi +# C-390's check asserted the old default; the shipped default is now opt-in. +if grep -q 'profiles: \["voice", "stt"\]' compose.yaml; then + ok "C-393: voice service carries the stt profile" +else + bad "C-393: voice service lost the stt profile" +fi # ── Native launcher path (AC-12) ────────────────────────────────────────── # On Darwin the engines run natively (Docker Desktop has no Metal @@ -67,12 +115,37 @@ done check "AC-12: run-native-llm.sh defaults to port 11434" grep -q 'LLM_PORT:-11434' bin/run-native-llm.sh check "AC-12: run-native-tts.sh defaults to port 8089" grep -q 'TTS_PORT:-8089' bin/run-native-tts.sh check "AC-12: run-native-stt.sh defaults to port 8087" grep -q 'STT_PORT:-8087' bin/run-native-stt.sh +check "AC-12: run-native-stt.sh drives stt_server.py (C-393 protocol)" grep -q 'stt_server.py' bin/run-native-stt.sh if [ "$(uname -s)" = "Darwin" ]; then ok "AC-12: Darwin — native path verified (no Metal passthrough, engines run natively)" else ok "AC-12: non-Darwin — native launchers shipped and port-defaulted" fi +# ── C-393 AC-11: STT models come from the manifest ────────────────────── +echo "== C-393 AC-11: STT manifest + no weights in images ==" +check "AC-11: manifest carries stt-moonshine-tiny-en-int8" grep -q '"id": "stt-moonshine-tiny-en-int8"' stack/models.manifest.json +check "AC-11: manifest carries stt-moonshine-base-en-int8" grep -q '"id": "stt-moonshine-base-en-int8"' stack/models.manifest.json +check "AC-11: manifest carries stt-whisper-tiny" grep -q '"id": "stt-whisper-tiny"' stack/models.manifest.json +check "AC-11: manifest carries stt-whisper-base" grep -q '"id": "stt-whisper-base"' stack/models.manifest.json +check "AC-11: manifest carries stt-whisper-small" grep -q '"id": "stt-whisper-small"' stack/models.manifest.json +check "AC-11: manifest carries stt-silero-vad" grep -q '"id": "stt-silero-vad"' stack/models.manifest.json +# The Dockerfile must not COPY any weights into an image layer. +if grep -q 'COPY .*models/' docker/voice/Dockerfile.sherpa; then + bad "AC-11: Dockerfile copies model weights into the image" +else + ok "AC-11: no weights COPYed into the voice image" +fi +# The fetcher must gate STT downloads by tier (Watch Point). +check "AC-11: fetcher tier-selects STT entries" grep -q 'selectSttEntries' stack/fetch_models.ts +check "AC-11: compose passes STT_STREAM_MODEL to the fetcher" grep -q 'STT_STREAM_MODEL' compose.yaml +# The voice container must NOT auto-download STT models (fetcher-owned). +if grep -E 'sherpa-onnx-moonshine.*curl|curl.*sherpa-onnx-moonshine' docker/voice/entrypoint.sh; then + bad "AC-11: entrypoint auto-downloads STT models" +else + ok "AC-11: entrypoint does not auto-download STT models" +fi + # ── Compose topology (AC-2, AC-3, AC-11) ───────────────────────────────── echo "== compose topology ==" @@ -91,6 +164,7 @@ check "compose parses: vulkan override" docker compose -f compose.yaml -f compos check "compose parses: intel override" docker compose -f compose.yaml -f compose.intel.yaml config --quiet check "compose parses: musa override" docker compose -f compose.yaml -f compose.musa.yaml config --quiet check "compose parses: models-path override" docker compose -f compose.yaml -f compose.models-path.yaml config --quiet +check "compose parses: stt override" docker compose -f compose.yaml -f compose.cpu.yaml -f compose.stt.yaml config --quiet for profile in text image voice stt web; do svcs=$(profile_services "$profile") @@ -184,8 +258,10 @@ done # AC-11: no service binds 8080 and engine ports match the table. The # container-side listen address may legitimately be 0.0.0.0 (engines bind # all container interfaces so the compose network can reach them); AC-11 -# constrains the HOST publish binding, which must be 127.0.0.1. -rendered_all=$(COMPOSE_PROFILES=text,image,voice,stt,web docker compose config 2>/dev/null) +# constrains the HOST publish binding, which must be 127.0.0.1. The +# all-profiles render includes compose.stt.yaml so the STT port publish is +# asserted where it actually lives (C-393 AC-7). +rendered_all=$(COMPOSE_FILE="compose.yaml:compose.cpu.yaml:compose.stt.yaml" COMPOSE_PROFILES=text,image,voice,stt,web docker compose config 2>/dev/null) for port in 8080; do if printf '%s' "$rendered_all" | grep -qE "published: *[\"']?${port}[\"']?"; then bad "AC-11: host port $port is published (Nordclaw-owned)" @@ -393,11 +469,89 @@ if [ "${LOCAL_STACK_LIVE:-0}" = "1" ]; then else bad "AC-4: web client HTTP 200" fi + + # ── C-393 live STT probes (AC-1..AC-10) ──────────────────────────── + echo "== C-393 live STT probes ==" + if curl -fsS http://127.0.0.1:8087/health >/dev/null 2>&1; then + ok "C-393: STT /health" + else + bad "C-393: STT /health (is the stt profile enabled?)" + fi + if curl -fsS http://127.0.0.1:8087/v1/capabilities 2>/dev/null | grep -q '\"moonshine\"'; then + ok "C-393: STT /v1/capabilities reports the streaming engine" + else + bad "C-393: STT /v1/capabilities" + fi + # The full wire contract (AC-1 partials+final, AC-2 VAD, AC-3 batch, + # AC-4 schema, AC-5 language, AC-6 format, AC-9 origin) is exercised by + # the integration spec against the live service. + if STT_URL=http://127.0.0.1:8087 timeout 180 bun test stack/stt_service.test.ts \ + >/tmp/aikami-stt-service.log 2>&1; then + ok "C-393: stt_service.test.ts passed (wire contract)" + else + bad "C-393: stt_service.test.ts failed — see /tmp/aikami-stt-service.log" + tail -40 /tmp/aikami-stt-service.log >&2 + fi + + # ── AC-8: audio is never persisted or logged ──────────────────────── + # Scan ONLY service-writable paths: /tmp and the container root (the + # /models tree legitimately contains the model tarballs' bundled + # test_wavs — fetched artifacts, not service-written audio). + if docker compose exec -T voice sh -c 'find /tmp /app /root -type f \( -name "*.wav" -o -name "*.pcm" -o -name "*.raw" \) 2>/dev/null' \ + | grep -q .; then + bad "AC-8: audio files found in the voice container" + else + ok "AC-8: no audio files written in the voice container" + fi + if docker compose logs voice 2>&1 | grep -iE 'hello world|good morning|transcript:'; then + bad "AC-8: transcript text appears in voice logs" + else + ok "AC-8: no transcript text in voice logs" + fi + + # ── AC-10: missing model → unhealthy naming the file ──────────────── + # Run a throwaway voice container (no port publish — the main stack + # owns 8087) with a nonexistent STT model and assert /health reports + # 503 naming the missing file; the rest of the stack is untouched. + if docker image inspect aikami-local-stack-voice >/dev/null 2>&1; then + MISSING_IMG="aikami-local-stack-voice" + else + MISSING_IMG=$(docker compose config --images 2>/dev/null | grep -i voice | head -1 || true) + fi + if [ -n "${MISSING_IMG:-}" ]; then + cid=$(docker run -d --rm --name aikami-stt-missing-test \ + -v aikami-models:/models \ + -e ENABLE_STT=true \ + -e STT_STREAM_MODEL=stt/does-not-exist \ + -e STT_VAD_MODEL=stt/does-not-exist.onnx \ + -e STT_BIND_ADDRESS=0.0.0.0 \ + "$MISSING_IMG" 2>/dev/null || true) + if [ -n "$cid" ]; then + body="" + for _ in 1 2 3 4 5 6; do + sleep 3 + body=$(docker exec aikami-stt-missing-test curl -s http://127.0.0.1:8087/health 2>/dev/null || true) + if printf '%s' "$body" | grep -q 'unhealthy'; then + break + fi + done + if printf '%s' "$body" | grep -q 'unhealthy' && printf '%s' "$body" | grep -q 'does-not-exist'; then + ok "AC-10: missing model reported unhealthy naming the file" + else + bad "AC-10: /health body does not name the missing model (got: $body)" + fi + docker rm -f aikami-stt-missing-test >/dev/null 2>&1 || true + else + skip "AC-10: throwaway voice container could not start (no assertion made)" + fi + else + skip "AC-10: voice image not found (no assertion made)" + fi fi echo if [ "$fail" -ne 0 ]; then - echo "❌ local-stack checks failed: $fail failure(s), $pass pass" + echo "❌ local-stack checks failed: $fail failure(s), $pass pass, $skip_count skipped" exit 1 fi -echo "✅ local-stack checks passed: $pass pass, 0 failures" +echo "✅ local-stack checks passed: $pass pass, $skip_count skipped, 0 failures" diff --git a/apps/backend/local-stack/stack/env_writer.ts b/apps/backend/local-stack/stack/env_writer.ts index 05ead641..5e5b252a 100644 --- a/apps/backend/local-stack/stack/env_writer.ts +++ b/apps/backend/local-stack/stack/env_writer.ts @@ -49,7 +49,11 @@ export const renderEnv = (options: { }): string => { const { profile, plan, manifest, extras } = options; const separator = composeSeparator(profile.platform); - const composeFile = BACKEND_COMPOSE_FILE[plan.backend].split(':').join(separator); + // C-393: the STT port lives in compose.stt.yaml (AC-7 — the default stack + // must bind no STT port). `stack init` appends the override when the plan + // includes the stt modality so the generated .env actually publishes it. + const sttOverride = plan.modalities.includes('stt') ? `${separator}compose.stt.yaml` : ''; + const composeFile = BACKEND_COMPOSE_FILE[plan.backend].split(':').join(separator) + sttOverride; const lines: string[] = []; lines.push('# Generated by `bun run stack init` (C-391). Edit freely; a re-run shows a diff.'); diff --git a/apps/backend/local-stack/stack/fetch_models.ts b/apps/backend/local-stack/stack/fetch_models.ts index 96d94d49..f8675047 100644 --- a/apps/backend/local-stack/stack/fetch_models.ts +++ b/apps/backend/local-stack/stack/fetch_models.ts @@ -14,6 +14,10 @@ * pinned digest in models.manifest.json; a corrupt file is re-fetched. * - Profile-scoped: only modalities enabled via COMPOSE_PROFILES are * fetched (text → text, image → image, voice → tts, stt → stt). + * - STT tier-scoped (C-393): the stt profile fetches exactly the selected + * streaming + batch models plus the Silero VAD entry, gated by the + * STT_STREAM_MODEL / STT_BATCH_MODEL envs (default: minimal tier). + * An explicit --entry request bypasses the tier filter. * - Non-fatal: a failed download for one modality does not change the exit * status (0) as long as the enabled modalities were attempted, so one * dead mirror cannot prevent unrelated engines from starting. @@ -69,6 +73,62 @@ export const PROFILE_MODALITY: Readonly> = { export const ALL_MODALITIES: readonly Modality[] = ['text', 'image', 'tts', 'stt'] as const; +// C-393: STT model tiers. The stt profile must NOT download every tier — +// selecting a tier means fetching exactly the VAD model plus the entries +// for the chosen streaming + batch models (Watch Point: “the fetcher +// downloads every entry of a modality”). Values are manifest targetPaths. +export const STT_VAD_ENTRY_ID = 'stt-silero-vad'; +export const DEFAULT_STT_STREAM_MODEL = 'stt/sherpa-onnx-moonshine-tiny-en-int8'; +export const DEFAULT_STT_BATCH_MODEL = 'stt/whisper-tiny/ggml-tiny.bin'; + +/** + * Filters manifest entries to the STT tier selection. + * + * The Silero VAD model is mandatory; the streaming and batch entries are + * matched by targetPath (exact, or a directory prefix for archive entries). + * When an env is unset the shipped default (minimal tier) is used. + */ +export const selectSttEntries = (options: { + entries: readonly ManifestEntry[]; + streamModel?: string; + batchModel?: string; +}): ManifestEntry[] => { + const { entries } = options; + // Empty strings (compose interpolation of an unset var) mean "use the + // shipped default" just like undefined. + const stream = + options.streamModel && options.streamModel.length > 0 + ? options.streamModel + : DEFAULT_STT_STREAM_MODEL; + const batch = + options.batchModel && options.batchModel.length > 0 + ? options.batchModel + : DEFAULT_STT_BATCH_MODEL; + const matches = + (value: string): ((entry: ManifestEntry) => boolean) => + (entry) => + entry.targetPath === value || entry.targetPath.startsWith(`${value}/`); + const sttEntries = entries.filter((entry) => entry.modality === 'stt'); + const selected = sttEntries.filter( + (entry) => entry.id === STT_VAD_ENTRY_ID || matches(stream)(entry) || matches(batch)(entry), + ); + // Warn when a configured selector matches no manifest entry — the fetcher + // would silently skip the requested model (Watch Point: tier selection). + if (!sttEntries.some(matches(stream))) { + // biome-ignore lint/suspicious/noConsole: container log (standalone script) + console.warn( + `[fetcher] STT_STREAM_MODEL '${stream}' matches no manifest entry — streaming model will not be fetched`, + ); + } + if (!sttEntries.some(matches(batch))) { + // biome-ignore lint/suspicious/noConsole: container log (standalone script) + console.warn( + `[fetcher] STT_BATCH_MODEL '${batch}' matches no manifest entry — batch model will not be fetched`, + ); + } + return selected; +}; + /** * Parses and validates the manifest JSON. * @param path — Path to models.manifest.json. @@ -424,6 +484,10 @@ export const run = async (options: { entryId?: string; /** Explicit entry ids to fetch — limits the run to exactly these planned models. */ entryIds?: readonly string[]; + /** C-393 STT tier selection (overrides STT_STREAM_MODEL env). */ + sttStreamModel?: string; + /** C-393 STT tier selection (overrides STT_BATCH_MODEL env). */ + sttBatchModel?: string; onProgress?: (options: { entryId: string; received: number; expected: number }) => void; }): Promise => { const manifestPath = options.manifestPath ?? join(import.meta.dir, 'models.manifest.json'); @@ -474,11 +538,24 @@ export const run = async (options: { await mkdir(modelsDir, { recursive: true }); + // C-393: the stt profile is tier-selected by STT_STREAM_MODEL / STT_BATCH_MODEL + // (defaults: minimal tier). An explicit --entry / entryIds request bypasses + // the tier filter — the caller asked for exactly those models. + const explicitEntryFilter = options.entryId || options.entryIds; + const sttSelection = new Set( + selectSttEntries({ + entries: manifest.entries, + streamModel: options.sttStreamModel ?? process.env.STT_STREAM_MODEL ?? undefined, + batchModel: options.sttBatchModel ?? process.env.STT_BATCH_MODEL ?? undefined, + }).map((entry) => entry.id), + ); + const selectedEntries = manifest.entries.filter( (entry) => enabledModalities.has(entry.modality) && (!options.entryId || entry.id === options.entryId) && - (!options.entryIds || options.entryIds.includes(entry.id)), + (!options.entryIds || options.entryIds.includes(entry.id)) && + (explicitEntryFilter || entry.modality !== 'stt' || sttSelection.has(entry.id)), ); if (selectedEntries.length === 0) { diff --git a/apps/backend/local-stack/stack/fixtures/.gitkeep b/apps/backend/local-stack/stack/fixtures/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/apps/backend/local-stack/stack/fixtures/stt_test_utterance.wav b/apps/backend/local-stack/stack/fixtures/stt_test_utterance.wav new file mode 100644 index 00000000..4098e202 Binary files /dev/null and b/apps/backend/local-stack/stack/fixtures/stt_test_utterance.wav differ diff --git a/apps/backend/local-stack/stack/models.manifest.json b/apps/backend/local-stack/stack/models.manifest.json index 1f1b141e..abeadd2d 100644 --- a/apps/backend/local-stack/stack/models.manifest.json +++ b/apps/backend/local-stack/stack/models.manifest.json @@ -94,6 +94,72 @@ "targetPath": "stt/sherpa-onnx-moonshine-tiny-en-int8", "bytes": 107600538, "sha256": "d5fe6ec4334fef36255b2a4010412cad4c007e33103fec62fb5d17cad88086f2" + }, + { + "id": "stt-moonshine-base-en-int8", + "modality": "stt", + "tier": "any", + "license": "MIT", + "requiresAcknowledgement": false, + "kind": "archive", + "url": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-moonshine-base-en-int8.tar.bz2", + "targetPath": "stt/sherpa-onnx-moonshine-base-en-int8", + "bytes": 250807309, + "sha256": "21870cecaa2e44e4e2bf63e02d1072bed183ccd10284871353bd9d24dad14e5e" + }, + { + "id": "stt-whisper-tiny", + "modality": "stt", + "tier": "any", + "license": "MIT", + "requiresAcknowledgement": false, + "kind": "file", + "repo": "ggerganov/whisper.cpp", + "revision": "5359861c739e955e79d9a303bcbc70fb988958b1", + "file": "ggml-tiny.bin", + "targetPath": "stt/whisper-tiny/ggml-tiny.bin", + "bytes": 77691713, + "sha256": "be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21" + }, + { + "id": "stt-whisper-base", + "modality": "stt", + "tier": "any", + "license": "MIT", + "requiresAcknowledgement": false, + "kind": "file", + "repo": "ggerganov/whisper.cpp", + "revision": "5359861c739e955e79d9a303bcbc70fb988958b1", + "file": "ggml-base.bin", + "targetPath": "stt/whisper-base/ggml-base.bin", + "bytes": 147951465, + "sha256": "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe" + }, + { + "id": "stt-whisper-small", + "modality": "stt", + "tier": "any", + "license": "MIT", + "requiresAcknowledgement": false, + "kind": "file", + "repo": "ggerganov/whisper.cpp", + "revision": "5359861c739e955e79d9a303bcbc70fb988958b1", + "file": "ggml-small.bin", + "targetPath": "stt/whisper-small/ggml-small.bin", + "bytes": 487601967, + "sha256": "1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b" + }, + { + "id": "stt-silero-vad", + "modality": "stt", + "tier": "any", + "license": "MIT", + "requiresAcknowledgement": false, + "kind": "file", + "url": "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx", + "targetPath": "stt/silero_vad.onnx", + "bytes": 643854, + "sha256": "9e2449e1087496d8d4caba907f23e0bd3f78d91fa552479bb9c23ac09cbb1fd6" } ] } diff --git a/apps/backend/local-stack/stack/stt.test.ts b/apps/backend/local-stack/stack/stt.test.ts new file mode 100644 index 00000000..d3f38d33 --- /dev/null +++ b/apps/backend/local-stack/stack/stt.test.ts @@ -0,0 +1,234 @@ +/** + * apps/backend/local-stack/stack/stt.test.ts + * + * C-393 STT model tier tests: + * - the manifest carries every STT tier entry with a pinned digest + * (AC-11); + * - the fetcher's STT selection downloads exactly the chosen tier + the + * Silero VAD model — never every tier (Watch Point); + * - an explicit entry request bypasses the tier filter. + * + * Downloads run against a local HTTP server with known content so no + * external network is needed. + */ + +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { Server } from 'bun'; +import { + DEFAULT_STT_BATCH_MODEL, + DEFAULT_STT_STREAM_MODEL, + loadManifest, + type ManifestEntry, + run, + STT_VAD_ENTRY_ID, + selectSttEntries, +} from './fetch_models.ts'; + +const CONTENT = Buffer.from('stt tier test model bytes 0123456789'); +const contentSha = await (async () => { + const { createHash } = await import('node:crypto'); + return createHash('sha256').update(CONTENT).digest('hex'); +})(); + +let server: Server; +let baseUrl = ''; + +const handler = async (req: Request): Promise => { + const url = new URL(req.url); + if (url.pathname === '/model.bin') { + return new Response(CONTENT, { headers: { 'Content-Length': String(CONTENT.length) } }); + } + return new Response('not found', { status: 404 }); +}; + +beforeAll(async () => { + server = Bun.serve({ port: 0, fetch: handler }); + baseUrl = `http://127.0.0.1:${server.port}`; +}); + +afterAll(() => { + server.stop(true); +}); + +const makeSttEntry = (overrides: Partial = {}): ManifestEntry => ({ + id: 'stt-moonshine-tiny-en-int8', + modality: 'stt', + tier: 'any', + license: 'MIT', + requiresAcknowledgement: false, + kind: 'file', + url: `${baseUrl}/model.bin`, + targetPath: DEFAULT_STT_STREAM_MODEL, + bytes: CONTENT.length, + sha256: contentSha, + ...overrides, +}); + +const makeTmpDir = async (): Promise => mkdtemp(join(tmpdir(), 'aikami-stt-')); + +const MANIFEST_PATH = join(import.meta.dir, 'models.manifest.json'); + +describe('AC-11 — STT manifest entries', () => { + it('carries every tier entry with a pinned sha256', async () => { + const manifest = await loadManifest(MANIFEST_PATH); + const stt = manifest.entries.filter((entry) => entry.modality === 'stt'); + const ids = stt.map((entry) => entry.id); + expect(ids).toContain('stt-moonshine-tiny-en-int8'); + expect(ids).toContain('stt-moonshine-base-en-int8'); + expect(ids).toContain('stt-whisper-tiny'); + expect(ids).toContain('stt-whisper-base'); + expect(ids).toContain('stt-whisper-small'); + expect(ids).toContain(STT_VAD_ENTRY_ID); + for (const entry of stt) { + expect(entry.sha256).toMatch(/^[0-9a-f]{64}$/); + expect(entry.bytes).toBeGreaterThan(0); + } + }); + + it('ships the minimal tier as the default (matches the shipped stack)', async () => { + const manifest = await loadManifest(MANIFEST_PATH); + const byPath = new Map(manifest.entries.map((entry) => [entry.targetPath, entry])); + // The shipped default references (compose/.env/entrypoint) must resolve + // to real manifest entries — Watch Point: reconcile default-tier model + // with the shipped stack. + expect(byPath.has(DEFAULT_STT_STREAM_MODEL)).toBe(true); + expect(byPath.has(DEFAULT_STT_BATCH_MODEL)).toBe(true); + expect(byPath.has('stt/silero_vad.onnx')).toBe(true); + }); + + it('whisper models are pinned to a repository revision', async () => { + const manifest = await loadManifest(MANIFEST_PATH); + const whispers = manifest.entries.filter((entry) => entry.id.startsWith('stt-whisper-')); + for (const entry of whispers) { + expect(entry.kind).toBe('file'); + expect(entry.repo).toBe('ggerganov/whisper.cpp'); + expect(entry.revision).toMatch(/^[0-9a-f]{40}$/); + } + }); +}); + +describe('C-393 — STT tier selection (selectSttEntries)', () => { + const entries = [ + makeSttEntry({ + id: 'stt-moonshine-tiny-en-int8', + targetPath: 'stt/sherpa-onnx-moonshine-tiny-en-int8', + }), + makeSttEntry({ + id: 'stt-moonshine-base-en-int8', + targetPath: 'stt/sherpa-onnx-moonshine-base-en-int8', + }), + makeSttEntry({ id: 'stt-whisper-tiny', targetPath: 'stt/whisper-tiny/ggml-tiny.bin' }), + makeSttEntry({ id: 'stt-whisper-base', targetPath: 'stt/whisper-base/ggml-base.bin' }), + makeSttEntry({ id: 'stt-whisper-small', targetPath: 'stt/whisper-small/ggml-small.bin' }), + makeSttEntry({ id: STT_VAD_ENTRY_ID, targetPath: 'stt/silero_vad.onnx' }), + ]; + + it('defaults to the minimal tier + VAD', () => { + const selected = selectSttEntries({ entries }); + const ids = selected.map((entry) => entry.id).sort(); + expect(ids).toEqual( + ['stt-moonshine-tiny-en-int8', 'stt-silero-vad', 'stt-whisper-tiny'].sort(), + ); + }); + + it('selects an accuracy-tier stream model + whisper-small', () => { + const selected = selectSttEntries({ + entries, + streamModel: 'stt/sherpa-onnx-moonshine-base-en-int8', + batchModel: 'stt/whisper-small/ggml-small.bin', + }); + const ids = selected.map((entry) => entry.id).sort(); + expect(ids).toEqual( + ['stt-moonshine-base-en-int8', 'stt-silero-vad', 'stt-whisper-small'].sort(), + ); + }); + + it('treats an empty-string model selection as the shipped default', () => { + // Compose interpolates an unset env var to an empty string; that must + // select the minimal tier, not fetch nothing. + const selected = selectSttEntries({ entries, streamModel: '', batchModel: '' }); + const ids = selected.map((entry) => entry.id).sort(); + expect(ids).toEqual( + ['stt-moonshine-tiny-en-int8', 'stt-silero-vad', 'stt-whisper-tiny'].sort(), + ); + }); + + it('always includes the VAD model', () => { + const selected = selectSttEntries({ + entries, + streamModel: 'stt/sherpa-onnx-moonshine-base-en-int8', + }); + expect(selected.map((entry) => entry.id)).toContain(STT_VAD_ENTRY_ID); + }); +}); + +describe('C-393 — fetcher STT tier scoping (run)', () => { + it('fetches only the selected tier when the stt profile is enabled', async () => { + const dir = await makeTmpDir(); + const manifestPath = join(dir, 'models.manifest.json'); + const entries = [ + makeSttEntry({ + id: 'stt-moonshine-tiny-en-int8', + targetPath: 'stt/sherpa-onnx-moonshine-tiny-en-int8', + }), + makeSttEntry({ + id: 'stt-moonshine-base-en-int8', + targetPath: 'stt/sherpa-onnx-moonshine-base-en-int8', + }), + makeSttEntry({ id: 'stt-whisper-tiny', targetPath: 'stt/whisper-tiny/ggml-tiny.bin' }), + makeSttEntry({ id: 'stt-whisper-base', targetPath: 'stt/whisper-base/ggml-base.bin' }), + makeSttEntry({ id: STT_VAD_ENTRY_ID, targetPath: 'stt/silero_vad.onnx' }), + ]; + await writeFile(manifestPath, JSON.stringify({ schemaVersion: 1, entries })); + const code = await run({ + manifestPath, + modelsDir: dir, + profiles: 'stt', + sttStreamModel: 'stt/sherpa-onnx-moonshine-tiny-en-int8', + sttBatchModel: 'stt/whisper-tiny/ggml-tiny.bin', + }); + expect(code).toBe(0); + await expect( + Bun.file(join(dir, 'stt/sherpa-onnx-moonshine-tiny-en-int8')).exists(), + ).resolves.toBe(true); + await expect(Bun.file(join(dir, 'stt/silero_vad.onnx')).exists()).resolves.toBe(true); + await expect(Bun.file(join(dir, 'stt/whisper-tiny/ggml-tiny.bin')).exists()).resolves.toBe( + true, + ); + // Other tiers must NOT be fetched. + await expect( + Bun.file(join(dir, 'stt/sherpa-onnx-moonshine-base-en-int8')).exists(), + ).resolves.toBe(false); + await expect(Bun.file(join(dir, 'stt/whisper-base/ggml-base.bin')).exists()).resolves.toBe( + false, + ); + await rm(dir, { recursive: true, force: true }); + }); + + it('an explicit entry request bypasses the tier filter', async () => { + const dir = await makeTmpDir(); + const manifestPath = join(dir, 'models.manifest.json'); + const entries = [ + makeSttEntry({ id: 'stt-whisper-base', targetPath: 'stt/whisper-base/ggml-base.bin' }), + makeSttEntry({ id: STT_VAD_ENTRY_ID, targetPath: 'stt/silero_vad.onnx' }), + ]; + await writeFile(manifestPath, JSON.stringify({ schemaVersion: 1, entries })); + const code = await run({ + manifestPath, + modelsDir: dir, + profiles: 'stt', + entryIds: ['stt-whisper-base'], + }); + expect(code).toBe(0); + await expect(Bun.file(join(dir, 'stt/whisper-base/ggml-base.bin')).exists()).resolves.toBe( + true, + ); + // Explicit entries bypass the tier filter — the VAD model is not part + // of the explicit request and must NOT be fetched. + await expect(Bun.file(join(dir, 'stt/silero_vad.onnx')).exists()).resolves.toBe(false); + await rm(dir, { recursive: true, force: true }); + }); +}); diff --git a/apps/backend/local-stack/stack/stt_service.test.ts b/apps/backend/local-stack/stack/stt_service.test.ts new file mode 100644 index 00000000..dad883d2 --- /dev/null +++ b/apps/backend/local-stack/stack/stt_service.test.ts @@ -0,0 +1,389 @@ +/** + * apps/backend/local-stack/stack/stt_service.test.ts + * + * C-393 integration tests against a LIVE STT service (the `stt` compose + * profile or bin/run-native-stt.sh). Every test is skipped when no service + * is reachable, so `bun moon run local-stack:test` stays green without the + * stack; run against a live stack with: + * + * STT_URL=http://127.0.0.1:8087 bun test stack/stt_service.test.ts + * + * Fixture: stack/fixtures/stt_test_utterance.wav — 16 kHz mono 16-bit PCM + * speech, committed (generated from the Kokoro TTS endpoint during C-393 + * implementation). ASR assertions use keyword/edit-distance tolerance, not + * string equality (Test Hooks). + * + * Coverage: AC-1 (partials + final), AC-2 (VAD endpoints), AC-3 (batch + * OpenAI shape), AC-4 (capabilities schema), AC-5 (language limits), AC-6 + * (bad audio format), AC-9 (Origin rejection), AC-10 (health). + */ + +import { describe, expect, it } from 'bun:test'; +import { existsSync, readFileSync } from 'node:fs'; +import { connect as tcpConnect } from 'node:net'; +import { join } from 'node:path'; +import { SttCapabilitiesSchema, type SttServerMessage } from '@aikami/schemas'; +import type { SttCapabilities } from '@aikami/types'; +import { Value } from 'typebox/value'; + +const STT_URL = process.env.STT_URL ?? 'http://127.0.0.1:8087'; +const FIXTURE = join(import.meta.dir, 'fixtures', 'stt_test_utterance.wav'); +const SAMPLE_RATE = 16000; +const BYTES_PER_SECOND = SAMPLE_RATE * 2; + +/** + * True when the STT service is up AND STT_URL was explicitly set — + * otherwise every test is skipped. The explicit env gate keeps the plain + * `bun test stack/*.test.ts` run (check.sh's unit section, moon CI) + * hermetic even when a dev stack happens to be running; run the live suite + * with `STT_URL=http://127.0.0.1:8087 bun test stack/stt_service.test.ts`. + */ +const reachable = process.env.STT_URL + ? await (async (): Promise => { + try { + const response = await fetch(`${STT_URL}/health`, { signal: AbortSignal.timeout(1500) }); + return response.ok; + } catch { + return false; + } + })() + : false; + +const parseWav = (bytes: Uint8Array): { sampleRate: number; channels: number; pcm: Uint8Array } => { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const channels = view.getUint16(22, true); + const sampleRate = view.getUint32(24, true); + const bitsPerSample = view.getUint16(34, true); + if (bitsPerSample !== 16) { + throw new Error(`fixture is not 16-bit: ${bitsPerSample}`); + } + // The wire contract fixes the audio format to 16 kHz mono (AC-6) — a + // fixture that does not match cannot exercise the service correctly. + if (sampleRate !== SAMPLE_RATE) { + throw new Error(`fixture sample rate is not ${SAMPLE_RATE} Hz: ${sampleRate}`); + } + if (channels !== 1) { + throw new Error(`fixture is not mono: ${channels} channels`); + } + // Walk RIFF chunks to find 'data' regardless of the header layout. + let offset = 12; + let pcm: Uint8Array = new Uint8Array(0); + while (offset + 8 <= bytes.length) { + const chunkId = String.fromCharCode(...bytes.subarray(offset, offset + 4)); + const size = view.getUint32(offset + 4, true); + if (chunkId === 'data') { + pcm = bytes.slice(offset + 8, offset + 8 + size); + break; + } + offset += 8 + size + (size % 2); + } + return { sampleRate, channels, pcm }; +}; + +// Parsed lazily and defensively: a malformed fixture must not crash module +// import (the live suite skips when STT_URL is unavailable). Tests that +// need the fixture fail with the existing per-test “fixture missing” path. +let fixture: { + sampleRate: number; + channels: number; + pcm: Uint8Array; + wav: Uint8Array; +} | null = null; +if (existsSync(FIXTURE)) { + try { + const bytes = readFileSync(FIXTURE); + fixture = { ...parseWav(bytes), wav: bytes }; + } catch (error) { + // biome-ignore lint/suspicious/noConsole: visible warning when the live fixture is malformed + console.warn(`C-393 fixture parse failed: ${String(error)}`); + fixture = null; + } +} + +/** + * Streams PCM chunks over a websocket at real-time pace (the server + * validates the byte rate, so the test must deliver ~32000 B/s). + * Returns the parsed server messages until close. + */ +const streamWav = (options: { + pcm: Uint8Array; + leadSilenceMs?: number; + trailSilenceMs?: number; + start?: Record; + /** Wire byte rate to simulate — defaults to 16k mono (32000 B/s). */ + bytesPerSecond?: number; +}): Promise => { + const { + pcm, + leadSilenceMs = 0, + trailSilenceMs = 0, + start, + bytesPerSecond = BYTES_PER_SECOND, + } = options; + return new Promise((resolve, reject) => { + const ws = new WebSocket(`${STT_URL.replace('http', 'ws')}/v1/stream`); + const events: SttServerMessage[] = []; + const sendPcm = (bytes: Uint8Array): void => { + ws.send(bytes); + }; + const chunkMs = 100; + const chunkBytes = Math.floor((bytesPerSecond * chunkMs) / 1000 / 2) * 2; + + const sleep = (ms: number): Promise => + new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); + + ws.onopen = async () => { + try { + ws.send( + JSON.stringify({ + type: 'start', + protocolVersion: 1, + audio: { sampleRate: 16000, channels: 1, encoding: 'pcm_s16le' }, + ...start, + }), + ); + const lead = Math.floor((BYTES_PER_SECOND * leadSilenceMs) / 1000 / 2) * 2; + for (let i = 0; i < lead; i += chunkBytes) { + sendPcm(new Uint8Array(Math.min(chunkBytes, lead - i))); + await sleep(chunkMs * 0.95); + } + for (let i = 0; i < pcm.length; i += chunkBytes) { + sendPcm(pcm.subarray(i, Math.min(i + chunkBytes, pcm.length))); + await sleep(chunkMs * 0.95); + } + const trail = Math.floor((BYTES_PER_SECOND * trailSilenceMs) / 1000 / 2) * 2; + for (let i = 0; i < trail; i += chunkBytes) { + sendPcm(new Uint8Array(Math.min(chunkBytes, trail - i))); + await sleep(chunkMs * 0.95); + } + ws.send(JSON.stringify({ type: 'stop' })); + } catch (error) { + reject(error); + } + }; + ws.onmessage = (event: MessageEvent) => { + if (typeof event.data === 'string') { + try { + events.push(JSON.parse(event.data) as SttServerMessage); + } catch { + // ignore non-JSON frames + } + } + }; + ws.onclose = () => resolve(events); + ws.onerror = (error) => reject(new Error(`websocket error: ${String(error)}`)); + }); +}; + +/** Levenshtein distance for edit-distance tolerance on ASR output. */ +const editDistance = (a: string, b: string): number => { + let prev: number[] = Array.from({ length: b.length + 1 }, (_, j) => j); + for (let i = 1; i <= a.length; i += 1) { + const curr: number[] = [i]; + for (let j = 1; j <= b.length; j += 1) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost); + } + prev = curr; + } + return prev[b.length] ?? 0; +}; + +const normalize = (text: string): string => + text + .toLowerCase() + .replace(/[^a-z0-9 ]/g, '') + .replace(/\s+/g, ' ') + .trim(); + +// The service serializes streaming sessions (STT_MAX_SESSIONS=1 by default, +// overloaded beyond that). Bun may run async it() blocks concurrently, so +// the websocket tests must be serialized or they collide on the semaphore. +let wsTestQueue: Promise = Promise.resolve(); +const withWs = (fn: () => Promise): Promise => { + const run = wsTestQueue.then(fn); + wsTestQueue = run.then( + () => undefined, + () => undefined, + ); + return run; +}; + +describe.skipIf(!reachable)('C-393 STT service (live)', () => { + it('AC-4: GET /v1/capabilities validates against the shared schema', async () => { + const response = await fetch(`${STT_URL}/v1/capabilities`); + expect(response.status).toBe(200); + const caps = (await response.json()) as SttCapabilities; + expect(Value.Check(SttCapabilitiesSchema, caps)).toBe(true); + expect(caps.streaming.engine).toBe('moonshine'); + expect(caps.streaming.languages).toEqual(['en']); + expect(caps.streaming.vad).toBe(true); + expect(caps.streaming.wordTimestamps).toBe(false); + expect(caps.batch.engine).toBe('whisper-cpp'); + expect(caps.audio).toEqual({ sampleRate: 16000, channels: 1, encoding: 'pcm_s16le' }); + expect(caps.protocolVersion).toBe(1); + }); + + it('AC-10: GET /health reports ready when models are loaded', async () => { + const response = await fetch(`${STT_URL}/health`); + expect(response.status).toBe(200); + }); + + it( + 'AC-1 + AC-2: streaming a spoken sentence yields speech-start, partials, one final, speech-end', + () => + withWs(async () => { + if (!fixture) { + throw new Error(`fixture missing: ${FIXTURE}`); + } + const events = await streamWav({ + pcm: fixture.pcm, + leadSilenceMs: 600, + trailSilenceMs: 800, + }); + const types = events.map((event) => event.type); + expect(types).toContain('ready'); + expect(types).toContain('speech-start'); + expect(types).toContain('partial'); + expect(types).toContain('speech-end'); + const finals = events.filter((event) => event.type === 'final'); + expect(finals).toHaveLength(1); + const final = finals[0] as { type: 'final'; text: string }; + expect(final.text.length).toBeGreaterThan(0); + // The fixture is the TTS sentence "Hello world, this is a speech + // recognition test." — compare against the FULL sentence with a + // proportional edit-distance tolerance (ASR output varies). + const expected = normalize('hello world this is a speech recognition test'); + const actual = normalize(final.text); + expect(editDistance(actual, expected)).toBeLessThanOrEqual( + Math.max(3, expected.length / 3), + ); + // Keyword guard alongside edit distance: the transcript must retain + // the salient words, not merely approximate the sentence length. + expect(actual).toContain('hello'); + expect(actual).toContain('recognition'); + }), + 30000, + ); + + it( + 'AC-2: VAD emits speech-start after leading silence and speech-end after trailing silence', + () => + withWs(async () => { + if (!fixture) { + throw new Error(`fixture missing: ${FIXTURE}`); + } + const events = await streamWav({ + pcm: fixture.pcm, + leadSilenceMs: 600, + trailSilenceMs: 800, + }); + const start = events.findIndex((event) => event.type === 'speech-start'); + const end = events.findIndex((event) => event.type === 'speech-end'); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + // speech-start must come after `ready` (never inferred by the client). + expect(events.findIndex((event) => event.type === 'ready')).toBeLessThan(start); + }), + 30000, + ); + + it('AC-5: language=de is reported as unsupported, not transcribed as English', () => + withWs(async () => { + const events = await streamWav({ pcm: new Uint8Array(0), start: { language: 'de' } }); + const error = events.find((event) => event.type === 'error') as + | { type: 'error'; code: string; message: string } + | undefined; + expect(error).toBeDefined(); + expect(error?.code).toBe('unsupported-language'); + expect(error?.message).toContain('transcriptions'); + })); + + it( + 'AC-6: a 44.1 kHz stereo stream fails fast with bad-audio-format', + () => + withWs(async () => { + // Synthesize 44.1k stereo PCM (176400 B/s) and stream at that byte rate. + const seconds = 2; + const totalBytes = 176400 * seconds; + const pcm = new Uint8Array(totalBytes); + for (let i = 0; i < totalBytes; i += 2) { + pcm[i] = 0x01; + pcm[i + 1] = 0x02; + } + const events = await streamWav({ pcm, bytesPerSecond: 176400 }); + const error = events.find((event) => event.type === 'error') as + | { type: 'error'; code: string; message: string } + | undefined; + expect(error).toBeDefined(); + expect(error?.code).toBe('bad-audio-format'); + expect(error?.message).toContain('16 kHz'); + }), + 20000, + ); + + it('AC-9: a cross-origin websocket connection is rejected before audio', async () => { + // Real socket upgrade: connect to /v1/stream with a disallowed Origin + // and assert the handshake is refused with HTTP 403 before any audio + // is accepted (fetch with websocket headers never performs an upgrade). + const url = new URL(STT_URL); + const status = await new Promise((resolve, reject) => { + const socket = tcpConnect(Number(url.port), url.hostname); + socket.setTimeout(5000, () => { + socket.destroy(); + reject(new Error('AC-9 socket timed out')); + }); + socket.on('connect', () => { + socket.write( + [ + 'GET /v1/stream HTTP/1.1', + `Host: ${url.host}`, + 'Upgrade: websocket', + 'Connection: Upgrade', + 'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==', + 'Sec-WebSocket-Version: 13', + 'Origin: http://evil.example.com', + '', + '', + ].join('\r\n'), + ); + }); + let data = ''; + socket.on('data', (chunk: Buffer) => { + data += chunk.toString('utf8'); + if (data.includes('\r\n\r\n')) { + socket.destroy(); + resolve(data.split('\r\n')[0] ?? ''); + } + }); + socket.on('error', reject); + socket.on('close', () => { + if (!data) { + reject(new Error('AC-9 socket closed before a response')); + } + }); + }); + expect(status).toContain('403'); + }); + + it('AC-3: batch endpoint returns an OpenAI-shaped transcription', async () => { + if (!fixture) { + throw new Error(`fixture missing: ${FIXTURE}`); + } + const form = new FormData(); + form.append('file', new Blob([fixture.wav], { type: 'audio/wav' }), 'utterance.wav'); + form.append('model', 'whisper-1'); + form.append('response_format', 'json'); + const response = await fetch(`${STT_URL}/v1/audio/transcriptions`, { + method: 'POST', + body: form, + }); + expect(response.ok).toBe(true); + const body = (await response.json()) as { text?: string }; + expect(typeof body.text).toBe('string'); + expect((body.text ?? '').length).toBeGreaterThan(0); + const expected = normalize('hello world this is a speech recognition test'); + const actual = normalize(body.text ?? ''); + expect(editDistance(actual, expected)).toBeLessThanOrEqual(Math.max(3, expected.length / 3)); + }, 60000); +}); diff --git a/apps/frontend/docs/src/content/docs/guides/run-locally.mdx b/apps/frontend/docs/src/content/docs/guides/run-locally.mdx index a114d7fb..f1ce7f05 100644 --- a/apps/frontend/docs/src/content/docs/guides/run-locally.mdx +++ b/apps/frontend/docs/src/content/docs/guides/run-locally.mdx @@ -5,9 +5,10 @@ description: Get the full local AI stack running on your machine in two commands Aikami's local AI stack runs entirely on your machine — text (llama.cpp), image (stable-diffusion.cpp), voice (sherpa-onnx Kokoro TTS), and -speech-to-text (sherpa-onnx Moonshine) — behind one Compose topology whose -**profiles select modalities** and whose **override files select your -hardware backend**. Two commands is the whole story: +speech-to-text (sherpa-onnx Moonshine streaming + whisper.cpp batch) — +behind one Compose topology whose **profiles select modalities** and whose +**override files select your hardware backend**. Two commands is the whole +story: ```bash bun run stack init # detects your hardware, recommends models, writes .env @@ -35,9 +36,16 @@ init` sets the two variables that matter — or set them by hand: ```bash # .env COMPOSE_PROFILES=text,image,voice,stt -COMPOSE_FILE=compose.yaml:compose.cuda.yaml # see the table +COMPOSE_FILE=compose.yaml:compose.cuda.yaml:compose.stt.yaml # see the table ``` +> **Speech-to-text is opt-in.** A microphone-adjacent service must not +> start unasked, so the shipped defaults start text, image, and voice only. +> Enabling `stt` requires all three of: `stt` in `COMPOSE_PROFILES`, +> `ENABLE_STT=true`, and `compose.stt.yaml` in `COMPOSE_FILE` (the base +> file never publishes the STT port). `stack init` wires all three +> automatically when you include the stt modality. + | Backend | COMPOSE_FILE | When | |---|---|---| | CPU | `compose.yaml:compose.cpu.yaml` | Any machine; slow but works everywhere | @@ -58,13 +66,29 @@ The first start pulls the engine images and downloads the models you enabled | text | `11434` | OpenAI-compatible `/v1` | | image | `8188` | sd-server `/sdcpp/v1` + `/sdapi/v1` | | voice (TTS) | `8089` | `/v1/audio/speech` | -| stt | `8087` | websocket | +| stt (streaming) | `8087` | `WS /v1/stream` (16 kHz mono PCM) | +| stt (batch) | `8087` | `POST /v1/audio/transcriptions` (OpenAI-compatible) | +| stt (introspection) | `8087` | `GET /v1/capabilities`, `GET /health` | | web client (optional) | `5274` | static SPA | These ports match Aikami's allocation table (`development_ports.ts`), so the Tauri desktop app's CSP and defaults keep working with no client change. The engines bind `127.0.0.1` only. +### Speech-to-text wire contract + +Streaming audio is always **16 kHz mono 16-bit PCM** (resampling is the +client's job). A session is: `{"type":"start","protocolVersion":1,"audio":{...}}` +(text frame — `audio` declares the fixed 16 kHz mono 16-bit PCM format) → +binary PCM frames → `{"type":"stop"}`; the server replies with +`ready`, `speech-start`, `partial`*, `final`, `speech-end`, and `error` +events. VAD runs server-side (Silero) so the client never infers +endpointing. Moonshine is English-only — requesting another language returns +`unsupported-language` and points at the batch endpoint, which covers ~99 +languages via whisper.cpp. The full protocol (schemas, error codes, model +tiers, privacy posture) is documented in +[`apps/backend/local-stack/README.md`](https://github.com/BearlySleeping/aikami/blob/dev/apps/backend/local-stack/README.md). + ## Connecting the client The SPA reads engine endpoints from a runtime `config.json` — see @@ -81,11 +105,14 @@ Mac is CPU-only and slow. On Darwin, run the engines natively: cd apps/backend/local-stack ./bin/run-native-llm.sh # llama-server on 11434 ./bin/run-native-tts.sh # sherpa-onnx Kokoro TTS on 8089 -./bin/run-native-stt.sh # sherpa-onnx Moonshine STT on 8087 +./bin/run-native-stt.sh # STT on 8087 — Moonshine streaming + whisper.cpp batch ``` Only the optional web client is containerised (`COMPOSE_PROFILES=web docker -compose up -d`). The native path exposes the same ports as the containers. +compose up -d`). The native path exposes the same ports and the same +protocol as the containers. `run-native-stt.sh` needs `pip install +sherpa-onnx` on the host; batch transcription additionally needs the +whisper.cpp `whisper-server` binary (the script header documents the build). ## Models and licences diff --git a/packages/shared/schemas/src/index.ts b/packages/shared/schemas/src/index.ts index 78881cd7..30754807 100644 --- a/packages/shared/schemas/src/index.ts +++ b/packages/shared/schemas/src/index.ts @@ -57,6 +57,7 @@ export * from './lib/local_ai/hardware_profile.ts'; export * from './lib/local_ai/model_manifest.ts'; export * from './lib/local_ai/stack_backend.ts'; export * from './lib/local_ai/stack_plan.ts'; +export * from './lib/local_ai/stt.ts'; export * from './lib/logging/index.ts'; export * from './lib/media/audio_track_catalog.ts'; export * from './lib/media/image_engine.ts'; diff --git a/packages/shared/schemas/src/lib/local_ai/stt.test.ts b/packages/shared/schemas/src/lib/local_ai/stt.test.ts new file mode 100644 index 00000000..071b0dd2 --- /dev/null +++ b/packages/shared/schemas/src/lib/local_ai/stt.test.ts @@ -0,0 +1,181 @@ +// packages/shared/schemas/src/lib/local_ai/stt.test.ts +// +// Schema validation tests for the C-393 STT wire protocol: capabilities +// (AC-4 introspectability), client/server message unions (AC-1/AC-2/AC-5/ +// AC-6), error codes, and the fixed audio format. The service and C-359's +// client share these schemas so the wire contract cannot drift. + +import { describe, expect, test } from 'bun:test'; +import { Value } from 'typebox/value'; +import { + SttCapabilitiesSchema, + SttClientMessageSchema, + SttErrorCodeSchema, + SttServerMessageSchema, +} from './stt.ts'; + +const check = (schema: unknown, value: unknown): boolean => Value.Check(schema as never, value); + +const VALID_START = { + type: 'start', + protocolVersion: 1, + audio: { sampleRate: 16000, channels: 1, encoding: 'pcm_s16le' }, +} as const; + +const VALID_CAPABILITIES = { + streaming: { + available: true, + engine: 'moonshine', + model: 'sherpa-onnx-moonshine-tiny-en-int8', + languages: ['en'], + vad: true, + wordTimestamps: false, + }, + batch: { + available: true, + engine: 'whisper-cpp', + model: 'ggml-tiny.bin', + languages: ['en', 'de', 'fr'], + }, + audio: { sampleRate: 16000, channels: 1, encoding: 'pcm_s16le' }, + protocolVersion: 1, +}; + +describe('SttCapabilitiesSchema (AC-4)', () => { + test('accepts the full capabilities document', () => { + expect(check(SttCapabilitiesSchema, VALID_CAPABILITIES)).toBe(true); + }); + + test('rejects a future engine value before the schema is extended', () => { + expect( + check(SttCapabilitiesSchema, { + ...VALID_CAPABILITIES, + streaming: { ...VALID_CAPABILITIES.streaming, engine: 'crisperwhisper' }, + }), + ).toBe(false); + }); + + test('accepts wordTimestamps true — the seam for a licensed provider (AC contract)', () => { + // The field exists so a future provider can report true without a + // protocol change; the SHIPPED engines report false (service-side + // behaviour, asserted by the smoke test, not this schema). + expect( + check(SttCapabilitiesSchema, { + ...VALID_CAPABILITIES, + streaming: { ...VALID_CAPABILITIES.streaming, wordTimestamps: true }, + }), + ).toBe(true); + }); + + test('rejects capabilities missing the vad flag', () => { + const { vad: _vad, ...withoutVad } = VALID_CAPABILITIES.streaming; + expect( + check(SttCapabilitiesSchema, { + ...VALID_CAPABILITIES, + streaming: withoutVad, + }), + ).toBe(false); + }); + + test('rejects a non-16k audio format', () => { + expect( + check(SttCapabilitiesSchema, { + ...VALID_CAPABILITIES, + audio: { sampleRate: 44100, channels: 2, encoding: 'pcm_s16le' }, + }), + ).toBe(false); + }); + + test('rejects an unsupported protocolVersion', () => { + expect(check(SttCapabilitiesSchema, { ...VALID_CAPABILITIES, protocolVersion: 2 })).toBe(false); + }); +}); + +describe('SttClientMessageSchema', () => { + test('accepts a start message', () => { + expect(check(SttClientMessageSchema, { ...VALID_START })).toBe(true); + }); + + test('accepts a start message with language', () => { + expect(check(SttClientMessageSchema, { ...VALID_START, language: 'de' })).toBe(true); + }); + + test('accepts a stop message', () => { + expect(check(SttClientMessageSchema, { type: 'stop' })).toBe(true); + }); + + test('rejects an unknown message type', () => { + expect(check(SttClientMessageSchema, { type: 'pause' })).toBe(false); + }); + + test('rejects a start without the required audio format', () => { + expect(check(SttClientMessageSchema, { type: 'start', protocolVersion: 1 })).toBe(false); + }); + + test('rejects a start declaring a non-16k audio format', () => { + expect( + check(SttClientMessageSchema, { + ...VALID_START, + audio: { sampleRate: 44100, channels: 2, encoding: 'pcm_s16le' }, + }), + ).toBe(false); + }); + + test('rejects a start without protocolVersion', () => { + expect(check(SttClientMessageSchema, { type: 'start' })).toBe(false); + }); + + test('rejects protocolVersion 2', () => { + expect(check(SttClientMessageSchema, { ...VALID_START, protocolVersion: 2 })).toBe(false); + }); +}); + +describe('SttServerMessageSchema', () => { + const messages = [ + { type: 'ready', capabilities: VALID_CAPABILITIES }, + { type: 'speech-start', atMs: 120 }, + { type: 'partial', text: 'hello', atMs: 1400 }, + { type: 'final', text: 'hello world', startMs: 120, endMs: 2300 }, + { type: 'speech-end', atMs: 2350 }, + { type: 'error', code: 'unsupported-language', message: 'batch is multilingual' }, + ]; + + for (const message of messages) { + test(`accepts ${message.type}`, () => { + expect(check(SttServerMessageSchema, message)).toBe(true); + }); + } + + test('rejects a final without endMs', () => { + expect(check(SttServerMessageSchema, { type: 'final', text: 'x', startMs: 0 })).toBe(false); + }); + + test('rejects an unknown server event', () => { + expect(check(SttServerMessageSchema, { type: 'error', code: 'oops', message: 'x' })).toBe( + false, + ); + }); + + test('rejects a bare string frame', () => { + expect(check(SttServerMessageSchema, 'speech-start')).toBe(false); + }); +}); + +describe('SttErrorCodeSchema', () => { + test('accepts every documented code', () => { + for (const code of [ + 'model-not-loaded', + 'unsupported-language', + 'bad-audio-format', + 'protocol-version-mismatch', + 'overloaded', + 'internal', + ]) { + expect(check(SttErrorCodeSchema, code)).toBe(true); + } + }); + + test('rejects an unknown code', () => { + expect(check(SttErrorCodeSchema, 'no-such-code')).toBe(false); + }); +}); diff --git a/packages/shared/schemas/src/lib/local_ai/stt.ts b/packages/shared/schemas/src/lib/local_ai/stt.ts new file mode 100644 index 00000000..1203e2c0 --- /dev/null +++ b/packages/shared/schemas/src/lib/local_ai/stt.ts @@ -0,0 +1,166 @@ +// packages/shared/schemas/src/lib/local_ai/stt.ts +// +// C-393 STT wire protocol — the single source of truth shared by the +// streaming service (apps/backend/local-stack/docker/voice/stt_server.py's +// documented protocol) and C-359's client. The service serialises JSON +// against these shapes; the client validates and derives types from them, +// so the wire contract cannot drift between the two. +// +// Wire contract summary: +// - Audio is always 16 kHz mono 16-bit PCM (`pcm_s16le`). Resampling is +// the client's job; the server rejects anything that does not match +// (AC-6). +// - `WS {stt}/v1/stream` — client sends `start` (JSON text frame, +// declaring the audio format it will stream), then binary frames of +// raw PCM, then `stop`; the server replies with `ready`, +// `speech-start`, `partial`*, `final`, `speech-end`, and `error` +// events (AC-1/AC-2). +// - `POST {stt}/v1/audio/transcriptions` — OpenAI-compatible batch +// endpoint served by whisper.cpp (AC-3). +// - `GET {stt}/v1/capabilities` — introspection (AC-4). +// - `GET {stt}/v1/health` — readiness, model-presence aware (AC-10). +import Type from 'typebox'; + +/** + * Which engine serves the streaming protocol. Env-selected + * (`STT_STREAM_ENGINE`) and extensible: a future licensed CrisperWhisper + * provider becomes a new literal plus a container, not a protocol change. + */ +export const SttStreamEngineSchema = Type.Union([Type.Literal('moonshine')]); +export type SttStreamEngine = Type.Static; + +/** Which engine serves the batch protocol (`STT_BATCH_ENGINE`). */ +export const SttBatchEngineSchema = Type.Union([Type.Literal('whisper-cpp')]); +export type SttBatchEngine = Type.Static; + +/** The only audio format the service accepts: 16 kHz mono 16-bit PCM. */ +export const SttAudioFormatSchema = Type.Object({ + sampleRate: Type.Literal(16000), + channels: Type.Literal(1), + encoding: Type.Literal('pcm_s16le'), +}); +export type SttAudioFormat = Type.Static; + +/** Streaming capabilities (AC-4). Moonshine is English-only (AC-5). */ +export const SttStreamingCapabilitiesSchema = Type.Object({ + available: Type.Boolean(), + engine: SttStreamEngineSchema, + model: Type.String(), + languages: Type.Array(Type.String()), + vad: Type.Boolean(), + wordTimestamps: Type.Boolean(), +}); +export type SttStreamingCapabilities = Type.Static; + +/** Batch capabilities (AC-4). whisper.cpp covers ~99 languages. */ +export const SttBatchCapabilitiesSchema = Type.Object({ + available: Type.Boolean(), + engine: SttBatchEngineSchema, + model: Type.String(), + languages: Type.Array(Type.String()), +}); +export type SttBatchCapabilities = Type.Static; + +/** `GET /v1/capabilities` response (AC-4). */ +export const SttCapabilitiesSchema = Type.Object({ + streaming: SttStreamingCapabilitiesSchema, + batch: SttBatchCapabilitiesSchema, + audio: SttAudioFormatSchema, + protocolVersion: Type.Literal(1), +}); +export type SttCapabilities = Type.Static; + +/** Server error codes — see SttServerErrorMessageSchema. */ +export const SttErrorCodeSchema = Type.Union([ + Type.Literal('model-not-loaded'), + Type.Literal('unsupported-language'), + Type.Literal('bad-audio-format'), + Type.Literal('protocol-version-mismatch'), + Type.Literal('overloaded'), + Type.Literal('internal'), +]); +export type SttErrorCode = Type.Static; + +/** Client → server: begin a streaming session. + * + * `audio` is required — the client declares the format it will stream so + * the server can reject a mismatch before accepting any audio (AC-6). The + * only accepted value is the fixed 16 kHz mono 16-bit PCM format. + */ +export const SttClientStartMessageSchema = Type.Object({ + type: Type.Literal('start'), + audio: SttAudioFormatSchema, + language: Type.Optional(Type.String()), + protocolVersion: Type.Literal(1), +}); +export type SttClientStartMessage = Type.Static; + +/** Client → server: end the session and request the final transcript. */ +export const SttClientStopMessageSchema = Type.Object({ + type: Type.Literal('stop'), +}); +export type SttClientStopMessage = Type.Static; + +/** Client → server messages (JSON text frames; audio is binary frames). */ +export const SttClientMessageSchema = Type.Union([ + SttClientStartMessageSchema, + SttClientStopMessageSchema, +]); +export type SttClientMessage = Type.Static; + +/** Server → client: session accepted; capabilities mirror `GET /v1/capabilities`. */ +export const SttServerReadyMessageSchema = Type.Object({ + type: Type.Literal('ready'), + capabilities: SttCapabilitiesSchema, +}); +export type SttServerReadyMessage = Type.Static; + +/** Server → client: VAD detected speech onset (AC-2). */ +export const SttServerSpeechStartMessageSchema = Type.Object({ + type: Type.Literal('speech-start'), + atMs: Type.Number(), +}); +export type SttServerSpeechStartMessage = Type.Static; + +/** Server → client: incremental hypothesis while the user is speaking (AC-1). */ +export const SttServerPartialMessageSchema = Type.Object({ + type: Type.Literal('partial'), + text: Type.String(), + atMs: Type.Number(), +}); +export type SttServerPartialMessage = Type.Static; + +/** Server → client: final transcript for the completed utterance (AC-1). */ +export const SttServerFinalMessageSchema = Type.Object({ + type: Type.Literal('final'), + text: Type.String(), + startMs: Type.Number(), + endMs: Type.Number(), +}); +export type SttServerFinalMessage = Type.Static; + +/** Server → client: VAD detected speech end (AC-2). */ +export const SttServerSpeechEndMessageSchema = Type.Object({ + type: Type.Literal('speech-end'), + atMs: Type.Number(), +}); +export type SttServerSpeechEndMessage = Type.Static; + +/** Server → client: a session failed; the socket closes after this frame. */ +export const SttServerErrorMessageSchema = Type.Object({ + type: Type.Literal('error'), + code: SttErrorCodeSchema, + message: Type.String(), +}); +export type SttServerErrorMessage = Type.Static; + +/** Server → client messages. */ +export const SttServerMessageSchema = Type.Union([ + SttServerReadyMessageSchema, + SttServerSpeechStartMessageSchema, + SttServerPartialMessageSchema, + SttServerFinalMessageSchema, + SttServerSpeechEndMessageSchema, + SttServerErrorMessageSchema, +]); +export type SttServerMessage = Type.Static; diff --git a/packages/shared/types/src/index.ts b/packages/shared/types/src/index.ts index 51334ae9..9df17c16 100644 --- a/packages/shared/types/src/index.ts +++ b/packages/shared/types/src/index.ts @@ -70,6 +70,7 @@ export * from './lib/local_ai/hardware_profile.ts'; export * from './lib/local_ai/model_manifest.ts'; export * from './lib/local_ai/stack_backend.ts'; export * from './lib/local_ai/stack_plan.ts'; +export * from './lib/local_ai/stt.ts'; export * from './lib/media/image_engine.ts'; export * from './lib/media/image_style_profile.ts'; export * from './lib/media/music.ts'; diff --git a/packages/shared/types/src/lib/local_ai/stt.ts b/packages/shared/types/src/lib/local_ai/stt.ts new file mode 100644 index 00000000..8393808b --- /dev/null +++ b/packages/shared/types/src/lib/local_ai/stt.ts @@ -0,0 +1,45 @@ +// packages/shared/types/src/lib/local_ai/stt.ts +// +// C-393 STT wire-protocol types, derived from the TypeBox schemas in +// @aikami/schemas — the schemas are the single source of truth; these +// re-exports keep the protocol types on the @aikami/types surface that +// C-359's client imports. + +import type { + SttAudioFormatSchema, + SttBatchCapabilitiesSchema, + SttBatchEngineSchema, + SttCapabilitiesSchema, + SttClientMessageSchema, + SttClientStartMessageSchema, + SttClientStopMessageSchema, + SttErrorCodeSchema, + SttServerErrorMessageSchema, + SttServerFinalMessageSchema, + SttServerMessageSchema, + SttServerPartialMessageSchema, + SttServerReadyMessageSchema, + SttServerSpeechEndMessageSchema, + SttServerSpeechStartMessageSchema, + SttStreamEngineSchema, + SttStreamingCapabilitiesSchema, +} from '@aikami/schemas'; +import type { Static } from 'typebox'; + +export type SttStreamEngine = Static; +export type SttBatchEngine = Static; +export type SttAudioFormat = Static; +export type SttStreamingCapabilities = Static; +export type SttBatchCapabilities = Static; +export type SttCapabilities = Static; +export type SttErrorCode = Static; +export type SttClientStartMessage = Static; +export type SttClientStopMessage = Static; +export type SttClientMessage = Static; +export type SttServerReadyMessage = Static; +export type SttServerSpeechStartMessage = Static; +export type SttServerPartialMessage = Static; +export type SttServerFinalMessage = Static; +export type SttServerSpeechEndMessage = Static; +export type SttServerErrorMessage = Static; +export type SttServerMessage = Static;