diff --git a/.changeset/elevenlabs-stt-previous-text.md b/.changeset/elevenlabs-stt-previous-text.md new file mode 100644 index 000000000..8346d7029 --- /dev/null +++ b/.changeset/elevenlabs-stt-previous-text.md @@ -0,0 +1,5 @@ +--- +'@livekit/agents-plugin-elevenlabs': minor +--- + +Support previous text context for ElevenLabs realtime STT. diff --git a/plugins/elevenlabs/src/stt.test.ts b/plugins/elevenlabs/src/stt.test.ts index aae58a8b1..868d40fd1 100644 --- a/plugins/elevenlabs/src/stt.test.ts +++ b/plugins/elevenlabs/src/stt.test.ts @@ -1,13 +1,13 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { mergeFrames, stt as sttLib } from '@livekit/agents'; +import { log, mergeFrames, stt as sttLib } from '@livekit/agents'; import { AudioFrame, AudioResampler } from '@livekit/rtc-node'; import { once } from 'node:events'; import { readFileSync } from 'node:fs'; import { type RequestListener, type Server, createServer } from 'node:http'; import type { AddressInfo } from 'node:net'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { WebSocketServer } from 'ws'; import { STT } from './stt.js'; @@ -153,6 +153,59 @@ describe('ElevenLabs STT', () => { expect(stt.capabilities.alignedTranscript).toBe('word'); }); + it('keeps previousText for realtime model', async () => { + const { wss, baseURL } = await startWebSocketServer(); + const receivedMessages: Record[] = []; + + wss.on('connection', (ws) => { + ws.on('message', (raw) => { + receivedMessages.push(JSON.parse(raw.toString()) as Record); + }); + }); + + try { + const eleven = new STT({ + apiKey: 'test-key', + baseURL, + model: 'scribe_v2_realtime', + previousText: 'prior context', + }); + const stream = eleven.stream(); + + await waitUntil(() => receivedMessages.length > 0); + stream.close(); + + expect(receivedMessages[0]).toMatchObject({ + message_type: 'input_audio_chunk', + audio_base_64: '', + commit: false, + sample_rate: 16000, + previous_text: 'prior context', + }); + } finally { + await closeWebSocketServer(wss); + } + }); + + it('ignores previousText for non-realtime model', () => { + const warn = vi.spyOn(log(), 'warn').mockImplementation(() => undefined); + + try { + const eleven = new STT({ + apiKey: 'test-key', + model: 'scribe_v2', + previousText: 'prior context', + }); + + expect(eleven.model).toBe('scribe_v2'); + expect(warn).toHaveBeenCalledWith( + '`previousText` is only supported for Scribe v2 realtime model and will be ignored', + ); + } finally { + warn.mockRestore(); + } + }); + it('sends batch recognition form fields and maps word metadata', async () => { let request: | { diff --git a/plugins/elevenlabs/src/stt.ts b/plugins/elevenlabs/src/stt.ts index 950a42118..5bad3ecdb 100644 --- a/plugins/elevenlabs/src/stt.ts +++ b/plugins/elevenlabs/src/stt.ts @@ -62,6 +62,11 @@ export interface STTOptions { modelId?: ElevenLabsSTTModels | string; keyterms?: string[]; noVerbatim?: boolean; + /** + * Preceding text context sent once on the first realtime audio chunk to improve transcription + * accuracy. Only supported for Scribe v2 realtime. + */ + previousText?: string; } interface ResolvedSTTOptions { @@ -75,6 +80,7 @@ interface ResolvedSTTOptions { serverVad?: VADOptions | null; keyterms?: string[]; noVerbatim: boolean; + previousText: string | null; } export interface STTRecognizeOptions { @@ -224,6 +230,14 @@ export class STT extends stt.STT { log().warn('Server-side VAD is only supported for Scribe v2 realtime model'); } + let previousText = opts.previousText ?? null; + if (!useRealtime && previousText !== null) { + log().warn( + '`previousText` is only supported for Scribe v2 realtime model and will be ignored', + ); + previousText = null; + } + const includeTimestamps = opts.includeTimestamps ?? false; super({ streaming: useRealtime, @@ -249,6 +263,7 @@ export class STT extends stt.STT { modelId, keyterms: opts.keyterms, noVerbatim: opts.noVerbatim ?? false, + previousText, }; this.#session = opts.httpSession ?? {}; } @@ -546,6 +561,18 @@ export class SpeechStream extends stt.SpeechStream { try { ws = await this.#connectWs(); + if (this.#opts.previousText) { + // Must be the first input_audio_chunk on the connection. + ws.send( + JSON.stringify({ + message_type: 'input_audio_chunk', + audio_base_64: '', + commit: false, + sample_rate: this.#opts.sampleRate, + previous_text: this.#opts.previousText, + }), + ); + } const keepaliveTask = Task.from(async (controller) => { while (!controller.signal.aborted) {