Backport: feat(provider/google): support Gemini Interactions video output

.changeset/google-interactions-video-output-v6.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/google': patch
+---
+
+Backport Gemini Interactions video output parsing and per-modality output token breakdown for AI SDK v6.

packages/google/src/interactions/build-google-interactions-stream-transform.test.ts

@@ -0,0 +1,134 @@
+import {
+  convertArrayToReadableStream,
+  convertReadableStreamToArray,
+} from '@ai-sdk/provider-utils/test';
+import type { ParseResult } from '@ai-sdk/provider-utils';
+import { describe, expect, it } from 'vitest';
+import { buildGoogleInteractionsStreamTransform } from './build-google-interactions-stream-transform';
+import type { GoogleInteractionsEvent } from './google-interactions-api';
+
+function runTransform(events: Array<GoogleInteractionsEvent>) {
+  const transform = buildGoogleInteractionsStreamTransform({
+    warnings: [],
+    generateId: () => 'test-id',
+  });
+  const input = convertArrayToReadableStream(
+    events.map(value => ({ success: true, value, rawValue: value })),
+  ) as ReadableStream<ParseResult<GoogleInteractionsEvent>>;
+  return convertReadableStreamToArray(input.pipeThrough(transform));
+}
+
+describe('buildGoogleInteractionsStreamTransform — video deltas', () => {
+  it('emits a file stream part for a video delta carrying inline data', async () => {
+    const parts = await runTransform([
+      {
+        event_type: 'interaction.created',
+        interaction: { id: 'v1_video-stream', status: 'in_progress' },
+      },
+      {
+        event_type: 'step.start',
+        index: 0,
+        step: { type: 'model_output' },
+      },
+      {
+        event_type: 'step.delta',
+        index: 0,
+        delta: {
+          type: 'video',
+          data: 'AAAAIGZ0eXBpc29t',
+          mime_type: 'video/mp4',
+        },
+      },
+      {
+        event_type: 'step.stop',
+        index: 0,
+      },
+      {
+        event_type: 'interaction.completed',
+        interaction: { id: 'v1_video-stream', status: 'completed' },
+      },
+    ] as Array<GoogleInteractionsEvent>);
+
+    const fileParts = parts.filter(p => p.type === 'file');
+    expect(fileParts).toEqual([
+      expect.objectContaining({
+        type: 'file',
+        mediaType: 'video/mp4',
+        data: 'AAAAIGZ0eXBpc29t',
+      }),
+    ]);
+  });
+
+  it('emits a file stream part with videoUri metadata for a uri-only video delta', async () => {
+    const parts = await runTransform([
+      {
+        event_type: 'interaction.created',
+        interaction: { id: 'v1_video-stream', status: 'in_progress' },
+      },
+      {
+        event_type: 'step.start',
+        index: 0,
+        step: { type: 'model_output' },
+      },
+      {
+        event_type: 'step.delta',
+        index: 0,
+        delta: { type: 'video', uri: 'https://example.test/clip.mp4' },
+      },
+      {
+        event_type: 'step.stop',
+        index: 0,
+      },
+      {
+        event_type: 'interaction.completed',
+        interaction: { id: 'v1_video-stream', status: 'completed' },
+      },
+    ] as Array<GoogleInteractionsEvent>);
+
+    const fileParts = parts.filter(p => p.type === 'file');
+    expect(fileParts).toEqual([
+      expect.objectContaining({
+        type: 'file',
+        mediaType: 'video/mp4',
+        data: '',
+        providerMetadata: {
+          google: {
+            interactionId: 'v1_video-stream',
+            videoUri: 'https://example.test/clip.mp4',
+          },
+        },
+      }),
+    ]);
+  });
+});
+
+describe('buildGoogleInteractionsStreamTransform — usage modality', () => {
+  it('surfaces output_tokens_by_modality on the finish part providerMetadata', async () => {
+    const parts = await runTransform([
+      {
+        event_type: 'interaction.created',
+        interaction: { id: 'v1_usage', status: 'in_progress' },
+      },
+      {
+        event_type: 'interaction.completed',
+        interaction: {
+          id: 'v1_usage',
+          status: 'completed',
+          usage: {
+            total_output_tokens: 57939,
+            output_tokens_by_modality: [
+              { modality: 'video', tokens: 57920 },
+              { modality: 'text', tokens: 19 },
+            ],
+          },
+        },
+      },
+    ] as Array<GoogleInteractionsEvent>);
+
+    const finish = parts.find(p => p.type === 'finish');
+    expect(finish?.providerMetadata?.google?.outputTokensByModality).toEqual({
+      video: 57920,
+      text: 19,
+    });
+  });
+});

packages/google/src/interactions/build-google-interactions-stream-transform.ts

@@ -11,7 +11,10 @@ import type {
   GoogleInteractionsEvent,
   GoogleInteractionsUsage,
 } from './google-interactions-api';
-import { convertGoogleInteractionsUsage } from './convert-google-interactions-usage';
+import {
+  convertGoogleInteractionsUsage,
+  getGoogleInteractionsOutputTokensByModality,
+} from './convert-google-interactions-usage';
 import {
   annotationsToSources,
   builtinToolResultToSources,
@@ -490,6 +493,47 @@ export function buildGoogleInteractionsStreamTransform({
             break;
           }
 
+          /*
+           * `video` deltas inside `model_output` carry the full payload in a
+           * single chunk (no per-byte streaming), mirroring `image`. Emit the
+           * `file` part as soon as the delta arrives so it surfaces regardless
+           * of whether a text block is currently open at the same index.
+           */
+          if (
+            dtype === 'video' &&
+            (open.kind === 'pending_model_output' || open.kind === 'text')
+          ) {
+            const videoDelta = event.delta as
+              | { data?: string; mime_type?: string; uri?: string }
+              | undefined;
+            const google: Record<string, string> = {};
+            if (interactionId != null) google.interactionId = interactionId;
+            const providerMetadata =
+              Object.keys(google).length > 0 ? { google } : undefined;
+            if (videoDelta?.data != null && videoDelta.data.length > 0) {
+              controller.enqueue({
+                type: 'file',
+                mediaType: videoDelta.mime_type ?? 'video/mp4',
+                data: videoDelta.data,
+                ...(providerMetadata ? { providerMetadata } : {}),
+              });
+            } else if (videoDelta?.uri != null && videoDelta.uri.length > 0) {
+              const uriProviderMetadata = {
+                google: {
+                  ...(interactionId != null ? { interactionId } : {}),
+                  videoUri: videoDelta.uri,
+                },
+              };
+              controller.enqueue({
+                type: 'file',
+                mediaType: videoDelta.mime_type ?? 'video/mp4',
+                data: '',
+                providerMetadata: uriProviderMetadata,
+              });
+            }
+            break;
+          }
+
           const delta = event.delta as
             | {
                 type?: string;
@@ -824,10 +868,14 @@ export function buildGoogleInteractionsStreamTransform({
         raw: finishStatus,
       };
 
+      const outputTokensByModality =
+        getGoogleInteractionsOutputTokensByModality(usage);
+
       const providerMetadata: SharedV3ProviderMetadata = {
         google: {
           ...(interactionId != null ? { interactionId } : {}),
           ...(serviceTier != null ? { serviceTier } : {}),
+          ...(outputTokensByModality != null ? { outputTokensByModality } : {}),
         },
       };
 

packages/google/src/interactions/convert-google-interactions-usage.ts

@@ -45,3 +45,25 @@ export function convertGoogleInteractionsUsage(
     raw: usage as unknown as JSONObject,
   };
 }
+
+/**
+ * Extracts the per-modality output token breakdown from an Interactions usage
+ * record (e.g. `{ video: 57920, text: 12 }`).
+ */
+export function getGoogleInteractionsOutputTokensByModality(
+  usage: GoogleInteractionsUsage | undefined | null,
+): Record<string, number> | undefined {
+  const byModality = usage?.output_tokens_by_modality;
+  if (byModality == null) {
+    return undefined;
+  }
+
+  const result: Record<string, number> = {};
+  for (const entry of byModality) {
+    if (entry?.modality != null && entry.tokens != null) {
+      result[entry.modality] = entry.tokens;
+    }
+  }
+
+  return Object.keys(result).length > 0 ? result : undefined;
+}

packages/google/src/interactions/google-interactions-api.ts

@@ -140,9 +140,19 @@ const contentBlockSchema = () => {
     })
     .loose();
 
+  const videoContent = z
+    .object({
+      type: z.literal('video'),
+      data: z.string().nullish(),
+      mime_type: z.string().nullish(),
+      uri: z.string().nullish(),
+    })
+    .loose();
+
   return z.union([
     textContent,
     imageContent,
+    videoContent,
     z.object({ type: z.string() }).loose(),
   ]);
 };
@@ -384,6 +394,19 @@ export const googleInteractionsEventSchema = lazySchema(() =>
         })
         .loose();
 
+      /*
+       * `video` deltas carry the entire payload per delta (`data` base64 +
+       * `mime_type`, or `uri`) — there is no per-byte streaming.
+       */
+      const stepDeltaVideo = z
+        .object({
+          type: z.literal('video'),
+          data: z.string().nullish(),
+          mime_type: z.string().nullish(),
+          uri: z.string().nullish(),
+        })
+        .loose();
+
       /*
        * Built-in tool call/result step deltas mirror the shape of their step
        * counterparts (full payload per delta — there is no per-token
@@ -419,6 +442,7 @@ export const googleInteractionsEventSchema = lazySchema(() =>
       const stepDeltaUnion = z.union([
         stepDeltaText,
         stepDeltaImage,
+        stepDeltaVideo,
         stepDeltaThoughtSummary,
         stepDeltaThoughtSignature,
         stepDeltaArgumentsDelta,

packages/google/src/interactions/google-interactions-language-model.test.ts

@@ -451,6 +451,46 @@ describe('GoogleInteractionsLanguageModel.doGenerate', () => {
     });
   });
 
+  describe('output token modality breakdown', () => {
+    it('surfaces output_tokens_by_modality on providerMetadata.google.outputTokensByModality', async () => {
+      const fixture = JSON.parse(
+        fs.readFileSync('src/interactions/__fixtures__/basic.json', 'utf8'),
+      );
+      server.urls[TEST_URL].response = {
+        type: 'json-value',
+        body: {
+          ...fixture,
+          usage: {
+            ...fixture.usage,
+            output_tokens_by_modality: [
+              { modality: 'video', tokens: 57920 },
+              { modality: 'text', tokens: 19 },
+            ],
+          },
+        },
+      };
+
+      const { providerMetadata } = await model.doGenerate({
+        prompt: TEST_PROMPT,
+      });
+
+      expect(providerMetadata?.google?.outputTokensByModality).toEqual({
+        video: 57920,
+        text: 19,
+      });
+    });
+
+    it('omits outputTokensByModality when the response has no breakdown', async () => {
+      prepareJsonFixtureResponse('basic');
+
+      const { providerMetadata } = await model.doGenerate({
+        prompt: TEST_PROMPT,
+      });
+
+      expect(providerMetadata?.google?.outputTokensByModality).toBeUndefined();
+    });
+  });
+
   describe('stateful chaining (previousInteractionId, store, thinking)', () => {
     beforeEach(() => {
       prepareJsonFixtureResponse('basic');

packages/google/src/interactions/google-interactions-language-model.ts

@@ -20,7 +20,10 @@ import {
 } from '@ai-sdk/provider-utils';
 import { googleFailedResponseHandler } from '../google-error';
 import { buildGoogleInteractionsStreamTransform } from './build-google-interactions-stream-transform';
-import { convertGoogleInteractionsUsage } from './convert-google-interactions-usage';
+import {
+  convertGoogleInteractionsUsage,
+  getGoogleInteractionsOutputTokensByModality,
+} from './convert-google-interactions-usage';
 import { convertToGoogleInteractionsInput } from './convert-to-google-interactions-input';
 import {
   googleInteractionsEventSchema,
@@ -519,10 +522,15 @@ export class GoogleInteractionsLanguageModel implements LanguageModelV3 {
      * `response.id` is omitted when `store: false` (fully stateless mode), so
      * `interactionId` is only surfaced when the API actually returned one.
      */
+    const outputTokensByModality = getGoogleInteractionsOutputTokensByModality(
+      response.usage,
+    );
+
     const providerMetadata: SharedV3ProviderMetadata = {
       google: {
         ...(interactionId != null ? { interactionId } : {}),
         ...(serviceTier != null ? { serviceTier } : {}),
+        ...(outputTokensByModality != null ? { outputTokensByModality } : {}),
       },
     };
 

packages/google/src/interactions/google-interactions-provider-metadata.ts

@@ -15,6 +15,14 @@ export type GoogleInteractionsProviderMetadata = {
    */
   serviceTier?: string;
 
+  /**
+   * Output token counts keyed by modality (e.g. `{ video: 57920 }`), sourced
+   * from the Interactions API `output_tokens_by_modality`. Present only when
+   * the response reports a breakdown. Preview surface for per-modality billing;
+   * may be promoted to a first-class usage field later.
+   */
+  outputTokensByModality?: Record<string, number>;
+
   /**
    * Per-block signature hash for backend validation. Set by the SDK on output
    * reasoning / tool-call parts and round-tripped on input parts.