Skip to content

chore(bidi): implement screencasting using browsingContext.captureScreenshot #41569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Skn0tt merged 1 commit into from
Jul 7, 2026
Merged

chore(bidi): implement screencasting using browsingContext.captureScreenshot #41569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions packages/playwright-core/src/server/bidi/bidiPage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { BidiNetworkManager } from './bidiNetworkManager';
import { BidiPDF } from './bidiPdf';
import * as bidi from './third_party/bidiProtocol';
import { nullProgress } from '../progress';
import { startAutomaticVideoRecording } from '../videoRecorder';

import * as frames from '../frames';
import * as network from '../network';
Expand Down Expand Up @@ -56,6 +57,8 @@ export class BidiPage implements PageDelegate {
private _initScriptIds = new Map<InitScript, string>();
private readonly _fragmentNavigations = new Set<string>();
private readonly _failedNavigations = new Map<string, string>();
private _screencastTimer: NodeJS.Timeout | undefined;
private _waitingForScreenshot = false;

constructor(browserContext: BidiBrowserContext, bidiSession: BidiSession, opener: BidiPage | null) {
this._session = bidiSession;
Expand Down Expand Up @@ -103,6 +106,7 @@ export class BidiPage implements PageDelegate {
// If the page is created by the Playwright client's call, some initialization
// may be pending. Wait for it to complete before reporting the page as new.
]);
startAutomaticVideoRecording(this._page);
}

didClose() {
Expand Down Expand Up @@ -576,9 +580,53 @@ export class BidiPage implements PageDelegate {
}

startScreencast(options: { width: number, height: number, quality: number }) {
if (this._screencastTimer)
return;

this._waitingForScreenshot = false;
this._screencastTimer = setInterval(async () => {
if (this._waitingForScreenshot)
return;
if (this._session.isDisposed()) {
this.stopScreencast();
return;
}

this._waitingForScreenshot = true;
const payload = await this._session.sendMayFail('browsingContext.captureScreenshot', {

@Skn0tt Skn0tt Jul 1, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems very expensive, especially on typical QA pages where paints are <25fps. Have you checked performance? I wonder how this compares to non-bidi Firefox, for example on todomvc, and on media and animation heavy pages like microsoft.com or github.com.

@hbenl hbenl Jul 1, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I've checked and didn't see any issues even at higher resolutions and frame rates. At some point it'll start "dropping frames" (i.e. skipping captureScreenshot calls if the previous one hasn't finished yet) but Playwright's screencast pipeline is designed to handle these "gaps" and the worst thing I saw happening was a slight stutter in animations in the video.
In my experiments, Firefox with Juggler always sends frames at 25fps, even on static pages. Compared to that, the only downsides with this implementation are the lack of browser-side scaling and the overhead of having to send a command for every screenshot (instead of receiving them as events).

context: this._session.sessionId,
format: {
type: 'image/jpeg',
quality: options.quality / 100
}
});
if (payload) {
const buffer = Buffer.from(payload.data, 'base64');
const { width, height } = jpegDimensions(buffer);
this._page.screencast.onScreencastFrame({
buffer,
frameSwapWallTime: Date.now(),
viewportWidth: width,
viewportHeight: height,
});
}
this._waitingForScreenshot = false;
}, 40);
}

stopScreencast() {
if (this._screencastTimer) {
clearInterval(this._screencastTimer);
this._screencastTimer = undefined;
}
}

getFFmpegVideoFilterArgs({ width, height }: { width: number, height: number }) {
// We use "scale" and "pad" video filters (-vf option) to resize incoming frames
// that might be of a different size to the desired video size.
// https://ffmpeg.org/ffmpeg-filters.html#scale
// https://ffmpeg.org/ffmpeg-filters.html#pad-1
return `scale=w='min(iw,${width})':h='min(ih,${height})':force_original_aspect_ratio=decrease:eval=frame,pad=${width}:${height}:0:0:gray`;
}

rafCountForStablePosition(): number {
Expand Down Expand Up @@ -679,3 +727,22 @@ export class BidiPage implements PageDelegate {
function toBidiExecutionContext(executionContext: dom.FrameExecutionContext): BidiExecutionContext {
return executionContext.delegate as BidiExecutionContext;
}

function jpegDimensions(buffer: Buffer): { width: number, height: number } {
let i = 2; // skip SOI marker (FF D8)
while (i < buffer.length - 8) {
if (buffer[i] !== 0xFF)
break;
const marker = buffer[i + 1];
const segmentLength = buffer.readUInt16BE(i + 2);
// SOF markers: C0 (baseline), C2 (progressive), C1, C3, C5-C7, C9-CB, CD-CF
if ((marker >= 0xC0 && marker <= 0xC3) || (marker >= 0xC5 && marker <= 0xC7) ||
(marker >= 0xC9 && marker <= 0xCB) || (marker >= 0xCD && marker <= 0xCF)) {
const height = buffer.readUInt16BE(i + 5);
const width = buffer.readUInt16BE(i + 7);
return { width, height };
}
i += 2 + segmentLength;
}
throw new Error('Could not parse JPEG dimensions');
}
2 changes: 2 additions & 0 deletions packages/playwright-core/src/server/page.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ export interface PageDelegate {
// WebKit hack.
shouldToggleStyleSheetToSyncAnimations(): boolean;
setDockTile(image: Buffer): Promise<void>;
// Allow Bidi to set different ffmpeg video filter args.
getFFmpegVideoFilterArgs?: (options: { width: number, height: number }) => string;
}

type EmulatedSize = { screen: types.Size, viewport: types.Size };
Expand Down
13 changes: 7 additions & 6 deletions packages/playwright-core/src/server/videoRecorder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import { registry } from './registry';
import type * as types from './types';
import type { ChildProcess } from 'child_process';
import type { Screencast, ScreencastClient } from './screencast';
import type { Page } from './page';
import type { Page, PageDelegate } from './page';

const fps = 25;

Expand Down Expand Up @@ -60,7 +60,7 @@ export class VideoRecorder {
const { size } = this._screencast.addClient(this._client);
// For video files only, prioritize encoding into the given size, regardless of the actual pixel data.
const videoSize = options.size ?? size;
this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, outputFile);
this._videoRecorder = new FfmpegVideoRecorder(ffmpegPath, videoSize, outputFile, this._screencast.page.delegate);
this._artifact = new Artifact(this._screencast.page.browserContext, outputFile);
return this._artifact;
}
Expand Down Expand Up @@ -108,16 +108,16 @@ class FfmpegVideoRecorder {
private _launchPromise: Promise<Error | null>;
private _outputFile: string;

constructor(ffmpegPath: string, size: types.Size, outputFile: string) {
constructor(ffmpegPath: string, size: types.Size, outputFile: string, page: PageDelegate) {
if (!outputFile.endsWith('.webm'))
throw new Error('File must have .webm extension');
this._outputFile = outputFile;
this._ffmpegPath = ffmpegPath;
this._size = size;
this._launchPromise = this._launch().catch(e => e);
this._launchPromise = this._launch(page).catch(e => e);
}

private async _launch() {
private async _launch(page: PageDelegate) {
await mkdirIfNeeded(this._outputFile);
// How to tune the codec:
// 1. Read vp8 documentation to figure out the options.
Expand Down Expand Up @@ -163,7 +163,8 @@ class FfmpegVideoRecorder {

const w = this._size.width;
const h = this._size.height;
const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v 1M -threads 1 -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' ');
const videoFilterArgs = page.getFFmpegVideoFilterArgs?.({ width: w, height: h }) ?? `pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`;
const args = `-loglevel error -f matroska -fpsprobesize 0 -probesize 32 -analyzeduration 0 -i pipe:0 -y -an -r ${fps} -c:v vp8 -qmin 0 -qmax 50 -crf 8 -deadline realtime -speed 8 -b:v 1M -threads 1 -vf ${videoFilterArgs}`.split(' ');
args.push(this._outputFile);

const { launchedProcess, gracefullyClose } = await launchProcess({
Expand Down
51 changes: 2 additions & 49 deletions tests/bidi/expectations/moz-firefox-nightly-library.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,10 @@ library/browsercontext-viewport-mobile.spec.ts › mobile viewport › should su
library/browsercontext-viewport.spec.ts › should drag with high dpi [fail]
library/browsercontext-viewport.spec.ts › should support touch with null viewport [fail]
library/browsertype-connect.spec.ts › launchServer › should be able to connect 20 times to a single server without warnings [timeout]
library/browsertype-connect.spec.ts › launchServer › should name video file after page guid when connecting to remote browser with artifactsDir [fail]
library/browsertype-connect.spec.ts › launchServer › should save download [fail]
library/browsertype-connect.spec.ts › launchServer › should save videos to artifactsDir [fail]
library/browsertype-connect.spec.ts › launchServer › should saveAs videos from remote browser [timeout]
library/browsertype-connect.spec.ts › launchServer › should upload a folder [fail]
library/browsertype-connect.spec.ts › run-server › should be able to connect 20 times to a single server without warnings [timeout]
library/browsertype-connect.spec.ts › run-server › should name video file after page guid when connecting to remote browser with artifactsDir [fail]
library/browsertype-connect.spec.ts › run-server › should save download [fail]
library/browsertype-connect.spec.ts › run-server › should save videos to artifactsDir [fail]
library/browsertype-connect.spec.ts › run-server › should saveAs videos from remote browser [timeout]
library/browsertype-connect.spec.ts › run-server › should upload a folder [fail]
library/browsertype-launch.spec.ts › should reject if launched browser fails immediately [fail]
library/capabilities.spec.ts › SharedArrayBuffer should work @smoke [timeout]
Expand Down Expand Up @@ -155,11 +149,11 @@ library/inspector/cli-codegen-3.spec.ts › cli codegen › should generate fram
library/launcher.spec.ts › should throw a friendly error if its headed and there is no xserver on linux running [fail]
library/locator-dispatchevent-touch.spec.ts › should support touch points in touch event arguments [fail]
library/multiclient.spec.ts › last emulateMedia wins [fail]
library/multiclient.spec.ts › screencast should deliver cached last frame to a new client [fail]
library/multiclient.spec.ts › should chain routes [timeout]
library/multiclient.spec.ts › should unroute websockets [timeout]
library/page-clock.spec.ts › popup › should tick after popup [flaky]
library/page-event-crash.spec.ts › should be able to close context when page crashes [timeout]
library/page-event-crash.spec.ts › should be able to close page after crash [timeout]
library/page-event-crash.spec.ts › should cancel navigation when page crashes [timeout]
library/page-event-crash.spec.ts › should cancel waitForEvent when page crashes [timeout]
library/page-event-crash.spec.ts › should emit crash event when page crashes [timeout]
Expand All @@ -181,13 +175,7 @@ library/role-utils.spec.ts › axe-core accessible-text [timeout]
library/role-utils.spec.ts › wpt accname #2 [timeout]
library/role-utils.spec.ts › wpt accname #3 [timeout]
library/route-web-socket.spec.ts › should emit close upon frame detach [timeout]
library/screencast.spec.ts › onFrame receives viewport size [fail]
library/screencast.spec.ts › screencast.start delivers frames via onFrame callback [fail]
library/screencast.spec.ts › start dispose stops recording [fail]
library/screencast.spec.ts › start returns a disposable that stops screencast [fail]
library/screencast.spec.ts › start reuses existing screencast when video recording is running [fail]
library/screencast.spec.ts › start should work when recordVideo is set [fail]
library/screencast.spec.ts › supports multiple onFrame listeners [fail]
library/screenshot.spec.ts › page screenshot › should work with device scale factor and scale:css [fail]
library/screenshot.spec.ts › page screenshot › should work with device scale factor, clip and scale:css [fail]
library/selector-generator.spec.ts › selector generator › should work in dynamic iframes without navigation [fail]
Expand All @@ -201,9 +189,6 @@ library/tap.spec.ts › should send well formed touch points [timeout]
library/tap.spec.ts › should wait until an element is visible to tap it [fail]
library/tap.spec.ts › should work with modifiers [timeout]
library/tap.spec.ts › trial run should not tap [fail]
library/trace-viewer.spec.ts › canvas clipping [timeout]
library/trace-viewer.spec.ts › canvas clipping in iframe [timeout]
library/trace-viewer.spec.ts › canvas disabled title [fail]
library/trace-viewer.spec.ts › should filter network requests by multiple resource types [flaky]
library/trace-viewer.spec.ts › should filter network requests by url [fail]
library/trace-viewer.spec.ts › should filter network requests by websocket type [fail]
Expand All @@ -213,41 +198,9 @@ library/trace-viewer.spec.ts › should pick locator in iframe [fail]
library/trace-viewer.spec.ts › should render console [fail]
library/trace-viewer.spec.ts › should show only one pointer with multilevel iframes [fail]
library/trace-viewer.spec.ts › should show websocket messages [fail]
library/tracing.spec.ts › should collect trace with resources, but no js [fail]
library/tracing.spec.ts › should produce screencast frames crop [fail]
library/tracing.spec.ts › should produce screencast frames fit [fail]
library/tracing.spec.ts › should produce screencast frames scale [fail]
library/video.spec.ts › screencast › should be 800x450 by default [timeout]
library/video.spec.ts › screencast › should be 800x600 with null viewport [timeout]
library/video.spec.ts › screencast › should capture css transformation [timeout]
library/video.spec.ts › screencast › should capture full viewport [fail]
library/video.spec.ts › screencast › should capture full viewport on hidpi [fail]
library/video.spec.ts › screencast › should capture navigation [timeout]
library/video.spec.ts › screencast › should capture static page [timeout]
library/video.spec.ts › screencast › should capture static page in persistent context @smoke [fail]
library/video.spec.ts › screencast › should close ffmpeg even if there were no frames [fail]
library/video.spec.ts › screencast › should continue recording main page after popup closes [timeout]
library/video.spec.ts › screencast › should delete video [timeout]
library/video.spec.ts › screencast › should emulate an iphone [timeout]
library/video.spec.ts › screencast › should expose video path [timeout]
library/video.spec.ts › screencast › should expose video path blank page [timeout]
library/video.spec.ts › screencast › should expose video path blank popup [timeout]
library/video.spec.ts › screencast › should fail when recordVideo is set [fail]
library/video.spec.ts › screencast › should not create video for internal pages [fail]
library/video.spec.ts › screencast › should scale frames down to the requested size [timeout]
library/video.spec.ts › screencast › should throw if browser dies [fail]
library/video.spec.ts › screencast › should throw on browser close [fail]
library/video.spec.ts › screencast › should use viewport scaled down to fit into 800x800 as default size [timeout]
library/video.spec.ts › screencast › should wait for video to finish if page was closed [fail]
library/video.spec.ts › screencast › should work for popups [timeout]
library/video.spec.ts › screencast › should work with old options [timeout]
library/video.spec.ts › screencast › should work with relative path for recordVideo.dir [timeout]
library/video.spec.ts › screencast › should work with video+trace [timeout]
library/video.spec.ts › screencast › should work with weird screen resolution [timeout]
library/video.spec.ts › screencast › video.start dispose stops recording [fail]
library/video.spec.ts › screencast › video.start should fail when recordVideo is set, but stop should work [fail]
library/video.spec.ts › screencast › video.start/stop twice [timeout]
library/video.spec.ts › should saveAs video [timeout]
library/tracing.spec.ts › should save trace while a WebSocket keeps streaming frames [fail]
library/web-socket.spec.ts › should emit binary frame events [timeout]
library/web-socket.spec.ts › should emit close events [timeout]
library/web-socket.spec.ts › should emit error [timeout]
Expand Down
3 changes: 3 additions & 0 deletions tests/bidi/expectations/moz-firefox-nightly-page.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ page/page-keyboard.spec.ts › should press audio and media control keys [fail]
page/page-keyboard.spec.ts › should send a character with insertText [fail]
page/page-mouse.spec.ts › should always round down [fail]
page/page-mouse.spec.ts › should dblclick the div [timeout]
page/page-network-idle.spec.ts › should not wait for an open EventSource connection [timeout]
page/page-network-idle.spec.ts › should not wait for an open EventSource connection in setContent [timeout]
page/page-network-request.spec.ts › should not allow to access frame on popup main request [fail]
page/page-network-request.spec.ts › should not get preflight CORS requests when intercepting [fail]
page/page-network-request.spec.ts › should parse the data if content-type is application/x-www-form-urlencoded [fail]
Expand All @@ -122,6 +124,7 @@ page/page-network-response.spec.ts › should report all headers [fail]
page/page-network-response.spec.ts › should report if request was fromServiceWorker [fail]
page/page-network-response.spec.ts › should report multiple set-cookie headers [fail]
page/page-network-response.spec.ts › should wait until response completes [fail]
page/page-request-continue.spec.ts › continue with headers should send fresh cookie from the browser cookie store [fail]
page/page-request-continue.spec.ts › should not throw if request was cancelled by the page [timeout]
page/page-request-fulfill.spec.ts › should fulfill with gzip and readback [timeout]
page/page-request-fulfill.spec.ts › should not throw if request was cancelled by the page [timeout]
Expand Down
8 changes: 4 additions & 4 deletions tests/library/video.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,8 +334,8 @@ it.describe('screencast', () => {
}
});

it('should work for popups', async ({ browser, server, browserName, trace }, testInfo) => {
it.fixme(browserName === 'firefox', 'https://github.com/microsoft/playwright/issues/14557');
it('should work for popups', async ({ browser, server, browserName, isBidi }, testInfo) => {
it.fixme(browserName === 'firefox' && !isBidi, 'https://github.com/microsoft/playwright/issues/14557');
const videosPath = testInfo.outputPath('');
const size = { width: 600, height: 400 };
const context = await browser.newContext({
Expand Down Expand Up @@ -453,8 +453,8 @@ it.describe('screencast', () => {
expect(videoPlayer.videoHeight).toBe(450);
});

it('should be 800x600 with null viewport', async ({ browser, headless, browserName }, testInfo) => {
it.fixme(browserName === 'firefox' && headless, 'Fails in headless on bots');
it('should be 800x600 with null viewport', async ({ browser, headless, browserName, isBidi }, testInfo) => {
it.fixme(browserName === 'firefox' && headless && !isBidi, 'Fails in headless on bots');

const context = await browser.newContext({
recordVideo: {
Expand Down
Loading