diff --git a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx index f18226d354..360e5cfe89 100644 --- a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx @@ -367,6 +367,17 @@ ios: # WebDriverAgent host address, optional, defaults to localhost. wdaHost: + # For gateways with a path prefix, use wdaBaseUrl instead of wdaHost/wdaPort. + # Setting wdaBaseUrl together with either wdaHost or wdaPort causes an error. + # wdaBaseUrl: + + # Optional independent MJPEG stream URL, including a gateway path if needed. + # Cannot be combined with wdaMjpegPort. + # wdaMjpegUrl: + + # Local MJPEG stream port, optional. Use this or wdaMjpegUrl. + # wdaMjpegPort: + # Whether to auto dismiss keyboard, optional, defaults to false. autoDismissKeyboard: @@ -383,6 +394,16 @@ ios: # See the IOSDevice constructor documentation for the complete list ``` +For a gateway with a path prefix, set `wdaBaseUrl` and leave `wdaHost` and `wdaPort` unset. The MJPEG stream can use a separate URL: + +```yaml +ios: + wdaBaseUrl: ${WDA_BASE_URL} + wdaMjpegUrl: ${WDA_MJPEG_URL} +``` + +Set both environment variables before running the script. Studio Recorder exports use these references so gateway paths and access tokens stay out of the YAML file. + :::info View Complete iOS Configuration Options YAML scripts now support all configuration options from the `IOSDevice` constructor. For the complete list of options, see [`IOSDevice`](./reference/#iosdevice) in the iOS API reference. diff --git a/apps/site/docs/en/platforms/ios.mdx b/apps/site/docs/en/platforms/ios.mdx index ebf6b3792b..6835fb4420 100644 --- a/apps/site/docs/en/platforms/ios.mdx +++ b/apps/site/docs/en/platforms/ios.mdx @@ -233,6 +233,27 @@ For remote devices, you also need to set up port forwarding accordingly: iproxy 8100 8100 YOUR_DEVICE_ID ``` +If a gateway exposes WDA under a path prefix, set the full API base URL instead: + +```typescript +const agent = await agentFromWebDriverAgent({ + wdaBaseUrl: 'https://gateway.example/device/wda', +}); +``` + +Midscene appends `/status`, `/session`, and session commands to this base URL. Set either `wdaBaseUrl` or `wdaHost`/`wdaPort`; combining them throws an error. + +`wdaBaseUrl` only configures the WDA API. By default, the native MJPEG stream still connects to `http://localhost:9100` when using a gateway. If the gateway also exposes a stream, set its complete URL separately: + +```typescript +const agent = await agentFromWebDriverAgent({ + wdaBaseUrl: 'https://gateway.example/device/wda', + wdaMjpegUrl: 'https://gateway.example/device/mjpeg', +}); +``` + +`wdaMjpegUrl` accepts an HTTP(S) stream URL with its own host, port, path, and query. Credentials and fragments in the URL are rejected. It cannot be combined with `wdaMjpegPort`. For a local port forward, leave `wdaMjpegUrl` unset and use `wdaMjpegPort` (default `9100`). Playground falls back to screenshot polling if the native stream is unavailable. + ### How to get smoother live screen preview in Playground? Playground's screen preview supports two modes: diff --git a/apps/site/docs/en/reference/index.mdx b/apps/site/docs/en/reference/index.mdx index a262f9610f..57ca9d63dd 100644 --- a/apps/site/docs/en/reference/index.mdx +++ b/apps/site/docs/en/reference/index.mdx @@ -2499,9 +2499,11 @@ const device = new IOSDevice({ - `wdaPort?: number` — WebDriverAgent port. Default: `8100`. - `wdaHost?: string` — WebDriverAgent host. Default: `'localhost'`. +- `wdaBaseUrl?: string` — Full HTTP(S) WDA API base URL, including a gateway path prefix. Cannot be combined with `wdaHost` or `wdaPort`. - `iOSDeviceClassOverride?: string` — Optional npm module path that replaces the default `IOSDevice` when using `agentFromWebDriverAgent()` or iOS Playground. The module must export an `IOSDevice` class or a default class. - `sessionId?: string` — Existing WebDriverAgent session ID to reuse. When provided, Midscene skips creating a new WDA session. During cleanup, Midscene detaches from the externally supplied WebDriver session instead of deleting it. - `wdaMjpegPort?: number` — WDA MJPEG server port for real-time screen streaming. Default: `9100`. +- `wdaMjpegUrl?: string` — Full HTTP(S) MJPEG stream URL, including any gateway path or query. Cannot be combined with `wdaMjpegPort`. - `wdaMjpegFrameSource?: { enabled?: boolean }` — Use WDA's MJPEG stream as the continuous frame source for `agent.startObserving()`. Disabled by default; when disabled, observers fall back to sequential `screenshotBase64()` capture. - `autoDismissKeyboard?: boolean` — Whether to hide the on-screen keyboard after text input. Default: `true`. - `keyboardTypeDelay?: number` — Finite non-negative delay in milliseconds between keystrokes. A positive value makes legacy input enter one Unicode code point at a time through WDA's `/wda/keys` endpoint. Use this option when an input field drops characters during fast input. @@ -2512,6 +2514,8 @@ const device = new IOSDevice({ - Ensure Developer Mode is enabled and WDA can reach the device; use `iproxy` when forwarding ports from a real device. - Use `wdaHost`/`wdaPort` to target remote devices or custom WDA deployments. +- Use `wdaBaseUrl` when a gateway routes WDA through a path prefix; all WDA API requests and readiness checks use that prefix. +- Set `wdaMjpegUrl` separately when the native MJPEG stream is available through a gateway. The API base URL does not determine the stream URL. - For multi-device concurrency, use distinct `wdaPort` and `wdaMjpegPort` values for each device so WDA commands and MJPEG streams do not conflict. - For shared interaction methods, see [Shared Agent APIs](#interaction-methods). diff --git a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx index ac16083c3e..1e576818cd 100644 --- a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx @@ -370,6 +370,16 @@ ios: # WebDriverAgent 主机地址,可选,默认 localhost wdaHost: + # 网关带路径前缀时,用 wdaBaseUrl 替代 wdaHost/wdaPort。 + # 同时设置 wdaBaseUrl 与 wdaHost 或 wdaPort 会报错。 + # wdaBaseUrl: + + # 可单独设置 MJPEG 流地址,包括网关路径;不能与 wdaMjpegPort 同时设置。 + # wdaMjpegUrl: + + # 本地 MJPEG 流端口,可选。与 wdaMjpegUrl 二选一。 + # wdaMjpegPort: + # 是否自动关闭键盘,可选,默认 false autoDismissKeyboard: @@ -386,6 +396,16 @@ ios: # 完整配置项请参考 IOSDevice 的构造函数文档 ``` +连接带路径前缀的网关时,只设置 `wdaBaseUrl`。MJPEG 流可以使用单独的地址: + +```yaml +ios: + wdaBaseUrl: ${WDA_BASE_URL} + wdaMjpegUrl: ${WDA_MJPEG_URL} +``` + +运行脚本前,请设置这两个环境变量。Studio Recorder 导出的 YAML 也引用环境变量。这样,网关路径和访问令牌不会写入文件。 + :::info 查看完整的 iOS 配置项 YAML 脚本现在支持 `IOSDevice` 构造函数的所有配置选项。完整的配置项列表请参考 [iOS API 参考中的 IOSDevice](./reference/#iosdevice)。 diff --git a/apps/site/docs/zh/platforms/ios.mdx b/apps/site/docs/zh/platforms/ios.mdx index 6376057ea6..ed4c69b89c 100644 --- a/apps/site/docs/zh/platforms/ios.mdx +++ b/apps/site/docs/zh/platforms/ios.mdx @@ -233,6 +233,29 @@ const agent = await agentFromWebDriverAgent({ iproxy 8100 8100 YOUR_DEVICE_ID ``` +如果网关把 WDA 放在带路径前缀的地址下,可以直接配置完整的 API 基地址: + +```typescript +const agent = await agentFromWebDriverAgent({ + wdaBaseUrl: 'https://gateway.example/device/wda', +}); +``` + +Midscene 会在该地址后追加 `/status`、`/session` 和会话接口路径。连接网关时,只设置 `wdaBaseUrl`。如果使用主机和端口,则设置 `wdaHost` 和 `wdaPort`。两种方式混用会报错。 + +`wdaBaseUrl` 只配置 WDA API。MJPEG 流默认连接 `http://localhost:9100`。如果网关也提供画面流,请单独设置完整地址: + +```typescript +const agent = await agentFromWebDriverAgent({ + wdaBaseUrl: 'https://gateway.example/device/wda', + wdaMjpegUrl: 'https://gateway.example/device/mjpeg', +}); +``` + +`wdaMjpegUrl` 可以使用独立的协议、主机、端口、路径和查询参数。协议须为 HTTP(S),地址中不能包含用户名、密码或片段。`wdaMjpegUrl` 与 `wdaMjpegPort` 不能同时设置。 + +如果使用本地端口转发,可以设置 `wdaMjpegPort`,默认值为 `9100`。画面流不可用时,Playground 会退回到截图轮询。 + ### 如何在 Playground 中获得更流畅的实时画面? Playground 的画面预览支持两种模式: diff --git a/apps/site/docs/zh/reference/index.mdx b/apps/site/docs/zh/reference/index.mdx index d5a99895d1..a6ad30b09c 100644 --- a/apps/site/docs/zh/reference/index.mdx +++ b/apps/site/docs/zh/reference/index.mdx @@ -2457,9 +2457,11 @@ const device = new IOSDevice({ - `wdaPort?: number` —— WebDriverAgent 端口,默认 `8100`。 - `wdaHost?: string` —— WebDriverAgent host,默认 `'localhost'`。 +- `wdaBaseUrl?: string` —— 完整的 HTTP(S) WDA API 基地址,可包含网关路径前缀。不能与 `wdaHost` 或 `wdaPort` 同时设置。 - `iOSDeviceClassOverride?: string` —— 使用 `agentFromWebDriverAgent()` 或 iOS Playground 时替换默认 `IOSDevice` 的 npm module path。目标模块必须导出 `IOSDevice` class 或 default class。 - `sessionId?: string` —— 复用已有的 WebDriverAgent session ID。传入后,Midscene 不再创建新的 WDA session;清理时只会从这个外部 WebDriver session 分离,不会删除它。 - `wdaMjpegPort?: number` —— WDA MJPEG 服务端口,用于实时画面流,默认 `9100`。 +- `wdaMjpegUrl?: string` —— 完整的 HTTP(S) MJPEG 画面流地址,可包含网关路径或查询参数。不能与 `wdaMjpegPort` 同时设置。 - `wdaMjpegFrameSource?: { enabled?: boolean }` —— 使用 WDA 的 MJPEG stream 作为 `agent.startObserving()` 的连续帧源。默认关闭;关闭时,观察逻辑会退回到连续调用 `screenshotBase64()`。 - `autoDismissKeyboard?: boolean` —— 文本输入后自动隐藏键盘,默认 `true`。 - `keyboardTypeDelay?: number` —— 按键间延迟,单位为毫秒。取值必须是有限的非负数。设为正数后,`legacy` 输入会通过 WDA 的 `/wda/keys` 接口逐个 Unicode 码点执行。适用于输入框在快速输入下丢字的场景。 @@ -2470,6 +2472,8 @@ const device = new IOSDevice({ - 请确认已开启开发者模式且 WDA 能访问设备;真机转发端口时可借助 `iproxy`。 - 通过 `wdaHost`/`wdaPort` 可指向远程设备或自建的 WDA。 +- 网关通过路径前缀转发 WDA 时,使用 `wdaBaseUrl`;WDA API 请求和就绪探测都会使用该前缀。 +- 网关提供原生 MJPEG 流时,单独设置 `wdaMjpegUrl`。WDA API 基地址不会决定画面流地址。 - 多设备并发时,请为每个设备设置不同的 `wdaPort` 和 `wdaMjpegPort`,避免 WDA 命令和 MJPEG stream 端口冲突。 - 通用交互方法请查阅 [API 参考(通用)](#interaction-methods)。 diff --git a/apps/studio/src/renderer/playground/selectors.ts b/apps/studio/src/renderer/playground/selectors.ts index c95df8be58..40f9fd063f 100644 --- a/apps/studio/src/renderer/playground/selectors.ts +++ b/apps/studio/src/renderer/playground/selectors.ts @@ -2,6 +2,7 @@ import type { PlaygroundRuntimeInfo, PlaygroundSessionTarget, } from '@midscene/playground'; +import { sha256Hex } from '@midscene/shared/utils'; import type { DiscoveredDevice, PlatformDiscoveryError, @@ -47,6 +48,40 @@ function buildHostPortId(host: string, port: number): string { return `${host}:${port}`; } +function iosFormValue(formValues: Record, key: string) { + const value = formValues[`ios.${key}`] ?? formValues[key]; + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + +function normalizedGatewayUrl(value: string) { + const url = new URL(value); + return url.toString().replace(/\/+$/, ''); +} + +export function resolveSelectedIosGatewayId( + formValues: Record, +): string | undefined { + const baseUrl = iosFormValue(formValues, 'baseUrl'); + if (!baseUrl) return undefined; + try { + const mjpegUrl = iosFormValue(formValues, 'mjpegUrl'); + const sessionId = iosFormValue(formValues, 'sessionId'); + const mjpegPort = normalizePort( + formValues['ios.mjpegPort'] ?? formValues.mjpegPort, + ); + return `ios-gateway-${sha256Hex( + JSON.stringify({ + baseUrl: normalizedGatewayUrl(baseUrl), + mjpegUrl: mjpegUrl ? new URL(mjpegUrl).toString() : undefined, + mjpegPort, + sessionId, + }), + )}`; + } catch { + return undefined; + } +} + /** * Map any incoming platform string (runtime metadata, form values, desktop * OS aliases like `macos`) to the canonical `StudioPlatformId`. Exported so @@ -84,7 +119,7 @@ export function normalizeStudioPlatformId( * Platforms use different metadata keys for the device id: * Android / Harmony → metadata.deviceId * Computer → metadata.displayId - * iOS → metadata.wdaHost + metadata.wdaPort + * iOS → gateway URL digest, or metadata.wdaHost + metadata.wdaPort */ export function resolveConnectedDeviceId( runtimeInfo: PlaygroundRuntimeInfo | null, @@ -96,6 +131,9 @@ export function resolveConnectedDeviceId( if (isString(metadata.displayId)) { return metadata.displayId; } + if (isString(metadata.wdaGatewayId)) { + return metadata.wdaGatewayId; + } if (isString(metadata.wdaHost)) { const wdaPort = normalizePort(metadata.wdaPort); if (wdaPort !== undefined) { @@ -108,6 +146,7 @@ export function resolveConnectedDeviceId( function resolveConnectedSessionValues( runtimeInfo: PlaygroundRuntimeInfo | null, platformKey: StudioSidebarPlatformKey, + formValues: Record = {}, ): Record | undefined { const metadata = runtimeInfo?.metadata || {}; @@ -126,7 +165,20 @@ function resolveConnectedSessionValues( } : undefined; case 'ios': { + if (isString(metadata.wdaGatewayId)) { + return resolveSelectedIosGatewayId(formValues) === metadata.wdaGatewayId + ? resolveSelectedSessionValues('ios', formValues) + : undefined; + } const wdaPort = normalizePort(metadata.wdaPort); + if ( + isString(metadata.wdaHost) && + wdaPort !== undefined && + resolveSelectedDeviceId({ ...formValues, platformId: 'ios' }) === + buildHostPortId(metadata.wdaHost, wdaPort) + ) { + return resolveSelectedSessionValues('ios', formValues); + } return isString(metadata.wdaHost) && wdaPort !== undefined ? { host: metadata.wdaHost, @@ -155,6 +207,9 @@ export function resolveConnectedDeviceLabel( } const deviceId = resolveConnectedDeviceId(runtimeInfo); if (deviceId) { + if (isString(metadata.wdaGatewayId) && isString(metadata.wdaHost)) { + return `${metadata.wdaHost} (WDA gateway)`; + } // "Display 1" reads better than a bare numeric id for computer. return isString(metadata.displayId) && !isString(metadata.deviceId) ? `Display ${deviceId}` @@ -169,13 +224,11 @@ export function resolveConnectedDeviceLabel( function buildGenericConnectedDeviceItem( runtimeInfo: PlaygroundRuntimeInfo | null, platformKey: StudioSidebarPlatformKey, + formValues: Record, ): StudioAndroidDeviceItem | null { const metadata = runtimeInfo?.metadata || {}; const deviceId = resolveConnectedDeviceId(runtimeInfo); - const label = isString(metadata.sessionDisplayName) - ? metadata.sessionDisplayName - : deviceId || - (isString(runtimeInfo?.title) ? runtimeInfo.title : undefined); + const label = resolveConnectedDeviceLabel(runtimeInfo, { emptyLabel: '' }); if (!label) { return null; @@ -184,10 +237,17 @@ function buildGenericConnectedDeviceItem( return { id: deviceId || `${platformKey}-connected`, label, - description: deviceId && deviceId !== label ? deviceId : undefined, + description: + deviceId && deviceId !== label && !isString(metadata.wdaGatewayId) + ? deviceId + : undefined, selected: true, status: 'active', - sessionValues: resolveConnectedSessionValues(runtimeInfo, platformKey), + sessionValues: resolveConnectedSessionValues( + runtimeInfo, + platformKey, + formValues, + ), }; } @@ -214,6 +274,8 @@ export function resolveSelectedDeviceId( const selectedPlatform = normalizeStudioPlatformId(formValues.platformId); if (selectedPlatform === 'ios') { + const gatewayId = resolveSelectedIosGatewayId(formValues); + if (gatewayId) return gatewayId; const host = isString(formValues['ios.host']) ? formValues['ios.host'] : isString(formValues.host) @@ -300,13 +362,35 @@ function resolveSelectedSessionValues( ? { deviceId: formValues.deviceId } : undefined; case 'ios': { + const baseUrl = iosFormValue(formValues, 'baseUrl'); + const mjpegUrl = iosFormValue(formValues, 'mjpegUrl'); + const sessionId = iosFormValue(formValues, 'sessionId'); + const mjpegPort = normalizePort( + formValues['ios.mjpegPort'] ?? formValues.mjpegPort, + ); + if (baseUrl) { + return { + baseUrl, + ...(mjpegUrl ? { mjpegUrl } : {}), + ...(mjpegPort !== undefined ? { mjpegPort } : {}), + ...(sessionId ? { sessionId } : {}), + }; + } const host = isString(formValues['ios.host']) ? formValues['ios.host'] : isString(formValues.host) ? formValues.host : undefined; const port = normalizePort(formValues['ios.port'] ?? formValues.port); - return host && port !== undefined ? { host, port } : undefined; + return host && port !== undefined + ? { + host, + port, + ...(mjpegUrl ? { mjpegUrl } : {}), + ...(mjpegPort !== undefined ? { mjpegPort } : {}), + ...(sessionId ? { sessionId } : {}), + } + : undefined; } default: return undefined; @@ -316,8 +400,20 @@ function resolveSelectedSessionValues( export function buildDeviceSelectionFormValues( platform: StudioSidebarPlatformKey, device: Pick, -): Record { +): Record { if (device.sessionValues) { + if (platform === 'ios') { + return { + platformId: platform, + 'ios.host': null, + 'ios.port': null, + 'ios.baseUrl': null, + 'ios.mjpegUrl': null, + 'ios.mjpegPort': null, + 'ios.sessionId': null, + ...prefixSessionValues(platform, device.sessionValues), + }; + } return { platformId: platform, ...prefixSessionValues(platform, device.sessionValues), @@ -505,6 +601,7 @@ export function buildStudioSidebarDeviceBuckets({ const connectedItem = buildGenericConnectedDeviceItem( runtimeInfo, runtimePlatformKey, + formValues, ); if (connectedItem) { diff --git a/apps/studio/src/renderer/recorder/export.ts b/apps/studio/src/renderer/recorder/export.ts index 653f2099a3..fa5befb420 100644 --- a/apps/studio/src/renderer/recorder/export.ts +++ b/apps/studio/src/renderer/recorder/export.ts @@ -313,6 +313,28 @@ export function generateStudioRecorderYaml(session: StudioRecordingSession) { `${session.target.platformId}:`, ]; + if (session.target.values.wdaBaseUrl === '${WDA_BASE_URL}') { + lines.splice( + 3, + 0, + '# Set WDA_BASE_URL to the original WDA gateway URL before replay.', + ); + } + if (session.target.values.wdaMjpegUrl === '${WDA_MJPEG_URL}') { + lines.splice( + 3, + 0, + '# Set WDA_MJPEG_URL to the original MJPEG URL before replay.', + ); + } + if (session.target.values.sessionId === '${WDA_SESSION_ID}') { + lines.splice( + 3, + 0, + '# Set WDA_SESSION_ID to the existing WDA session ID before replay.', + ); + } + const targetValues = Object.entries(session.target.values); if (targetValues.length > 0) { for (const [key, value] of targetValues) { diff --git a/apps/studio/src/renderer/recorder/selectors.ts b/apps/studio/src/renderer/recorder/selectors.ts index 0f125802b2..0119417e41 100644 --- a/apps/studio/src/renderer/recorder/selectors.ts +++ b/apps/studio/src/renderer/recorder/selectors.ts @@ -5,6 +5,7 @@ import { resolveConnectedDeviceId, resolveConnectedDeviceLabel, resolveSelectedDeviceId, + resolveSelectedIosGatewayId, } from '../playground/selectors'; import type { StudioPlaygroundContextValue } from '../playground/types'; import type { StudioRecorderTarget, StudioRecordingSession } from './types'; @@ -125,6 +126,37 @@ export function resolveStudioRecorderTarget( }; } case 'ios': { + const baseUrl = formValues['ios.baseUrl'] ?? formValues.baseUrl; + const mjpegUrl = formValues['ios.mjpegUrl'] ?? formValues.mjpegUrl; + const sessionId = formValues['ios.sessionId'] ?? formValues.sessionId; + const mjpegPort = normalizePort( + formValues['ios.mjpegPort'] ?? + formValues.mjpegPort ?? + metadata.wdaMjpegPort, + ); + if (isNonEmptyString(metadata.wdaGatewayId)) { + if ( + !isNonEmptyString(baseUrl) || + resolveSelectedIosGatewayId(formValues) !== metadata.wdaGatewayId + ) { + return null; + } + return { + platformId, + deviceId: metadata.wdaGatewayId, + label, + values: { + wdaBaseUrl: '${WDA_BASE_URL}', + ...(isNonEmptyString(mjpegUrl) + ? { wdaMjpegUrl: '${WDA_MJPEG_URL}' } + : {}), + ...(mjpegPort !== undefined ? { wdaMjpegPort: mjpegPort } : {}), + ...(isNonEmptyString(sessionId) + ? { sessionId: '${WDA_SESSION_ID}' } + : {}), + }, + }; + } const host = resolveNamespacedValue('ios', 'host', runtimeInfo, formValues) ?? metadata.wdaHost; @@ -140,8 +172,15 @@ export function resolveStudioRecorderTarget( deviceId: deviceId ?? `${host}:${port}`, label, values: { - host, - port, + wdaHost: host, + wdaPort: port, + ...(isNonEmptyString(mjpegUrl) + ? { wdaMjpegUrl: '${WDA_MJPEG_URL}' } + : {}), + ...(mjpegPort !== undefined ? { wdaMjpegPort: mjpegPort } : {}), + ...(isNonEmptyString(sessionId) + ? { sessionId: '${WDA_SESSION_ID}' } + : {}), }, }; } @@ -228,8 +267,11 @@ function createStudioRecorderHistoryTargetSignature( case 'ios': return JSON.stringify({ platformId: target.platformId, - host: target.values.host, - port: target.values.port, + ...(target.values.wdaBaseUrl ? { deviceId: target.deviceId } : {}), + wdaHost: target.values.wdaHost ?? target.values.host, + wdaPort: target.values.wdaPort ?? target.values.port, + wdaMjpegUrl: target.values.wdaMjpegUrl, + wdaMjpegPort: target.values.wdaMjpegPort, }); default: return createStudioRecorderTargetSignature(target); diff --git a/apps/studio/tests/playground-selectors.test.ts b/apps/studio/tests/playground-selectors.test.ts index 8ffa7bdaf1..74771e7ab6 100644 --- a/apps/studio/tests/playground-selectors.test.ts +++ b/apps/studio/tests/playground-selectors.test.ts @@ -93,6 +93,55 @@ describe('resolveVisibleSidebarPlatforms', () => { }); describe('buildStudioSidebarDeviceBuckets', () => { + it('keeps the gateway URL for reconnection while using a path-specific opaque id', () => { + const formValues = { + platformId: 'ios', + 'ios.baseUrl': 'https://gateway.example/device-a/wda', + 'ios.mjpegUrl': 'https://gateway.example/device-a/mjpeg?token=secret', + }; + const id = resolveSelectedDeviceId(formValues); + const otherId = resolveSelectedDeviceId({ + ...formValues, + 'ios.baseUrl': 'https://gateway.example/device-b/wda', + }); + expect(id).toMatch(/^ios-gateway-[a-f0-9]{64}$/); + expect(otherId).not.toBe(id); + expect( + resolveSelectedDeviceId({ + ...formValues, + 'ios.baseUrl': 'https://gateway.example/device-a/wda/', + }), + ).toBe(id); + + const bucket = buildStudioSidebarDeviceBuckets({ + formValues, + runtimeInfo: { + platformId: 'ios', + interface: { type: 'ios' }, + preview: { kind: 'mjpeg' }, + executionUxHints: [], + metadata: { + wdaGatewayId: id, + wdaHost: 'gateway.example', + wdaPort: 443, + sessionDisplayName: 'iPhone', + }, + }, + targets: [], + }).ios; + expect(bucket[0]?.id).toBe(id); + expect(bucket[0]?.label).toBe('iPhone'); + expect(bucket[0]?.sessionValues).toEqual({ + baseUrl: formValues['ios.baseUrl'], + mjpegUrl: formValues['ios.mjpegUrl'], + }); + expect(buildDeviceSelectionFormValues('ios', bucket[0])).toMatchObject({ + 'ios.baseUrl': formValues['ios.baseUrl'], + 'ios.host': null, + 'ios.port': null, + }); + expect(bucket[0]?.id).not.toContain('secret'); + }); it('only populates Android when the runtime is Android', () => { const buckets = buildStudioSidebarDeviceBuckets({ formValues: { @@ -626,6 +675,10 @@ describe('buildDeviceSelectionFormValues', () => { platformId: 'ios', 'ios.host': 'localhost', 'ios.port': 8100, + 'ios.baseUrl': null, + 'ios.mjpegUrl': null, + 'ios.mjpegPort': null, + 'ios.sessionId': null, }); }); }); diff --git a/apps/studio/tests/studio-recorder.test.ts b/apps/studio/tests/studio-recorder.test.ts index e6c58eca50..535d527352 100644 --- a/apps/studio/tests/studio-recorder.test.ts +++ b/apps/studio/tests/studio-recorder.test.ts @@ -1,5 +1,6 @@ import { DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS } from '@midscene/shared/recorder'; import { describe, expect, it } from 'vitest'; +import { resolveSelectedDeviceId } from '../src/renderer/playground/selectors'; import { toStudioRecorderCodegenInput } from '../src/renderer/recorder/codegen-adapter'; import { mapPreviewRecorderEventToStudioRecordedEvent } from '../src/renderer/recorder/event-mapper'; import { generateStudioRecorderYaml } from '../src/renderer/recorder/export'; @@ -71,7 +72,7 @@ describe('studio recorder selectors', () => { }, { platformId: 'ios' }, )?.values, - ).toEqual({ host: '127.0.0.1', port: 8100 }); + ).toEqual({ wdaHost: '127.0.0.1', wdaPort: 8100 }); expect( resolveStudioRecorderTarget( @@ -87,6 +88,68 @@ describe('studio recorder selectors', () => { ).toEqual({ displayId: '1' }); }); + it('separates gateway paths and exports a reconnectable environment reference', () => { + const formValues = { + platformId: 'ios', + 'ios.baseUrl': 'https://gateway.example/device-a/wda', + 'ios.mjpegUrl': 'https://gateway.example/device-a/mjpeg?token=secret', + 'ios.sessionId': 'secret-session', + }; + const gatewayId = resolveSelectedDeviceId(formValues); + const target = resolveStudioRecorderTarget( + { + platformId: 'ios', + interface: { type: 'ios' }, + preview: { kind: 'mjpeg' }, + executionUxHints: [], + metadata: { + wdaGatewayId: gatewayId, + wdaHost: 'gateway.example', + wdaPort: 443, + sessionDisplayName: 'iPhone', + }, + }, + formValues, + ); + expect(target).toMatchObject({ + deviceId: gatewayId, + label: 'iPhone', + values: { + wdaBaseUrl: '${WDA_BASE_URL}', + wdaMjpegUrl: '${WDA_MJPEG_URL}', + sessionId: '${WDA_SESSION_ID}', + }, + }); + expect(JSON.stringify(target)).not.toContain('secret'); + const otherTarget = { + ...target!, + deviceId: resolveSelectedDeviceId({ + ...formValues, + 'ios.baseUrl': 'https://gateway.example/device-b/wda', + }), + }; + const sessions = [ + { id: 'a', target }, + { id: 'b', target: otherTarget }, + ] as StudioRecordingSession[]; + expect( + filterStudioRecorderSessionsForTarget(sessions, target).map((s) => s.id), + ).toEqual(['a']); + const yaml = generateStudioRecorderYaml({ + id: 'a', + name: 'Gateway recording', + status: 'completed', + target: target!, + events: [], + createdAt: 1, + updatedAt: 1, + }); + expect(yaml).toContain('wdaBaseUrl: "${WDA_BASE_URL}"'); + expect(yaml).toContain('wdaMjpegUrl: "${WDA_MJPEG_URL}"'); + expect(yaml).toContain('sessionId: "${WDA_SESSION_ID}"'); + expect(yaml).not.toContain('secret'); + }); + it('filters recording history by platform-appropriate target granularity', () => { const webTarget = { platformId: 'web' as const, @@ -207,6 +270,12 @@ describe('studio recorder selectors', () => { (session) => session.id, ), ).toEqual(['ios-session']); + expect( + filterStudioRecorderSessionsForTarget(sessions, { + ...iosTarget, + values: { wdaHost: '127.0.0.1', wdaPort: 8100 }, + }).map((session) => session.id), + ).toEqual(['ios-session']); expect(filterStudioRecorderSessionsForTarget(sessions, null)).toEqual([]); }); }); diff --git a/packages/core/src/device/device-options.ts b/packages/core/src/device/device-options.ts index a6f402729b..c88d57a37d 100644 --- a/packages/core/src/device/device-options.ts +++ b/packages/core/src/device/device-options.ts @@ -199,6 +199,8 @@ export type IOSDeviceOpt = { wdaPort?: number; /** WebDriverAgent host (default: 'localhost') */ wdaHost?: string; + /** Full WDA API base URL, including any gateway path prefix. Cannot be combined with wdaHost or wdaPort. */ + wdaBaseUrl?: string; /** * Existing WebDriverAgent session ID to reuse. * When provided, Midscene skips creating a new WDA session and does not delete @@ -207,6 +209,8 @@ export type IOSDeviceOpt = { sessionId?: string; /** WDA MJPEG server port for real-time screen streaming (default: 9100) */ wdaMjpegPort?: number; + /** Full HTTP(S) MJPEG stream URL. Cannot be combined with wdaMjpegPort. */ + wdaMjpegUrl?: string; /** * Use WDA's MJPEG stream as a continuous frame source for UI observation * (`agent.startObserving()`). Disabled by default (opt-in), mirroring diff --git a/packages/ios/src/agent-tools.ts b/packages/ios/src/agent-tools.ts index 738d7c3571..0625e0ed9f 100644 --- a/packages/ios/src/agent-tools.ts +++ b/packages/ios/src/agent-tools.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { z } from '@midscene/core'; import { type AgentBehaviorInitArgs, @@ -13,10 +14,17 @@ import type { ToolDefinition } from '@midscene/shared/agent-tools/types'; import { getDebug } from '@midscene/shared/logger'; import { type IOSAgent, agentFromWebDriverAgent } from './agent'; import { IOSDevice, type IOSDeviceOpt } from './device'; +import { assertWdaConnectionOptions } from './wda-options'; const debug = getDebug('agent-tools:ios'); const iosInitArgShape = { + wdaBaseUrl: z + .string() + .optional() + .describe( + 'Full WebDriverAgent API base URL, including gateway path prefix', + ), wdaHost: z .string() .optional() @@ -30,6 +38,10 @@ const iosInitArgShape = { .number() .optional() .describe('WebDriverAgent MJPEG streaming port'), + wdaMjpegUrl: z + .string() + .optional() + .describe('Full HTTP(S) WebDriverAgent MJPEG stream URL'), wdaMjpegFrameSource: z .object({ enabled: z.boolean().optional() }) .optional() @@ -42,10 +54,25 @@ const iosInitArgShape = { type IOSInitArgs = AgentBehaviorInitArgs & Pick< IOSDeviceOpt, - 'wdaHost' | 'wdaPort' | 'sessionId' | 'wdaMjpegPort' | 'wdaMjpegFrameSource' + | 'wdaBaseUrl' + | 'wdaHost' + | 'wdaPort' + | 'sessionId' + | 'wdaMjpegPort' + | 'wdaMjpegUrl' + | 'wdaMjpegFrameSource' >; function getTargetIdentity(initArgs?: IOSInitArgs): string { + if (initArgs?.wdaBaseUrl || initArgs?.wdaMjpegUrl) { + const fingerprint = createHash('sha256') + .update( + `${initArgs.wdaBaseUrl ?? ''}\0${initArgs.wdaMjpegUrl ?? ''}\0${initArgs.sessionId ?? ''}`, + ) + .digest('hex') + .slice(0, 12); + return `wda-${fingerprint}`; + } if (initArgs?.wdaHost || initArgs?.wdaPort || initArgs?.sessionId) { const wdaHost = initArgs.wdaHost ?? 'localhost'; const wdaPort = initArgs.wdaPort ?? 'default'; @@ -85,6 +112,7 @@ export class IOSMidsceneTools extends BaseMidsceneTools { } protected async ensureAgent(opts?: IOSInitArgs): Promise { + assertWdaConnectionOptions(opts); const nextSignature = getAgentInitArgsSignature(opts); if ( @@ -103,7 +131,7 @@ export class IOSMidsceneTools extends BaseMidsceneTools { return this.agent; } - debug('Creating iOS agent with WebDriverAgent options:', opts || {}); + debug('Creating iOS agent with WebDriverAgent options'); const reportOptions = this.readCliReportAgentOptions(); this.agent = await agentFromWebDriverAgent({ autoDismissKeyboard: false, @@ -126,6 +154,7 @@ export class IOSMidsceneTools extends BaseMidsceneTools { cli: this.getAgentInitArgCliMetadata(), handler: async (args: Record) => { const initArgs = this.extractAgentInitParam(args); + assertWdaConnectionOptions(initArgs); const identity = getTargetIdentity(initArgs); const reportSession = this.createNewCliReportSession(identity); this.commitCliReportSession(reportSession); diff --git a/packages/ios/src/agent.ts b/packages/ios/src/agent.ts index 008c949dd6..d8854a9c6d 100644 --- a/packages/ios/src/agent.ts +++ b/packages/ios/src/agent.ts @@ -18,6 +18,7 @@ import { type IOSDeviceOpt, } from './device'; import { iosAgentTestRunnerNodeDefinitions } from './test-runner-nodes'; +import { assertWdaConnectionOptions } from './wda-options'; const debugAgent = getDebug('ios:agent'); type IOSDeviceClass = new (opts?: IOSDeviceOpt) => IOSDevice; @@ -122,6 +123,7 @@ export class IOSAgent extends PageAgent { export async function agentFromWebDriverAgent( opts?: IOSAgentOpt & IOSDeviceOpt, ) { + assertWdaConnectionOptions(opts); debugAgent('Creating iOS agent with WebDriverAgent'); const overrideModule = diff --git a/packages/ios/src/device.ts b/packages/ios/src/device.ts index 5c44929ad1..ee5b70234c 100644 --- a/packages/ios/src/device.ts +++ b/packages/ios/src/device.ts @@ -31,6 +31,10 @@ import { normalizeForComparison } from '@midscene/shared/utils'; import { WDAManager } from '@midscene/webdriver'; import { IOSWebDriverClient as WebDriverAgentBackend } from './ios-webdriver-client'; import { MjpegFrameSource } from './mjpeg-frame-source'; +import { + assertWdaConnectionOptions, + normalizeMjpegStreamUrl, +} from './wda-options'; // Re-export IOSDeviceOpt and IOSDeviceInputOpt for backward compatibility export type { IOSDeviceOpt, IOSDeviceInputOpt } from '@midscene/core/device'; @@ -340,6 +344,7 @@ export class IOSDevice implements AbstractInterface { } constructor(options?: IOSDeviceOpt) { + assertWdaConnectionOptions(options); // deviceId will be auto-detected from WebDriverAgent connection this.deviceId = 'pending-connection'; this.options = options; @@ -351,10 +356,20 @@ export class IOSDevice implements AbstractInterface { this.wdaBackend = new WebDriverAgentBackend({ port: wdaPort, host: wdaHost, + ...(options?.wdaBaseUrl !== undefined + ? { baseUrl: options.wdaBaseUrl } + : {}), ...(options?.sessionId ? { sessionId: options.sessionId } : {}), }); - this.wdaManager = WDAManager.getInstance(wdaPort, wdaHost); - this.mjpegStreamUrl = `http://${wdaHost}:${mjpegPort}`; + this.wdaManager = WDAManager.getInstance( + wdaPort, + wdaHost, + options?.wdaBaseUrl, + ); + this.mjpegStreamUrl = + options?.wdaMjpegUrl !== undefined + ? normalizeMjpegStreamUrl(options.wdaMjpegUrl) + : `http://${wdaHost}:${mjpegPort}`; // Opt-in (default off), mirroring Android scrcpy: only expose the MJPEG // frame-source capability when explicitly enabled. When off, UI observers diff --git a/packages/ios/src/mjpeg-frame-source.ts b/packages/ios/src/mjpeg-frame-source.ts index b49bc74155..442ea71e1a 100644 --- a/packages/ios/src/mjpeg-frame-source.ts +++ b/packages/ios/src/mjpeg-frame-source.ts @@ -112,7 +112,7 @@ export class MjpegFrameSource { await new Promise((resolve) => setTimeout(resolve, 30)); } throw new Error( - `MjpegFrameSource: no frame received from ${this.url} within ${timeoutMs}ms`, + `MjpegFrameSource: no frame received within ${timeoutMs}ms`, ); } @@ -157,7 +157,10 @@ export class MjpegFrameSource { } } catch (error) { if (this.stopped || signal.aborted) return; - debug('MJPEG stream error, will retry: %s', error); + debug( + 'MJPEG stream error, will retry: %s', + error instanceof Error ? error.name : 'unknown error', + ); } // Back off before reconnecting, whether the stream errored or ended // cleanly, so a short-lived connection cannot busy-spin reconnects. diff --git a/packages/ios/src/platform.ts b/packages/ios/src/platform.ts index b3b0275899..887f1d5347 100644 --- a/packages/ios/src/platform.ts +++ b/packages/ios/src/platform.ts @@ -9,11 +9,17 @@ import { PLAYGROUND_SERVER_PORT, } from '@midscene/shared/constants'; import { findAvailablePort } from '@midscene/shared/node'; +import { sha256Hex } from '@midscene/shared/utils'; +import { normalizeWebDriverBaseUrl } from '@midscene/webdriver'; import { type IOSAgent, type IOSAgentOpt, agentFromWebDriverAgent, } from './agent'; +import { + assertWdaConnectionOptions, + normalizeMjpegStreamUrl, +} from './wda-options'; export interface IOSPlatformOptions { staticDir?: string; @@ -72,7 +78,7 @@ export const iosPlaygroundPlatform = definePlaygroundPlatform< return { title: 'Connect WebDriverAgent', description: - 'Provide the WebDriverAgent host and port that are already running for your selected iPhone or simulator.', + 'Provide a WebDriverAgent base URL, or the host and port for your selected iPhone or simulator.', primaryActionLabel: 'Create Agent', autoSubmitWhenReady: wdaReady, fields: [ @@ -80,18 +86,37 @@ export const iosPlaygroundPlatform = definePlaygroundPlatform< key: 'host', label: 'WebDriverAgent host', type: 'text', - required: true, - defaultValue: 'localhost', + required: false, placeholder: 'localhost', }, { key: 'port', label: 'WebDriverAgent port', type: 'number', - required: true, - defaultValue: DEFAULT_WDA_PORT, + required: false, placeholder: DEFAULT_WDA_PORT.toString(), }, + { + key: 'baseUrl', + label: 'WebDriverAgent base URL (optional)', + type: 'text', + required: false, + placeholder: 'https://gateway.example/device/wda', + }, + { + key: 'mjpegUrl', + label: 'MJPEG stream URL (optional)', + type: 'text', + required: false, + placeholder: 'https://gateway.example/device/mjpeg', + }, + { + key: 'mjpegPort', + label: 'MJPEG stream port (optional)', + type: 'number', + required: false, + placeholder: '9100', + }, { key: 'sessionId', label: 'WebDriverAgent session ID', @@ -103,14 +128,47 @@ export const iosPlaygroundPlatform = definePlaygroundPlatform< }; }, async createSession(input) { + const baseUrl = + typeof input?.baseUrl === 'string' && input.baseUrl.trim() + ? normalizeWebDriverBaseUrl(input.baseUrl.trim()) + : undefined; + const hasHostInput = + input?.host !== undefined && String(input.host).trim() !== ''; + const hasPortInput = + input?.port !== undefined && String(input.port).trim() !== ''; + const mjpegUrl = + typeof input?.mjpegUrl === 'string' && input.mjpegUrl.trim() + ? normalizeMjpegStreamUrl(input.mjpegUrl.trim()) + : undefined; + const hasMjpegPortInput = + input?.mjpegPort != null && String(input.mjpegPort).trim() !== ''; + const mjpegPort = hasMjpegPortInput + ? Number(input?.mjpegPort) + : undefined; + assertWdaConnectionOptions({ + ...(baseUrl ? { wdaBaseUrl: baseUrl } : {}), + ...(hasHostInput ? { wdaHost: String(input?.host) } : {}), + ...(hasPortInput ? { wdaPort: Number(input?.port) } : {}), + ...(mjpegUrl ? { wdaMjpegUrl: mjpegUrl } : {}), + ...(mjpegPort !== undefined ? { wdaMjpegPort: mjpegPort } : {}), + }); + if ( + mjpegPort !== undefined && + (!Number.isInteger(mjpegPort) || mjpegPort < 1 || mjpegPort > 65535) + ) { + throw new Error( + `Invalid MJPEG stream port: ${String(input?.mjpegPort)}`, + ); + } const host = typeof input?.host === 'string' && input.host.trim() ? input.host.trim().replace(/^https?:\/\//, '') : 'localhost'; - const port = - typeof input?.port === 'number' + const port = !hasPortInput + ? DEFAULT_WDA_PORT + : typeof input?.port === 'number' ? input.port - : Number.parseInt(String(input?.port ?? DEFAULT_WDA_PORT), 10); + : Number.parseInt(String(input?.port), 10); if (Number.isNaN(port) || port < 1 || port > 65535) { throw new Error( @@ -123,19 +181,27 @@ export const iosPlaygroundPlatform = definePlaygroundPlatform< : undefined; const connectAgent = async (): Promise => { - return agentFromWebDriverAgent({ + const agentOptions = { ...options?.getAgentOptions?.(), - wdaHost: host, - wdaPort: port, + ...(baseUrl + ? { wdaBaseUrl: baseUrl } + : { wdaHost: host, wdaPort: port }), + ...(mjpegUrl ? { wdaMjpegUrl: mjpegUrl } : {}), + ...(mjpegPort !== undefined ? { wdaMjpegPort: mjpegPort } : {}), ...(sessionId ? { sessionId } : {}), - }); + }; + assertWdaConnectionOptions(agentOptions); + return agentFromWebDriverAgent(agentOptions); }; const agent = await connectAgent(); const deviceInfo = await agent.interface.getConnectedDeviceInfo?.(); + const gatewayUrl = baseUrl ? new URL(baseUrl) : undefined; const displayName = deviceInfo ? `${deviceInfo.name} (${deviceInfo.model})` - : `${host}:${port}`; + : gatewayUrl + ? `${gatewayUrl.host} (WDA gateway)` + : `${host}:${port}`; return { agent, @@ -145,8 +211,21 @@ export const iosPlaygroundPlatform = definePlaygroundPlatform< }), displayName, metadata: { - wdaHost: host, - wdaPort: port, + ...(baseUrl + ? { + wdaGatewayId: `ios-gateway-${sha256Hex( + JSON.stringify({ baseUrl, mjpegUrl, mjpegPort, sessionId }), + )}`, + } + : {}), + wdaHost: gatewayUrl?.hostname ?? host, + wdaPort: gatewayUrl + ? Number( + gatewayUrl.port || + (gatewayUrl.protocol === 'https:' ? 443 : 80), + ) + : port, + ...(mjpegPort !== undefined ? { wdaMjpegPort: mjpegPort } : {}), ...(sessionId ? { sessionId } : {}), ...(deviceInfo ? { deviceInfo } : {}), }, diff --git a/packages/ios/src/wda-options.ts b/packages/ios/src/wda-options.ts new file mode 100644 index 0000000000..81c6372583 --- /dev/null +++ b/packages/ios/src/wda-options.ts @@ -0,0 +1,48 @@ +import type { IOSDeviceOpt } from '@midscene/core/device'; + +export function assertWdaConnectionOptions( + options?: Pick< + IOSDeviceOpt, + 'wdaBaseUrl' | 'wdaHost' | 'wdaPort' | 'wdaMjpegUrl' | 'wdaMjpegPort' + >, +): void { + if ( + options?.wdaBaseUrl !== undefined && + (options.wdaHost !== undefined || options.wdaPort !== undefined) + ) { + throw new Error( + 'wdaBaseUrl cannot be used with wdaHost or wdaPort. Choose one WDA connection method.', + ); + } + if ( + options?.wdaMjpegUrl !== undefined && + options.wdaMjpegPort !== undefined + ) { + throw new Error( + 'wdaMjpegUrl cannot be used with wdaMjpegPort. Choose one MJPEG connection method.', + ); + } + if (options?.wdaMjpegUrl !== undefined) { + normalizeMjpegStreamUrl(options.wdaMjpegUrl); + } +} + +export function normalizeMjpegStreamUrl(value: string): string { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error('wdaMjpegUrl must be an absolute HTTP(S) URL.'); + } + if ( + (url.protocol !== 'http:' && url.protocol !== 'https:') || + url.username || + url.password || + value.includes('#') + ) { + throw new Error( + 'wdaMjpegUrl must use HTTP(S) without credentials or a fragment.', + ); + } + return url.toString(); +} diff --git a/packages/ios/tests/ai/ios-simulator-smoke.test.ts b/packages/ios/tests/ai/ios-simulator-smoke.test.ts index 20f256ec9b..3524b28fa1 100644 --- a/packages/ios/tests/ai/ios-simulator-smoke.test.ts +++ b/packages/ios/tests/ai/ios-simulator-smoke.test.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { type Server, createServer } from 'node:http'; +import { type Server, createServer, request as httpRequest } from 'node:http'; import path from 'node:path'; import { parseDumpScript } from '@midscene/core'; import { @@ -338,6 +338,67 @@ async function isSoftwareKeyboardVisible(agent: IOSAgent): Promise { } describe.skipIf(!RUN_LIVE_SMOKE)('iOS Simulator live smoke', () => { + it('connects through a path-prefixed WDA gateway', async () => { + const prefix = '/device/ios/wda'; + const seenPaths: string[] = []; + const gateway = createServer((incoming, outgoing) => { + const incomingUrl = new URL(incoming.url || '/', 'http://127.0.0.1'); + if (!incomingUrl.pathname.startsWith(`${prefix}/`)) { + outgoing.writeHead(404).end(); + return; + } + + const upstreamPath = `${incomingUrl.pathname.slice(prefix.length)}${incomingUrl.search}`; + seenPaths.push(incomingUrl.pathname); + const upstream = httpRequest( + { + hostname: '127.0.0.1', + port: 8100, + method: incoming.method, + path: upstreamPath, + headers: incoming.headers, + }, + (response) => { + outgoing.writeHead(response.statusCode || 502, response.headers); + response.pipe(outgoing); + }, + ); + upstream.on('error', (error) => outgoing.destroy(error)); + incoming.pipe(upstream); + }); + await new Promise((resolve) => + gateway.listen(0, '127.0.0.1', resolve), + ); + const address = gateway.address(); + if (!address || typeof address === 'string') { + throw new Error('Failed to start local WDA gateway'); + } + + let agent: IOSAgent | undefined; + try { + agent = await agentFromWebDriverAgent({ + wdaBaseUrl: `http://127.0.0.1:${address.port}${prefix}`, + autoPrintReportMsg: false, + generateReport: false, + }); + const screenshot = await agent.interface.screenshotBase64(); + expect(screenshot).toMatch(/^data:image\/\w+;base64,/); + expect(seenPaths).toContain(`${prefix}/status`); + expect(seenPaths).toContain(`${prefix}/session`); + expect( + seenPaths.some((value) => + new RegExp(`^${prefix}/session/[^/]+/screenshot$`).test(value), + ), + ).toBe(true); + } finally { + try { + await agent?.destroy(); + } finally { + await closeServer(gateway); + } + } + }); + it('awaits auto-dismiss before a second input starts', async () => { const diagnosticsEnv = process.env.MIDSCENE_IOS_DIAGNOSTICS_DIR; if (!diagnosticsEnv) { @@ -379,6 +440,7 @@ describe.skipIf(!RUN_LIVE_SMOKE)('iOS Simulator live smoke', () => { evidence.fixtureUrl = fixture.url; agent = await agentFromWebDriverAgent({ wdaHost: '127.0.0.1', + wdaPort: 8100, modelConfig: { [MIDSCENE_MODEL_NAME]: 'ios-ci-model-must-not-run', [MIDSCENE_MODEL_API_KEY]: 'ios-ci-unused-key', diff --git a/packages/ios/tests/unit-test/agent-tools.test.ts b/packages/ios/tests/unit-test/agent-tools.test.ts index abd937b13b..5b461579d0 100644 --- a/packages/ios/tests/unit-test/agent-tools.test.ts +++ b/packages/ios/tests/unit-test/agent-tools.test.ts @@ -49,8 +49,8 @@ describe('IOSMidsceneTools', () => { await takeScreenshotTool?.handler({ ios: { - 'wda-host': '127.0.0.1', - 'wda-port': 8100, + 'wda-base-url': 'https://gateway.example/code/wda', + 'wda-mjpeg-url': 'https://stream.example/live/mjpeg', sessionId: 'external-session-id', waitAfterAction: 650, replanningCycleLimit: 12, @@ -62,8 +62,8 @@ describe('IOSMidsceneTools', () => { expect(agentFromWebDriverAgent).toHaveBeenCalledWith( expect.objectContaining({ autoDismissKeyboard: false, - wdaHost: '127.0.0.1', - wdaPort: 8100, + wdaBaseUrl: 'https://gateway.example/code/wda', + wdaMjpegUrl: 'https://stream.example/live/mjpeg', sessionId: 'external-session-id', waitAfterAction: 650, replanningCycleLimit: 12, @@ -73,6 +73,42 @@ describe('IOSMidsceneTools', () => { ); }); + it('rejects conflicting WDA connection options before creating an agent', async () => { + const tools = new IOSMidsceneTools(); + await tools.initTools(); + const connectTool = tools + .getToolDefinitions() + .find((tool) => tool.name === 'ios_connect'); + + await expect( + connectTool?.handler({ + ios: { + wdaBaseUrl: 'https://gateway.example/code/wda', + wdaHost: 'localhost', + }, + }), + ).rejects.toThrow(/wdaBaseUrl cannot be used with wdaHost or wdaPort/); + expect(agentFromWebDriverAgent).not.toHaveBeenCalled(); + }); + + it('rejects conflicting MJPEG connection options before creating an agent', async () => { + const tools = new IOSMidsceneTools(); + await tools.initTools(); + const connectTool = tools + .getToolDefinitions() + .find((tool) => tool.name === 'ios_connect'); + + await expect( + connectTool?.handler({ + ios: { + wdaMjpegUrl: 'https://stream.example/live/mjpeg', + wdaMjpegPort: 9100, + }, + }), + ).rejects.toThrow(/wdaMjpegUrl cannot be used with wdaMjpegPort/); + expect(agentFromWebDriverAgent).not.toHaveBeenCalled(); + }); + it('passes top-level ios aliases to act', async () => { const mockAgent = createMockAgent(); rs.mocked(agentFromWebDriverAgent).mockResolvedValue(mockAgent as any); @@ -104,6 +140,32 @@ describe('IOSMidsceneTools', () => { }); }); + it('keeps gateway path identifiers out of persisted report identities', async () => { + const tools = new IOSMidsceneTools(); + await tools.initTools(); + const createReportSession = rs + .spyOn(tools as any, 'createNewCliReportSession') + .mockReturnValue({}); + rs.spyOn(tools as any, 'commitCliReportSession').mockImplementation( + () => {}, + ); + + const connectTool = tools + .getToolDefinitions() + .find((tool) => tool.name === 'ios_connect'); + await connectTool?.handler({ + ios: { + wdaBaseUrl: 'https://gateway.example/secret-code/wda', + wdaMjpegUrl: 'https://stream.example/live/mjpeg?token=stream-secret', + }, + }); + + const identity = createReportSession.mock.calls[0][0] as string; + expect(identity).toMatch(/^wda-[a-f0-9]{12}$/); + expect(identity).not.toContain('secret-code'); + expect(identity).not.toContain('stream-secret'); + }); + it('exposes ios init args on action and common tool schemas', async () => { const tools = new IOSMidsceneTools(); await tools.initTools(); @@ -119,6 +181,8 @@ describe('IOSMidsceneTools', () => { expect.objectContaining({ 'ios.wdaHost': expect.anything(), 'ios.wdaPort': expect.anything(), + 'ios.wdaBaseUrl': expect.anything(), + 'ios.wdaMjpegUrl': expect.anything(), 'ios.sessionId': expect.anything(), 'ios.waitAfterAction': expect.anything(), 'ios.replanningCycleLimit': expect.anything(), @@ -129,6 +193,8 @@ describe('IOSMidsceneTools', () => { expect.objectContaining({ 'ios.wdaHost': expect.anything(), 'ios.wdaPort': expect.anything(), + 'ios.wdaBaseUrl': expect.anything(), + 'ios.wdaMjpegUrl': expect.anything(), 'ios.sessionId': expect.anything(), 'ios.waitAfterAction': expect.anything(), }), diff --git a/packages/ios/tests/unit-test/device.test.ts b/packages/ios/tests/unit-test/device.test.ts index 3b1853d892..8c18fea4ee 100644 --- a/packages/ios/tests/unit-test/device.test.ts +++ b/packages/ios/tests/unit-test/device.test.ts @@ -147,6 +147,36 @@ describe('IOSDevice', () => { sessionId: 'external-session-id', }); }); + + it('passes the full WDA base URL to requests and readiness checks', () => { + new IOSDevice({ + wdaBaseUrl: 'https://gateway.example/code/wda', + }); + expect(MockedWdaClient).toHaveBeenCalledWith({ + port: DEFAULT_WDA_PORT, + host: 'localhost', + baseUrl: 'https://gateway.example/code/wda', + }); + expect(MockedWdaManager.getInstance).toHaveBeenCalledWith( + DEFAULT_WDA_PORT, + 'localhost', + 'https://gateway.example/code/wda', + ); + }); + + it.each([ + { wdaHost: 'localhost' }, + { wdaPort: 8100 }, + { wdaHost: 'localhost', wdaPort: 8100 }, + ])('rejects a base URL combined with host or port: %j', (connection) => { + expect( + () => + new IOSDevice({ + wdaBaseUrl: 'https://gateway.example/code/wda', + ...connection, + }), + ).toThrow(/wdaBaseUrl cannot be used with wdaHost or wdaPort/); + }); }); describe('Device Info', () => { diff --git a/packages/ios/tests/unit-test/mjpeg-frame-source.test.ts b/packages/ios/tests/unit-test/mjpeg-frame-source.test.ts index eb43381e94..6964eb4332 100644 --- a/packages/ios/tests/unit-test/mjpeg-frame-source.test.ts +++ b/packages/ios/tests/unit-test/mjpeg-frame-source.test.ts @@ -88,6 +88,45 @@ describe('IOSDevice MJPEG frame source opt-in', () => { expect(b.mjpegStreamUrl).toBe('http://127.0.0.1:9102'); }); + it('keeps a gateway API URL separate from the default local MJPEG stream', () => { + const device = new IOSDevice({ + wdaBaseUrl: 'https://gateway.example/code/wda', + }); + expect(device.mjpegStreamUrl).toBe('http://localhost:9100'); + }); + + it('uses an independent HTTPS MJPEG stream URL with a path and query', () => { + const device = new IOSDevice({ + wdaBaseUrl: 'https://gateway.example/code/wda', + wdaMjpegUrl: 'https://stream.example:8443/code/mjpeg?token=secret', + wdaMjpegFrameSource: { enabled: true }, + }); + expect(device.mjpegStreamUrl).toBe( + 'https://stream.example:8443/code/mjpeg?token=secret', + ); + expect(device.openFrameSource).toBeTypeOf('function'); + }); + + it('rejects conflicting or invalid MJPEG stream options without revealing the URL', () => { + const secretUrl = 'https://stream.example/private/mjpeg?token=secret'; + expect( + () => new IOSDevice({ wdaMjpegUrl: secretUrl, wdaMjpegPort: 9100 }), + ).toThrow(/wdaMjpegUrl cannot be used with wdaMjpegPort/); + for (const wdaMjpegUrl of [ + 'ftp://stream.example/mjpeg', + 'https://user:password@stream.example/mjpeg', + 'https://stream.example/mjpeg#', + 'not-a-url', + ]) { + expect(() => new IOSDevice({ wdaMjpegUrl })).toThrow(); + } + try { + new IOSDevice({ wdaMjpegUrl: secretUrl, wdaMjpegPort: 9100 }); + } catch (error) { + expect(String(error)).not.toContain('secret'); + } + }); + it('wraps the MJPEG stream as a frame source (identity decode, stop tears down)', async () => { const device = new IOSDevice({ wdaMjpegPort: 9123, @@ -156,4 +195,26 @@ describe('MjpegFrameSource', () => { (globalThis as any).fetch = originalFetch; } }); + + it('does not include a signed stream URL in a startup timeout', async () => { + const originalFetch = globalThis.fetch; + (globalThis as any).fetch = async () => + ({ ok: false, status: 503, body: null }) as Response; + let now = 0; + const source = new MjpegFrameSource( + 'https://stream.example/live/mjpeg?token=secret', + () => { + now += 1000; + return now; + }, + ); + try { + await expect(source.ensureStarted(3000)).rejects.toThrow( + 'no frame received within 3000ms', + ); + } finally { + source.stop(); + (globalThis as any).fetch = originalFetch; + } + }); }); diff --git a/packages/ios/tests/unit-test/platform-session-manager.test.ts b/packages/ios/tests/unit-test/platform-session-manager.test.ts index c78c90446a..433239de65 100644 --- a/packages/ios/tests/unit-test/platform-session-manager.test.ts +++ b/packages/ios/tests/unit-test/platform-session-manager.test.ts @@ -41,10 +41,15 @@ describe('iosPlaygroundPlatform session manager', () => { const setup = await prepared.sessionManager!.getSetupSchema!(); expect(setup?.fields).toMatchObject([ - { key: 'host', defaultValue: 'localhost' }, - { key: 'port', defaultValue: 8100 }, + { key: 'host', required: false }, + { key: 'port', required: false }, + { key: 'baseUrl', required: false }, + { key: 'mjpegUrl', required: false }, + { key: 'mjpegPort', required: false }, { key: 'sessionId', required: false }, ]); + expect(setup?.fields?.[0]).not.toHaveProperty('defaultValue'); + expect(setup?.fields?.[1]).not.toHaveProperty('defaultValue'); const created = await prepared.sessionManager?.createSession({ host: 'localhost', @@ -65,6 +70,120 @@ describe('iosPlaygroundPlatform session manager', () => { }); }); + test('passes a normalized gateway base URL to the agent and session factory', async () => { + const { iosPlaygroundPlatform } = await import('../../src/platform'); + const prepared = await iosPlaygroundPlatform.prepare({}); + const created = await prepared.sessionManager?.createSession({ + baseUrl: 'https://gateway.example/code/wda/', + }); + await created?.agentFactory?.(); + + expect(agentFromWebDriverAgentMock).toHaveBeenNthCalledWith(1, { + wdaBaseUrl: 'https://gateway.example/code/wda', + }); + expect(agentFromWebDriverAgentMock).toHaveBeenNthCalledWith(2, { + wdaBaseUrl: 'https://gateway.example/code/wda', + }); + }); + + test.each([ + { host: 'localhost' }, + { port: 8100 }, + { host: 'localhost', port: 8100 }, + ])('rejects gateway base URL with host or port input: %j', async (input) => { + const { iosPlaygroundPlatform } = await import('../../src/platform'); + const prepared = await iosPlaygroundPlatform.prepare({}); + + await expect( + prepared.sessionManager?.createSession({ + baseUrl: 'https://gateway.example/code/wda', + ...input, + }), + ).rejects.toThrow(/wdaBaseUrl cannot be used with wdaHost or wdaPort/); + expect(agentFromWebDriverAgentMock).not.toHaveBeenCalled(); + }); + + test('treats empty optional host and port fields as unset', async () => { + const { iosPlaygroundPlatform } = await import('../../src/platform'); + const prepared = await iosPlaygroundPlatform.prepare({}); + + await prepared.sessionManager?.createSession({ + baseUrl: 'https://gateway.example/code/wda', + host: '', + port: '', + }); + + expect(agentFromWebDriverAgentMock).toHaveBeenCalledWith({ + wdaBaseUrl: 'https://gateway.example/code/wda', + }); + }); + + test('passes a remote HTTPS MJPEG stream URL to the agent and its factory', async () => { + const { iosPlaygroundPlatform } = await import('../../src/platform'); + const prepared = await iosPlaygroundPlatform.prepare({}); + const created = await prepared.sessionManager?.createSession({ + baseUrl: 'https://gateway.example/code/wda', + mjpegUrl: 'https://stream.example/live/mjpeg?token=secret', + mjpegPort: '', + }); + await created?.agentFactory?.(); + + expect(agentFromWebDriverAgentMock).toHaveBeenNthCalledWith(1, { + wdaBaseUrl: 'https://gateway.example/code/wda', + wdaMjpegUrl: 'https://stream.example/live/mjpeg?token=secret', + }); + expect(agentFromWebDriverAgentMock).toHaveBeenNthCalledWith(2, { + wdaBaseUrl: 'https://gateway.example/code/wda', + wdaMjpegUrl: 'https://stream.example/live/mjpeg?token=secret', + }); + expect(JSON.stringify(created?.metadata)).not.toContain('secret'); + }); + + test('rejects a remote MJPEG URL combined with a port', async () => { + const { iosPlaygroundPlatform } = await import('../../src/platform'); + const prepared = await iosPlaygroundPlatform.prepare({}); + await expect( + prepared.sessionManager?.createSession({ + mjpegUrl: 'https://stream.example/live/mjpeg', + mjpegPort: 9100, + }), + ).rejects.toThrow(/wdaMjpegUrl cannot be used with wdaMjpegPort/); + expect(agentFromWebDriverAgentMock).not.toHaveBeenCalled(); + }); + + test('does not expose a gateway path identifier in session metadata', async () => { + getConnectedDeviceInfoMock.mockResolvedValue(null); + const { iosPlaygroundPlatform } = await import('../../src/platform'); + const prepared = await iosPlaygroundPlatform.prepare({}); + const created = await prepared.sessionManager?.createSession({ + baseUrl: 'https://gateway.example/secret-code/wda', + }); + + expect(created?.displayName).toBe('gateway.example (WDA gateway)'); + expect(created?.metadata).toMatchObject({ + wdaHost: 'gateway.example', + wdaPort: 443, + }); + expect(JSON.stringify(created?.metadata)).not.toContain('secret-code'); + expect(created?.metadata?.wdaGatewayId).toMatch( + /^ios-gateway-[a-f0-9]{64}$/, + ); + + const other = await prepared.sessionManager?.createSession({ + baseUrl: 'https://gateway.example/other-code/wda', + }); + expect(other?.metadata?.wdaGatewayId).not.toBe( + created?.metadata?.wdaGatewayId, + ); + const otherSession = await prepared.sessionManager?.createSession({ + baseUrl: 'https://gateway.example/secret-code/wda', + sessionId: 'session-2', + }); + expect(otherSession?.metadata?.wdaGatewayId).not.toBe( + created?.metadata?.wdaGatewayId, + ); + }); + test('reuses the agent factory for follow-up playground sessions', async () => { const { iosPlaygroundPlatform } = await import('../../src/platform'); const prepared = await iosPlaygroundPlatform.prepare({}); diff --git a/packages/playground/src/mjpeg-stream-handler.ts b/packages/playground/src/mjpeg-stream-handler.ts index db105b4751..112957813e 100644 --- a/packages/playground/src/mjpeg-stream-handler.ts +++ b/packages/playground/src/mjpeg-stream-handler.ts @@ -1,4 +1,5 @@ import http from 'node:http'; +import https from 'node:https'; import type { Agent as PageAgent } from '@midscene/core/agent'; import { getDebug } from '@midscene/shared/logger'; import type { Request, Response } from 'express'; @@ -29,6 +30,16 @@ function toMjpegFrameDataUrl(data: string, contentType?: string) { return `data:${contentType || 'image/jpeg'};base64,${data}`; } +function getNativeStream( + nativeUrl: string, + onResponse: (response: http.IncomingMessage) => void, +): http.ClientRequest { + const url = new URL(nativeUrl); + if (url.protocol === 'http:') return http.get(url, onResponse); + if (url.protocol === 'https:') return https.get(url, onResponse); + throw new Error('Unsupported native MJPEG URL protocol'); +} + /** * Inputs the handler reads on every request, late-bound through callbacks * so a single handler instance can survive across device reconnects without @@ -147,35 +158,44 @@ export class MjpegStreamHandler { res: Response, ): Promise { return new Promise((resolve) => { - debugMjpeg(`trying native stream from ${nativeUrl}`); - const proxyReq = http.get(nativeUrl, (proxyRes) => { - const statusCode = proxyRes.statusCode ?? 0; - if (statusCode >= 400) { - this.nativeAvailable = false; - this.nativeFailedAt = Date.now(); - proxyRes.resume(); - debugMjpeg( - `native stream returned HTTP ${statusCode}, using polling mode`, - ); - resolve(false); - return; - } - this.nativeAvailable = true; - this.nativeFailedAt = null; - debugMjpeg('streaming via native WDA MJPEG server'); - const contentType = proxyRes.headers['content-type']; - if (contentType) res.setHeader('Content-Type', contentType); - res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate'); - res.setHeader('Connection', 'keep-alive'); - proxyRes.pipe(res); - req.on('close', () => proxyReq.destroy()); - resolve(true); - }); + debugMjpeg('trying native MJPEG stream'); + let proxyReq: http.ClientRequest; + try { + proxyReq = getNativeStream(nativeUrl, (proxyRes) => { + const statusCode = proxyRes.statusCode ?? 0; + if (statusCode >= 400) { + this.nativeAvailable = false; + this.nativeFailedAt = Date.now(); + proxyRes.resume(); + debugMjpeg( + `native stream returned HTTP ${statusCode}, using polling mode`, + ); + resolve(false); + return; + } + this.nativeAvailable = true; + this.nativeFailedAt = null; + debugMjpeg('streaming via native WDA MJPEG server'); + const contentType = proxyRes.headers['content-type']; + if (contentType) res.setHeader('Content-Type', contentType); + res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate'); + res.setHeader('Connection', 'keep-alive'); + proxyRes.pipe(res); + req.on('close', () => proxyReq.destroy()); + resolve(true); + }); + } catch { + this.nativeAvailable = false; + this.nativeFailedAt = Date.now(); + debugMjpeg('native stream URL is invalid, using polling mode'); + resolve(false); + return; + } proxyReq.on('error', (err) => { this.nativeAvailable = false; this.nativeFailedAt = Date.now(); debugMjpeg( - `native stream unavailable (${err.message}), using polling mode`, + `native stream unavailable (${(err as NodeJS.ErrnoException).code ?? err.name}), using polling mode`, ); resolve(false); }); @@ -184,12 +204,18 @@ export class MjpegStreamHandler { private probeNativeLiveness(nativeUrl: string): Promise { return new Promise((resolve) => { - const probe = http.get(nativeUrl, (probeRes) => { - const statusCode = probeRes.statusCode ?? 0; - const reachable = statusCode >= 200 && statusCode < 400; - probeRes.destroy(); - resolve(reachable); - }); + let probe: http.ClientRequest; + try { + probe = getNativeStream(nativeUrl, (probeRes) => { + const statusCode = probeRes.statusCode ?? 0; + const reachable = statusCode >= 200 && statusCode < 400; + probeRes.destroy(); + resolve(reachable); + }); + } catch { + resolve(false); + return; + } probe.setTimeout(1000, () => { probe.destroy(); resolve(false); diff --git a/packages/playground/tests/unit/mjpeg-stream-handler.test.ts b/packages/playground/tests/unit/mjpeg-stream-handler.test.ts new file mode 100644 index 0000000000..a08b2fd9bc --- /dev/null +++ b/packages/playground/tests/unit/mjpeg-stream-handler.test.ts @@ -0,0 +1,120 @@ +import { EventEmitter } from 'node:events'; +import http from 'node:http'; +import https from 'node:https'; +import { afterEach, describe, expect, rs, test } from '@rstest/core'; +import { MjpegStreamHandler } from '../../src/mjpeg-stream-handler'; + +describe('MjpegStreamHandler native stream', () => { + afterEach(() => { + rs.restoreAllMocks(); + }); + + test('proxies a native HTTPS MJPEG URL with its path and query intact', async () => { + const nativeUrl = 'https://stream.example:8443/live/mjpeg?token=secret'; + const request = new EventEmitter() as any; + const response = { setHeader: rs.fn() } as any; + const nativeRequest = new EventEmitter(); + const nativeResponse = { + statusCode: 200, + headers: { 'content-type': 'multipart/x-mixed-replace' }, + pipe: rs.fn(), + }; + const httpsGet = rs + .spyOn(https, 'get') + .mockImplementation((_url: any, onResponse: any) => { + onResponse(nativeResponse); + return nativeRequest as any; + }); + const handler = new MjpegStreamHandler({ + getNativeUrl: () => nativeUrl, + getActiveInterface: () => null, + takeScreenshot: rs.fn(), + canTakeScreenshot: () => true, + isAgentReady: () => true, + }); + + await handler.serve(request, response); + + expect(httpsGet).toHaveBeenCalledWith( + new URL(nativeUrl), + expect.any(Function), + ); + expect(nativeResponse.pipe).toHaveBeenCalledWith(response); + expect(response.setHeader).toHaveBeenCalledWith( + 'Content-Type', + 'multipart/x-mixed-replace', + ); + handler.shutdown(); + }); + + test('keeps the existing HTTP stream path working', async () => { + const nativeUrl = 'http://localhost:9100/'; + const nativeRequest = new EventEmitter(); + const nativeResponse = { + statusCode: 200, + headers: {}, + pipe: rs.fn(), + }; + const httpGet = rs + .spyOn(http, 'get') + .mockImplementation((_url: any, onResponse: any) => { + onResponse(nativeResponse); + return nativeRequest as any; + }); + const handler = new MjpegStreamHandler({ + getNativeUrl: () => nativeUrl, + getActiveInterface: () => null, + takeScreenshot: rs.fn(), + canTakeScreenshot: () => true, + isAgentReady: () => true, + }); + + await handler.serve( + new EventEmitter() as any, + { setHeader: rs.fn() } as any, + ); + + expect(httpGet).toHaveBeenCalledWith( + new URL(nativeUrl), + expect.any(Function), + ); + handler.shutdown(); + }); + + test('falls back to screenshot polling when an HTTPS stream cannot connect', async () => { + const request = Object.assign(new EventEmitter(), { query: {} }) as any; + const response = { + setHeader: rs.fn(), + write: rs.fn(() => { + request.emit('close'); + return true; + }), + } as any; + const nativeRequest = new EventEmitter(); + rs.spyOn(https, 'get').mockImplementation(() => { + queueMicrotask(() => + nativeRequest.emit( + 'error', + Object.assign(new Error('secret URL'), { + code: 'ECONNREFUSED', + }), + ), + ); + return nativeRequest as any; + }); + const takeScreenshot = rs.fn().mockResolvedValue('FRAME'); + const handler = new MjpegStreamHandler({ + getNativeUrl: () => 'https://stream.example/live/mjpeg?token=secret', + getActiveInterface: () => null, + takeScreenshot, + canTakeScreenshot: () => true, + isAgentReady: () => true, + }); + + await handler.serve(request, response); + + expect(takeScreenshot).toHaveBeenCalledTimes(1); + expect(response.write).toHaveBeenCalled(); + handler.shutdown(); + }); +}); diff --git a/packages/shared/src/utils.ts b/packages/shared/src/utils.ts index dec7ef4eb3..7bd47723d0 100644 --- a/packages/shared/src/utils.ts +++ b/packages/shared/src/utils.ts @@ -12,6 +12,10 @@ export function uuid(): string { return generateUUID(); } +export function sha256Hex(value: string): string { + return sha256(value); +} + const hashMap: Record = {}; // id - combined export function generateHashId(rect: any, content = ''): string { diff --git a/packages/test/src/cli/create-template.ts b/packages/test/src/cli/create-template.ts index 190851723d..9f001b298d 100644 --- a/packages/test/src/cli/create-template.ts +++ b/packages/test/src/cli/create-template.ts @@ -48,13 +48,28 @@ const platformSetup: Record = { android: ` const agent = await agentFromAdbDevice(env.ANDROID_DEVICE_ID || undefined); onTeardown(() => agent.destroy()); return { agent };`, - ios: ` const port = Number(env.WDA_PORT || 8100); + ios: ` const wdaBaseUrl = env.WDA_BASE_URL || undefined; + if (wdaBaseUrl && (env.WDA_HOST !== undefined || env.WDA_PORT !== undefined)) { + throw new Error('WDA_BASE_URL cannot be used with WDA_HOST or WDA_PORT.'); + } + const port = Number(env.WDA_PORT || 8100); if (!Number.isInteger(port) || port < 1 || port > 65535) { throw new Error('WDA_PORT must be an integer between 1 and 65535.'); } + const wdaMjpegUrl = env.WDA_MJPEG_URL || undefined; + if (wdaMjpegUrl && env.WDA_MJPEG_PORT !== undefined) { + throw new Error('WDA_MJPEG_URL cannot be used with WDA_MJPEG_PORT.'); + } + const mjpegPort = env.WDA_MJPEG_PORT === undefined ? undefined : Number(env.WDA_MJPEG_PORT); + if (mjpegPort !== undefined && (!Number.isInteger(mjpegPort) || mjpegPort < 1 || mjpegPort > 65535)) { + throw new Error('WDA_MJPEG_PORT must be an integer between 1 and 65535.'); + } const agent = await agentFromWebDriverAgent({ - wdaHost: env.WDA_HOST || 'localhost', - wdaPort: port, + ...(wdaBaseUrl + ? { wdaBaseUrl } + : { wdaHost: env.WDA_HOST || 'localhost', wdaPort: port }), + ...(wdaMjpegUrl ? { wdaMjpegUrl } : {}), + ...(mjpegPort !== undefined ? { wdaMjpegPort: mjpegPort } : {}), }); onTeardown(() => agent.destroy()); return { agent };`, @@ -72,7 +87,7 @@ const platformEnv: Record = { web: 'HEADLESS=true\n', android: '# Optional: choose a device from adb devices.\nANDROID_DEVICE_ID=\n', - ios: 'WDA_HOST=localhost\nWDA_PORT=8100\n', + ios: '# WDA_HOST=localhost\n# WDA_PORT=8100\n# WDA_BASE_URL=https://gateway.example/device/wda\n# WDA_MJPEG_PORT=9100\n# WDA_MJPEG_URL=https://gateway.example/device/mjpeg\n', harmony: '# Optional: choose a device from hdc list targets.\nHARMONY_DEVICE_ID=\n', computer: @@ -94,7 +109,7 @@ MIDSCENE_MODEL_FAMILY= const platformInstructions: Record, string> = { android: 'Connect an Android device and verify it with `adb devices`. Set ANDROID_DEVICE_ID to select a device.', - ios: 'Start WebDriverAgent and set WDA_HOST and WDA_PORT for your iOS device.', + ios: 'Start WebDriverAgent and set WDA_HOST and WDA_PORT, or set WDA_BASE_URL for a gateway URL with a path prefix. Set WDA_MJPEG_URL for a separate remote MJPEG stream, or WDA_MJPEG_PORT for a direct connection.', harmony: 'Connect a HarmonyOS device and verify it with `hdc list targets`. Set HARMONY_DEVICE_ID to select a device. Set HDC_HOME if hdc is not on PATH.', computer: diff --git a/packages/webdriver/src/clients/WebDriverClient.ts b/packages/webdriver/src/clients/WebDriverClient.ts index bf01d77240..0a4bac0651 100644 --- a/packages/webdriver/src/clients/WebDriverClient.ts +++ b/packages/webdriver/src/clients/WebDriverClient.ts @@ -1,5 +1,6 @@ import { DEFAULT_WDA_PORT } from '@midscene/shared/constants'; import { getDebug } from '@midscene/shared/logger'; +import { normalizeWebDriverBaseUrl } from '../utils/base-url'; import { makeWebDriverRequest } from '../utils/request'; import type { DeviceInfo, @@ -23,11 +24,14 @@ export class WebDriverClient { this.port = options.port || DEFAULT_WDA_PORT; this.host = options.host || 'localhost'; this.timeout = options.timeout || 30000; - this.baseUrl = `http://${this.host}:${this.port}`; + this.baseUrl = + options.baseUrl !== undefined + ? normalizeWebDriverBaseUrl(options.baseUrl) + : `http://${this.host}:${this.port}`; this.sessionId = options.sessionId || null; this.ownsSession = !options.sessionId; - debugClient(`Initialized WebDriver client on ${this.host}:${this.port}`); + debugClient('Initialized WebDriver client'); } get sessionInfo(): WDASession | null { diff --git a/packages/webdriver/src/clients/types.ts b/packages/webdriver/src/clients/types.ts index 9a6d7a02a8..6d43fc844b 100644 --- a/packages/webdriver/src/clients/types.ts +++ b/packages/webdriver/src/clients/types.ts @@ -26,6 +26,8 @@ export interface WDAElementInfo { } export interface WebDriverOptions { + /** Full WebDriver API base URL, including any gateway path prefix. */ + baseUrl?: string; port?: number; host?: string; timeout?: number; diff --git a/packages/webdriver/src/index.ts b/packages/webdriver/src/index.ts index 9fe5fbba67..6f8f45fd07 100644 --- a/packages/webdriver/src/index.ts +++ b/packages/webdriver/src/index.ts @@ -22,3 +22,4 @@ export type { WDAConfig } from './managers/WDAManager'; // Utilities export { makeWebDriverRequest, WebDriverRequestError } from './utils/request'; +export { normalizeWebDriverBaseUrl } from './utils/base-url'; diff --git a/packages/webdriver/src/managers/WDAManager.ts b/packages/webdriver/src/managers/WDAManager.ts index 3bc569f0b2..4f92f4cd4d 100644 --- a/packages/webdriver/src/managers/WDAManager.ts +++ b/packages/webdriver/src/managers/WDAManager.ts @@ -2,6 +2,7 @@ import { exec } from 'node:child_process'; import { promisify } from 'node:util'; import { DEFAULT_WDA_PORT } from '@midscene/shared/constants'; import { getDebug } from '@midscene/shared/logger'; +import { normalizeWebDriverBaseUrl } from '../utils/base-url'; import { BaseServiceManager } from './ServiceManager'; const execAsync = promisify(exec); @@ -10,6 +11,7 @@ const debugWDA = getDebug('webdriver:wda-manager'); export interface WDAConfig { port: number; host?: string; + baseUrl?: string; wdaPath?: string; bundleId?: string; usePrebuiltWDA?: boolean; @@ -17,40 +19,48 @@ export interface WDAConfig { export class WDAManager extends BaseServiceManager { private static instances = new Map(); - private config: WDAConfig; + private readonly baseUrl: string; private isStarted = false; private constructor(config: WDAConfig) { - super(config.port, config.host); - this.config = { - bundleId: 'com.apple.WebDriverAgentRunner.xctrunner', - usePrebuiltWDA: true, - host: 'localhost', - ...config, - port: config.port || DEFAULT_WDA_PORT, - }; + const address = config.baseUrl ? new URL(config.baseUrl) : undefined; + super( + address + ? Number(address.port || (address.protocol === 'https:' ? 443 : 80)) + : config.port, + address?.hostname ?? config.host, + ); + this.baseUrl = config.baseUrl ?? super.getEndpoint(); } - static getInstance(port = DEFAULT_WDA_PORT, host?: string): WDAManager { - const key = `${host || 'localhost'}:${port}`; + static getInstance( + port = DEFAULT_WDA_PORT, + host?: string, + baseUrl?: string, + ): WDAManager { + const key = + baseUrl !== undefined + ? normalizeWebDriverBaseUrl(baseUrl) + : `http://${host || 'localhost'}:${port}`; if (!WDAManager.instances.has(key)) { - WDAManager.instances.set(key, new WDAManager({ port, host })); + WDAManager.instances.set( + key, + new WDAManager({ port, host, baseUrl: key }), + ); } return WDAManager.instances.get(key)!; } async start(): Promise { if (this.isStarted) { - debugWDA( - `WDA already started on ${this.config.host}:${this.config.port}`, - ); + debugWDA('WDA already started'); return; } try { - // Check if WDA is already running on the port + // Check if WDA is already reachable at the configured API base URL if (await this.isWDARunning()) { - debugWDA(`WDA already running on port ${this.config.port}`); + debugWDA('WDA already running'); this.isStarted = true; return; } @@ -65,9 +75,7 @@ export class WDAManager extends BaseServiceManager { await this.waitForWDA(); this.isStarted = true; - debugWDA( - `WDA started successfully on ${this.config.host}:${this.config.port}`, - ); + debugWDA('WDA started successfully'); } catch (error) { debugWDA(`Failed to start WDA: ${error}`); throw new Error(`Failed to start WebDriverAgent: ${error}`); @@ -81,7 +89,7 @@ export class WDAManager extends BaseServiceManager { try { this.isStarted = false; - debugWDA(`WDA stopped on ${this.config.host}:${this.config.port}`); + debugWDA('WDA stopped'); } catch (error) { debugWDA(`Error stopping WDA: ${error}`); // Don't throw, cleanup should be best-effort @@ -92,6 +100,10 @@ export class WDAManager extends BaseServiceManager { return this.isStarted; } + override getEndpoint(): string { + return this.baseUrl; + } + private async startWDA(): Promise { // We require WebDriverAgent to be started manually await this.checkWDAPreparation(); @@ -99,22 +111,22 @@ export class WDAManager extends BaseServiceManager { } private async checkWDAPreparation(): Promise { - // Check if WebDriverAgent is already running on the expected port + // Check if WebDriverAgent is reachable at the configured API base URL if (await this.isWDARunning()) { - debugWDA(`WebDriverAgent is already running on port ${this.config.port}`); + debugWDA('WebDriverAgent is already running'); return; } // If not running, throw error with setup instructions throw new Error( - `WebDriverAgent is not running on ${this.config.host}:${this.config.port}. Please start WebDriverAgent manually: + `WebDriverAgent is not reachable at the configured address. Please start WebDriverAgent manually or check the gateway route: 🔧 Setup Instructions: 1. Install WebDriverAgent: npm install appium-webdriveragent 2. Build and run WebDriverAgent: - For simulators: Use Xcode to run WebDriverAgentRunner on your target simulator - For real devices: Build WebDriverAgentRunner and install on your device -3. Ensure WebDriverAgent is listening on ${this.config.host}:${this.config.port} +3. Ensure WebDriverAgent is reachable at the configured address 💡 Alternative: You can also specify a different host/port where WebDriverAgent is running.`, ); @@ -122,7 +134,7 @@ export class WDAManager extends BaseServiceManager { private async isWDARunning(): Promise { try { - const url = `http://${this.config.host}:${this.config.port}/status`; + const url = `${this.getEndpoint()}/status`; const response = await fetch(url); if (!response.ok) { diff --git a/packages/webdriver/src/utils/base-url.ts b/packages/webdriver/src/utils/base-url.ts new file mode 100644 index 0000000000..28289f880d --- /dev/null +++ b/packages/webdriver/src/utils/base-url.ts @@ -0,0 +1,26 @@ +/** Normalize a WebDriver server URL before appending API endpoint paths. */ +export function normalizeWebDriverBaseUrl(baseUrl: string): string { + let url: URL; + try { + url = new URL(baseUrl); + } catch { + throw new Error('Invalid WebDriver base URL: expected a full HTTP(S) URL'); + } + + if ( + (url.protocol !== 'http:' && url.protocol !== 'https:') || + !url.hostname || + url.username || + url.password || + baseUrl.includes('?') || + baseUrl.includes('#') || + url.search || + url.hash + ) { + throw new Error( + 'WebDriver base URL must be an HTTP(S) URL without credentials, query, or fragment', + ); + } + + return url.toString().replace(/\/+$/, ''); +} diff --git a/packages/webdriver/src/utils/request.ts b/packages/webdriver/src/utils/request.ts index 9a8c5d785f..8390bbb3f0 100644 --- a/packages/webdriver/src/utils/request.ts +++ b/packages/webdriver/src/utils/request.ts @@ -30,7 +30,7 @@ export async function makeWebDriverRequest( const url = `${baseUrl}${endpoint}`; debugRequest( - `${method} ${url}${data ? ` with data: ${JSON.stringify(data)}` : ''}`, + `${method} ${endpoint}${data ? ` with data: ${JSON.stringify(data)}` : ''}`, ); const controller = new AbortController(); diff --git a/packages/webdriver/tests/unit-test/base-url.test.ts b/packages/webdriver/tests/unit-test/base-url.test.ts new file mode 100644 index 0000000000..1cda9a8bc6 --- /dev/null +++ b/packages/webdriver/tests/unit-test/base-url.test.ts @@ -0,0 +1,90 @@ +import { afterEach, describe, expect, it, rs } from '@rstest/core'; +import { WebDriverClient } from '../../src/clients/WebDriverClient'; +import { WDAManager } from '../../src/managers/WDAManager'; +import { normalizeWebDriverBaseUrl } from '../../src/utils/base-url'; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +describe('WebDriver base URL', () => { + it.each(['/code/wda', '/wda/code'])( + 'preserves the gateway prefix %s for session and command requests', + async (prefix) => { + const urls: string[] = []; + globalThis.fetch = rs.fn(async (input) => { + urls.push(String(input)); + return new Response( + JSON.stringify({ sessionId: 'test-session', value: 'screenshot' }), + { headers: { 'content-type': 'application/json' } }, + ); + }) as typeof fetch; + + const client = new WebDriverClient({ + baseUrl: `https://gateway.example:8100${prefix}/`, + }); + await client.getDeviceInfo(); + await client.createSession(); + await client.takeScreenshot(); + await client.deleteSession(); + + expect(urls).toEqual([ + `https://gateway.example:8100${prefix}/status`, + `https://gateway.example:8100${prefix}/session`, + `https://gateway.example:8100${prefix}/session/test-session/screenshot`, + `https://gateway.example:8100${prefix}/session/test-session`, + ]); + }, + ); + + it('uses the same base URL for readiness and keeps different prefixes separate', async () => { + const urls: string[] = []; + globalThis.fetch = rs.fn(async (input) => { + urls.push(String(input)); + return new Response(JSON.stringify({ sessionId: null }), { + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + + const first = WDAManager.getInstance( + 8100, + 'gateway.example', + 'https://gateway.example:8100/code/wda/', + ); + const same = WDAManager.getInstance( + 8100, + 'gateway.example', + 'https://gateway.example:8100/code/wda', + ); + const second = WDAManager.getInstance( + 8100, + 'gateway.example', + 'https://gateway.example:8100/wda/code', + ); + + expect(same).toBe(first); + expect(second).not.toBe(first); + expect(first.getHost()).toBe('gateway.example'); + expect(first.getPort()).toBe(8100); + await first.start(); + await second.start(); + expect(urls).toEqual([ + 'https://gateway.example:8100/code/wda/status', + 'https://gateway.example:8100/wda/code/status', + ]); + }); + + it.each([ + '', + 'ftp://gateway.example/wda', + 'https://gateway.example/wda?token=abc', + 'https://gateway.example/wda?', + 'https://gateway.example/wda#', + 'https://user:pass@gateway.example/wda', + 'not a URL', + ])('rejects an unsupported base URL: %s', (value) => { + expect(() => normalizeWebDriverBaseUrl(value)).toThrow(); + }); +});