diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 3189e37c4..1b7290c4e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -13,10 +13,6 @@ concurrency: group: bench-${{ github.head_ref }} cancel-in-progress: true -permissions: - contents: read - pull-requests: write - issues: write jobs: bench: @@ -34,33 +30,65 @@ jobs: path: base - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + + - name: Install PR benchmark dependencies + working-directory: pr + run: bun install --frozen-lockfile + - - name: Run bench on base + - name: Detect base replay/parser benchmark + id: base-benchmark + run: | + if [ -f base/test/benchmarks/replay-parser.bench.ts ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + + - name: Run benchmarks on base working-directory: base run: | - bun install --frozen-lockfile + bun install --no-save --ignore-scripts bun run test/benchmarks/pipeline.bench.ts mv bench-results.json ../base-results.json + if [ -f test/benchmarks/replay-parser.bench.ts ]; then + bun test/benchmarks/replay-parser.bench.ts --no-enforce --output=../base-telemetry-results.json + fi - - name: Run bench on PR + - name: Run benchmarks on PR working-directory: pr run: | - bun install --frozen-lockfile bun run test/benchmarks/pipeline.bench.ts mv bench-results.json ../pr-results.json + bun test/benchmarks/replay-parser.bench.ts --output=../pr-telemetry-results.json - name: Compare id: compare working-directory: pr run: | - bun scripts/quality/bench-compare.ts ../base-results.json ../pr-results.json --threshold=5 > ../comparison.md - cat ../comparison.md + bun scripts/quality/bench-compare.ts ../base-results.json ../pr-results.json --threshold=5 > ../pipeline-comparison.md + if [ -f ../base-telemetry-results.json ]; then + bun scripts/quality/replay-parser-bench-compare.ts ../base-telemetry-results.json ../pr-telemetry-results.json > ../telemetry-comparison.md + else + { + echo "## Replay/parser benchmark budgets" + echo + echo "Base commit has no replay/parser benchmark. Current hard memory budgets passed; same-machine throughput comparison starts after this benchmark lands on the base branch." + } > ../telemetry-comparison.md + fi + { + cat ../pipeline-comparison.md + echo + cat ../telemetry-comparison.md + } | tee ../comparison.md + cat ../comparison.md >> "$GITHUB_STEP_SUMMARY" { echo 'body<> "$GITHUB_OUTPUT" - - name: Find existing comment id: find uses: peter-evans/find-comment@v3 @@ -75,3 +103,23 @@ jobs: comment-id: ${{ steps.find.outputs.comment-id }} body: ${{ steps.compare.outputs.body }} edit-mode: replace + + - name: Enforce replay/parser budgets + if: steps.base-benchmark.outputs.available == 'true' + working-directory: pr + run: bun scripts/quality/replay-parser-bench-compare.ts ../base-telemetry-results.json ../pr-telemetry-results.json --fail-on-regression + + - name: Upload benchmark reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-reports + path: | + base-results.json + pr-results.json + base-telemetry-results.json + pr-telemetry-results.json + pipeline-comparison.md + telemetry-comparison.md + comparison.md + retention-days: 7 diff --git a/.gitignore b/.gitignore index 119e875ed..c582a4b93 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ laps.json client/debug-storybook.log client/src/stories/__snapshots__/results/ bench-results.json +telemetry-benchmark-results.json client/storybook-static/ .superset/ diff --git a/bun.lock b/bun.lock index 84ccea018..9135d1a1f 100644 --- a/bun.lock +++ b/bun.lock @@ -861,7 +861,7 @@ "@storybook/addon-docs": ["@storybook/addon-docs@10.5.5", "", { "dependencies": { "@mdx-js/react": "^3.0.0", "@storybook/csf-plugin": "10.5.5", "@storybook/icons": "^2.0.2", "@storybook/react-dom-shim": "10.5.5", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "ts-dedent": "^2.0.0" }, "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.5" }, "optionalPeers": ["@types/react"] }, "sha512-0YpKlimS4XE0kQ8Maa5coeefQxdyDrBHg1wOP3WTPuBe4FolFSCDveR0ge2+vuUBk+fZfn2+l+3Q2jmAWaRGDg=="], - "@storybook/builder-vite": ["@storybook/builder-vite@10.5.7", "", { "dependencies": { "@storybook/csf-plugin": "10.5.7", "ts-dedent": "^2.0.0" }, "peerDependencies": { "storybook": "^10.5.7", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-fShF/aQaITqcJuMCLr42BGNUAbhDi4IboqvlbZqXAwgrrTslnZEUnY8GcEcvpZmjl11VwlmazhMJdH50fIgBPg=="], + "@storybook/builder-vite": ["@storybook/builder-vite@10.5.8", "", { "dependencies": { "@storybook/csf-plugin": "10.5.8", "ts-dedent": "^2.0.0" }, "peerDependencies": { "storybook": "^10.5.8", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "sha512-UeRnn7yT55WmBlHNOQzLrvN7vsHEvVgIukhKDO+4cMbGXN87wZkbxhx6NstpuXRH8OxGqwKS0SZNVp+SC1ftLQ=="], "@storybook/csf-plugin": ["@storybook/csf-plugin@10.5.5", "", { "dependencies": { "unplugin": "^2.3.5" }, "peerDependencies": { "esbuild": "*", "rollup": "*", "storybook": "^10.5.5", "vite": "*", "webpack": "*" }, "optionalPeers": ["esbuild", "rollup", "vite", "webpack"] }, "sha512-/euibhRFqklYCZqUseokojmfYcQpXshVY2QmA1qCuxMz9SzVFD3iSTw+aFLTxpsJGGdcZJk8fnm/rEthLzZ9jA=="], @@ -869,11 +869,11 @@ "@storybook/icons": ["@storybook/icons@2.1.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Fxh9vYpX9bQqFeHRiY8h2ApeRGDzRSMLwJwNZ/AIRqnyOKHxRKL+yFe+ctEkVJmuptRE9u1Hrn8ZZNHyfDKKNg=="], - "@storybook/react": ["@storybook/react@10.5.7", "", { "dependencies": { "@storybook/global": "^5.0.0", "@storybook/react-dom-shim": "10.5.7", "react-docgen": "^8.0.2", "react-docgen-typescript": "^2.2.2" }, "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.7", "typescript": ">= 4.9.x" }, "optionalPeers": ["@types/react", "@types/react-dom", "typescript"] }, "sha512-uFvty2MMdFXzW5PcQe1JqDAZkz6cQq7q/9G/cbGVnBEvP6zsOVeL+bmrQ0/WBlFQN0Ko9+ZoCTvaQ9s65zBa5g=="], + "@storybook/react": ["@storybook/react@10.5.8", "", { "dependencies": { "@storybook/global": "^5.0.0", "@storybook/react-dom-shim": "10.5.8", "react-docgen": "^8.0.2", "react-docgen-typescript": "^2.2.2" }, "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.8", "typescript": ">= 4.9.x" }, "optionalPeers": ["@types/react", "@types/react-dom", "typescript"] }, "sha512-6qqkmqX6imtL+0Z9Uan2tIfYivOI0FiVmWr0zpqqQR15AkJ18JfNcNTQoyjeAlCO0Kei56SWqnu2qLq52TYplg=="], "@storybook/react-dom-shim": ["@storybook/react-dom-shim@10.5.5", "", { "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.5" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-PIk7N3LLrZIxfNxmkvmQN1d5UQ70XEedT8n0GhBiXnM6XL09xPGB8n8TZXeJBRYluKhDQcAyQeT0/OZmcDVQJg=="], - "@storybook/react-vite": ["@storybook/react-vite@10.5.7", "", { "dependencies": { "@joshwooding/vite-plugin-react-docgen-typescript": "^0.7.0", "@rollup/pluginutils": "^5.0.2", "@storybook/builder-vite": "10.5.7", "@storybook/react": "10.5.7", "empathic": "^2.0.0", "magic-string": "^0.30.0", "react-docgen": "^8.0.2", "resolve": "^1.22.8", "tsconfig-paths": "^4.2.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.7", "typescript": ">= 4.9.x", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["typescript"] }, "sha512-eEo3eVa2pvqrzQukKxAzx7YvswDAA1s6k/y+tdMxmRvWyHX6QEOsb9Tda6wcVaa7c8BeJM7Ggq+289cRMTH6Iw=="], + "@storybook/react-vite": ["@storybook/react-vite@10.5.8", "", { "dependencies": { "@joshwooding/vite-plugin-react-docgen-typescript": "^0.7.0", "@rollup/pluginutils": "^5.0.2", "@storybook/builder-vite": "10.5.8", "@storybook/react": "10.5.8", "empathic": "^2.0.0", "magic-string": "^0.30.0", "react-docgen": "^8.0.2", "resolve": "^1.22.8", "tsconfig-paths": "^4.2.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.8", "typescript": ">= 4.9.x", "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["typescript"] }, "sha512-ioMJGi4YzueGsJBlYio+2+UhfCFB9QV5Bs1lOilkek+a4BZgKJl0D1mVSJl6k96stQBPZmLgI9/l0hLVcUL6Kg=="], "@tailwindcss/node": ["@tailwindcss/node@4.3.3", "", { "dependencies": { "@jridgewell/remapping": "^2.3.5", "enhanced-resolve": "^5.24.1", "jiti": "^2.7.0", "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", "tailwindcss": "4.3.3" } }, "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg=="], @@ -2129,7 +2129,7 @@ "stdin-discarder": ["stdin-discarder@0.2.2", "", {}, "sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ=="], - "storybook": ["storybook@10.5.7", "", { "dependencies": { "@storybook/global": "^5.0.0", "@storybook/icons": "^2.0.2", "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "6.9.1", "@testing-library/user-event": "^14.6.1", "@vitest/expect": "3.2.4", "@vitest/spy": "3.2.4", "@webcontainer/env": "^1.1.1", "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0 || ^0.28.0", "jsonc-parser": "^3.3.1", "open": "^10.2.0", "oxc-parser": "^0.127.0", "oxc-resolver": "^11.19.1", "recast": "^0.23.5", "semver": "^7.7.3", "use-sync-external-store": "^1.5.0", "ws": "^8.21.1" }, "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "prettier": "^2 || ^3", "vite-plus": "^0.1.15 || ^0.2.0" }, "optionalPeers": ["@types/react", "prettier", "vite-plus"], "bin": "./dist/bin/dispatcher.js" }, "sha512-oiKvWIwIoOhFP1i6dASYyMXwPHKEtVZMshqSB7EvIVYjWRh0l9H7gHEt1z4Gh2rLGFMekWdsm4s94rvwpR7gkg=="], + "storybook": ["storybook@10.5.8", "", { "dependencies": { "@storybook/global": "^5.0.0", "@storybook/icons": "^2.0.2", "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "6.9.1", "@testing-library/user-event": "^14.6.1", "@vitest/expect": "3.2.4", "@vitest/spy": "3.2.4", "@webcontainer/env": "^1.1.1", "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0 || ^0.28.0", "jsonc-parser": "^3.3.1", "open": "^10.2.0", "oxc-parser": "^0.127.0", "oxc-resolver": "^11.19.1", "recast": "^0.23.5", "semver": "^7.7.3", "use-sync-external-store": "^1.5.0", "ws": "^8.21.1" }, "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "prettier": "^2 || ^3", "vite-plus": "^0.1.15 || ^0.2.0" }, "optionalPeers": ["@types/react", "prettier", "vite-plus"], "bin": "./dist/bin/dispatcher.js" }, "sha512-rR4oFMSiWBSqI0lvsJPtcQUPj8+hzj3TkLu+Mw61Wo6YxPSb5FsLSHai0jZnuaIdKIlmu25KCfwlSQl4e1uvnA=="], "stream-parser": ["stream-parser@0.3.1", "", { "dependencies": { "debug": "2" } }, "sha512-bJ/HgKq41nlKvlhccD5kaCr/P+Hu0wPNKPJOH7en+YrJu/9EgqUF+88w5Jb6KNcjOFMhfX4B2asfeAtIGuHObQ=="], @@ -2425,9 +2425,9 @@ "@sindresorhus/transliterate/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], - "@storybook/builder-vite/@storybook/csf-plugin": ["@storybook/csf-plugin@10.5.7", "", { "dependencies": { "unplugin": "^2.3.5" }, "peerDependencies": { "esbuild": "*", "rollup": "*", "storybook": "^10.5.7", "vite": "*", "webpack": "*" }, "optionalPeers": ["esbuild", "rollup", "vite", "webpack"] }, "sha512-IaX8FlM0H36HNFhJ2+4L9bCldqfvHGqcLg841SJNyK/DhfMlM7JsvY/GDH2ZFuWrUf8FSOx96GRRnHq6XfRKag=="], + "@storybook/builder-vite/@storybook/csf-plugin": ["@storybook/csf-plugin@10.5.8", "", { "dependencies": { "unplugin": "^2.3.5" }, "peerDependencies": { "esbuild": "*", "rollup": "*", "storybook": "^10.5.8", "vite": "*", "webpack": "*" }, "optionalPeers": ["esbuild", "rollup", "vite", "webpack"] }, "sha512-/FHiMyOWWEXfwK/lM0WxmkP9GLzbSJJuzGtfeuNWSOVDnvAMbjavitxfHb5wSbWKIQo0XYC1EJ2Y7x91XNYP4w=="], - "@storybook/react/@storybook/react-dom-shim": ["@storybook/react-dom-shim@10.5.7", "", { "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.7" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-lxOkyh+wu/MiBXvYQHjZfD+DRKOa4bHBzbuGuiHXnHXmdOcTRdcrQTsoeN2FPtfugmmOG66cZUEgDwNX+k5eRA=="], + "@storybook/react/@storybook/react-dom-shim": ["@storybook/react-dom-shim@10.5.8", "", { "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.5.8" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-N8D13/Xny+V3kfe1KBgsAHS0nKWXLLdgOOXS9poKdYzVwVCN+CGEGBxWX0zMMtdCptqa6/57em9coPlZMoO+bg=="], "@tailwindcss/node/lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], diff --git a/package.json b/package.json index 89e4a6f81..232db3cf4 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "test:ai": "bun test test/ai/evals/ai-quality.ai-eval.ts --timeout 120000", "ai:baseline": "bun scripts/quality/ai-baseline.ts", "bench": "bun test/benchmarks/pipeline.bench.ts", + "bench:telemetry": "bun scripts/quality/replay-parser-bench-local.ts", + "bench:telemetry:current": "bun test/benchmarks/replay-parser.bench.ts", "extract:tracks": "bun scripts/games/fm-2023/extract-tracks.ts && bun scripts/games/f1-2025/extract-tracks.ts", "extract:tracks:forza": "bun scripts/games/fm-2023/extract-tracks.ts", "extract:tracks:f1": "bun scripts/games/f1-2025/extract-tracks.ts", diff --git a/scripts/quality/README.md b/scripts/quality/README.md index f521b6173..dae40bff6 100644 --- a/scripts/quality/README.md +++ b/scripts/quality/README.md @@ -6,9 +6,11 @@ Capture and compare quality/performance measurements and exercise update install |---|---| | `bun scripts/quality/ai-baseline.ts` | Run AI evaluation fixtures and write a SHA/model baseline under `test/ai-fixtures/baselines`. Requires Gemini API credentials. | | `bun scripts/quality/bench-compare.ts [--threshold=5] [--fail-on-regression]` | Emit benchmark markdown diff; optionally fail when regressions exceed threshold. | +| `bun scripts/quality/replay-parser-bench-compare.ts [--fail-on-regression]` | Compare compatible same-machine reports; enforce relative throughput/memory and hard memory budgets when requested. | +| `bun scripts/quality/replay-parser-bench-local.ts [--base=main]` | Benchmark base ref and current worktree on same machine; bootstrap hard memory only when base predates benchmark. | | `bun scripts/quality/test-updater.ts` | Build/reuse local installer and start dev server with forced-update variables. | -Inputs: AI credentials and fixtures, Mitata JSON result files, or local package/installer state. Outputs: baseline JSON, markdown on stdout, or update-test process logs/status. +Inputs: AI credentials and fixtures, same-machine Mitata or replay/parser benchmark JSON result files, or local package/installer state. Outputs: baseline JSON, markdown on stdout, or update-test process logs/status. Boundary: measurement and local verification workflows only. Scripts do not own production builds, test fixtures, benchmark generation, or release publication. diff --git a/scripts/quality/replay-parser-bench-compare.ts b/scripts/quality/replay-parser-bench-compare.ts new file mode 100644 index 000000000..76d819a06 --- /dev/null +++ b/scripts/quality/replay-parser-bench-compare.ts @@ -0,0 +1,212 @@ +#!/usr/bin/env bun + +import { readFileSync } from "node:fs"; +import { z } from "zod"; +import type { ReplayParserBenchmarkReport, ReplayParserBenchmarkResult } from "../../test/benchmarks/replay-parser.bench"; + +const MAX_THROUGHPUT_REGRESSION_PERCENT = 10; +const MAX_MEMORY_REGRESSION_PERCENT = 15; +const args = process.argv.slice(2); +const files = args.filter((argument) => !argument.startsWith("--")); + +if (files.length !== 2) { + console.error("Usage: bun scripts/quality/replay-parser-bench-compare.ts [--fail-on-regression]"); + process.exit(1); +} + +const NonEmptyStringSchema = z.string().refine((value) => value.trim().length > 0); +const PositiveFiniteNumberSchema = z.number().finite().positive(); +const NonNegativeFiniteNumberSchema = z.number().finite().nonnegative(); + +const BenchmarkBudgetSchema = z.object({ + maxPeakRssBytes: PositiveFiniteNumberSchema, + maxIncrementalRssBytes: PositiveFiniteNumberSchema, + maxIncrementalHeapBytes: PositiveFiniteNumberSchema, +}); + +const BenchmarkResultSchema = z.object({ + name: z.enum(["parser", "replay"]), + fixture: NonEmptyStringSchema, + inputFrames: z.number().finite().int().positive(), + outputItems: z.number().finite().int().nonnegative(), + semanticCount: z.number().finite().int().nonnegative(), + durationMs: PositiveFiniteNumberSchema, + throughputPerSecond: PositiveFiniteNumberSchema, + baselineRssBytes: NonNegativeFiniteNumberSchema, + peakRssBytes: NonNegativeFiniteNumberSchema, + incrementalPeakRssBytes: NonNegativeFiniteNumberSchema, + baselineHeapBytes: NonNegativeFiniteNumberSchema, + peakHeapBytes: NonNegativeFiniteNumberSchema, + incrementalPeakHeapBytes: NonNegativeFiniteNumberSchema, + budget: BenchmarkBudgetSchema, +}); + +const BenchmarkReportSchema = z.object({ + schemaVersion: z.literal(2), + runtime: NonEmptyStringSchema, + platform: NonEmptyStringSchema, + architecture: NonEmptyStringSchema, + machine: z.object({ + cpuModel: NonEmptyStringSchema, + logicalCpuCount: z.number().finite().int().positive(), + totalMemoryBytes: PositiveFiniteNumberSchema, + }), + results: z.array(BenchmarkResultSchema).length(2).refine( + (results) => + results.filter((result) => result.name === "parser").length === 1 + && results.filter((result) => result.name === "replay").length === 1, + ), +}); + +function readReport(path: string): ReplayParserBenchmarkReport { + let report: unknown; + try { + report = JSON.parse(readFileSync(path, "utf8")) as unknown; + } catch { + throw new Error(`${path} is not a valid replay/parser benchmark report`); + } + const parsedReport = BenchmarkReportSchema.safeParse(report); + if (!parsedReport.success) { + throw new Error(`${path} is not a valid replay/parser benchmark report`); + } + return parsedReport.data; +} + +function indexedResults(report: ReplayParserBenchmarkReport, path: string): Map { + const results = new Map(); + for (const result of report.results) { + if (results.has(result.name)) throw new Error(`${path} contains duplicate ${result.name} results`); + results.set(result.name, result); + } + return results; +} + +function percentChange(current: number, baseline: number): number { + return baseline === 0 ? 0 : ((current - baseline) / baseline) * 100; +} + +function formatBytes(bytes: number): string { + return `${(bytes / 1024 / 1024).toFixed(1)} MiB`; +} + +function signedPercent(value: number): string { + return `${value > 0 ? "+" : ""}${value.toFixed(1)}%`; +} + +function absoluteBudgetFailures(result: ReplayParserBenchmarkResult): string[] { + const failures: string[] = []; + if (result.peakRssBytes > result.budget.maxPeakRssBytes) { + failures.push(`${result.name} peak RSS ${formatBytes(result.peakRssBytes)} exceeds ${formatBytes(result.budget.maxPeakRssBytes)}`); + } + if (result.incrementalPeakRssBytes > result.budget.maxIncrementalRssBytes) { + failures.push(`${result.name} incremental RSS ${formatBytes(result.incrementalPeakRssBytes)} exceeds ${formatBytes(result.budget.maxIncrementalRssBytes)}`); + } + if (result.incrementalPeakHeapBytes > result.budget.maxIncrementalHeapBytes) { + failures.push(`${result.name} incremental heap ${formatBytes(result.incrementalPeakHeapBytes)} exceeds ${formatBytes(result.budget.maxIncrementalHeapBytes)}`); + } + return failures; +} + +function budgetRelaxationFailures( + baseline: ReplayParserBenchmarkResult, + current: ReplayParserBenchmarkResult, +): string[] { + const failures: string[] = []; + if (current.budget.maxPeakRssBytes > baseline.budget.maxPeakRssBytes) { + failures.push( + `${current.name} maxPeakRssBytes budget relaxed: ${formatBytes(baseline.budget.maxPeakRssBytes)} → ${formatBytes(current.budget.maxPeakRssBytes)}`, + ); + } + if (current.budget.maxIncrementalRssBytes > baseline.budget.maxIncrementalRssBytes) { + failures.push( + `${current.name} maxIncrementalRssBytes budget relaxed: ${formatBytes(baseline.budget.maxIncrementalRssBytes)} → ${formatBytes(current.budget.maxIncrementalRssBytes)}`, + ); + } + if (current.budget.maxIncrementalHeapBytes > baseline.budget.maxIncrementalHeapBytes) { + failures.push( + `${current.name} maxIncrementalHeapBytes budget relaxed: ${formatBytes(baseline.budget.maxIncrementalHeapBytes)} → ${formatBytes(current.budget.maxIncrementalHeapBytes)}`, + ); + } + return failures; +} + +const [baselinePath, currentPath] = files; +const baselineReport = readReport(baselinePath); +const currentReport = readReport(currentPath); +const baseline = indexedResults(baselineReport, baselinePath); +const current = indexedResults(currentReport, currentPath); +const scenarioNames = [...new Set([...baseline.keys(), ...current.keys()])].sort(); +const rows: string[] = []; +const failures: string[] = []; + +const sameMachineConfiguration = + baselineReport.runtime === currentReport.runtime + && baselineReport.platform === currentReport.platform + && baselineReport.architecture === currentReport.architecture + && baselineReport.machine.cpuModel === currentReport.machine.cpuModel + && baselineReport.machine.logicalCpuCount === currentReport.machine.logicalCpuCount + && baselineReport.machine.totalMemoryBytes === currentReport.machine.totalMemoryBytes; +if (!sameMachineConfiguration) { + failures.push("Benchmark reports came from different runtime or machine configurations"); +} + +rows.push("| Scenario | Throughput baseline → current | Δ throughput | Peak RSS baseline → current | Δ RSS | Peak heap baseline → current | Δ heap | Hard memory budgets |"); +rows.push("|---|---:|---:|---:|---:|---:|---:|---|"); + +for (const name of scenarioNames) { + const baselineResult = baseline.get(name); + const currentResult = current.get(name); + if (!baselineResult || !currentResult) { + rows.push(`| ${name} | missing | — | missing | — | missing | — | failed |`); + failures.push(`${name} is missing from ${baselineResult ? "current" : "baseline"} report`); + continue; + } + + const sameWorkload = + baselineResult.fixture === currentResult.fixture + && baselineResult.inputFrames === currentResult.inputFrames + && baselineResult.semanticCount === currentResult.semanticCount; + if (!sameWorkload) { + failures.push( + `${name} workload changed: ${baselineResult.fixture}, ${baselineResult.inputFrames} frames/${baselineResult.semanticCount} semantics → ` + + `${currentResult.fixture}, ${currentResult.inputFrames} frames/${currentResult.semanticCount} semantics`, + ); + } + + const throughputChange = percentChange(currentResult.throughputPerSecond, baselineResult.throughputPerSecond); + const rssChange = percentChange(currentResult.peakRssBytes, baselineResult.peakRssBytes); + const heapChange = percentChange(currentResult.peakHeapBytes, baselineResult.peakHeapBytes); + const absoluteFailures = absoluteBudgetFailures(currentResult); + const relaxationFailures = budgetRelaxationFailures(baselineResult, currentResult); + failures.push(...absoluteFailures, ...relaxationFailures); + + if (sameMachineConfiguration && sameWorkload) { + if (throughputChange < -MAX_THROUGHPUT_REGRESSION_PERCENT) { + failures.push(`${name} throughput regressed ${(-throughputChange).toFixed(1)}%, limit is ${MAX_THROUGHPUT_REGRESSION_PERCENT}%`); + } + if (rssChange > MAX_MEMORY_REGRESSION_PERCENT) { + failures.push(`${name} peak RSS regressed ${rssChange.toFixed(1)}%, limit is ${MAX_MEMORY_REGRESSION_PERCENT}%`); + } + if (heapChange > MAX_MEMORY_REGRESSION_PERCENT) { + failures.push(`${name} peak heap regressed ${heapChange.toFixed(1)}%, limit is ${MAX_MEMORY_REGRESSION_PERCENT}%`); + } + } + + rows.push( + `| ${name} | ${baselineResult.throughputPerSecond.toFixed(0)}/s → ${currentResult.throughputPerSecond.toFixed(0)}/s | ${signedPercent(throughputChange)} | ` + + `${formatBytes(baselineResult.peakRssBytes)} → ${formatBytes(currentResult.peakRssBytes)} | ${signedPercent(rssChange)} | ` + + `${formatBytes(baselineResult.peakHeapBytes)} → ${formatBytes(currentResult.peakHeapBytes)} | ${signedPercent(heapChange)} | ${absoluteFailures.length === 0 && relaxationFailures.length === 0 ? "pass" : "failed"} |`, + ); +} + +const summary = failures.length === 0 + ? "All hard memory and same-machine relative budgets passed." + : `Budget failures:\n${failures.map((failure) => `- ${failure}`).join("\n")}`; + +console.log( + `## Replay/parser benchmark budgets\n\n` + + `Runtime: \`${currentReport.runtime}\` on \`${currentReport.machine.cpuModel}\` (${currentReport.machine.logicalCpuCount} logical CPUs). ` + + `Throughput regression limit: ${MAX_THROUGHPUT_REGRESSION_PERCENT}%. Peak-memory regression limit: ${MAX_MEMORY_REGRESSION_PERCENT}%.\n\n${rows.join("\n")}\n\n${summary}`, +); + +if (failures.length > 0 && args.includes("--fail-on-regression")) process.exit(1); diff --git a/scripts/quality/replay-parser-bench-local.ts b/scripts/quality/replay-parser-bench-local.ts new file mode 100644 index 000000000..2a7e3d9c7 --- /dev/null +++ b/scripts/quality/replay-parser-bench-local.ts @@ -0,0 +1,77 @@ +#!/usr/bin/env bun + +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const root = process.cwd(); +const benchmarkController = "test/benchmarks/replay-parser.bench.ts"; + +function argumentValue(name: string): string | undefined { + const prefix = `${name}=`; + return process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length); +} + +async function run(command: readonly string[], cwd: string): Promise { + console.log(`[telemetry-bench] ${command.join(" ")}`); + const child = Bun.spawn({ + cmd: [...command], + cwd, + env: process.env, + stdout: "inherit", + stderr: "inherit", + }); + const exitCode = await child.exited; + if (exitCode !== 0) throw new Error(`${command[0]} exited with code ${exitCode}`); +} + +const baseRef = argumentValue("--base") ?? process.env.BENCH_BASE_REF ?? "main"; +const nodeModules = resolve(root, "node_modules"); +if (!existsSync(nodeModules)) { + throw new Error("Run bun install before the same-machine telemetry benchmark"); +} +const tempRoot = mkdtempSync(join(tmpdir(), "raceiq-telemetry-compare-")); +const baseWorktree = join(tempRoot, "base"); +const baseReport = join(tempRoot, "base.json"); +const currentReport = join(tempRoot, "current.json"); +let worktreeAdded = false; +let worktreeRemovalFailed = false; + +try { + await run(["git", "worktree", "add", "--detach", baseWorktree, baseRef], root); + worktreeAdded = true; + + const baseBenchmark = join(baseWorktree, benchmarkController); + if (existsSync(baseBenchmark)) { + await run([process.execPath, "install", "--no-save", "--ignore-scripts"], baseWorktree); + await run([process.execPath, benchmarkController, "--no-enforce", `--output=${baseReport}`], baseWorktree); + await run([process.execPath, benchmarkController, "--no-enforce", `--output=${currentReport}`], root); + await run([ + process.execPath, + resolve(root, "scripts/quality/replay-parser-bench-compare.ts"), + baseReport, + currentReport, + "--fail-on-regression", + ], root); + } else { + console.log(`[telemetry-bench] ${baseRef} has no replay/parser benchmark; bootstrapping hard memory budgets only`); + await run([process.execPath, benchmarkController, `--output=${currentReport}`], root); + } +} finally { + if (worktreeAdded) { + try { + await run(["git", "worktree", "remove", "--force", baseWorktree], root); + } catch (error) { + console.error(`[telemetry-bench] failed to remove temporary worktree ${baseWorktree}:`, error); + worktreeRemovalFailed = true; + } + } + rmSync(tempRoot, { recursive: true, force: true }); + if (worktreeRemovalFailed) { + try { + await run(["git", "worktree", "prune"], root); + } catch (error) { + console.error("[telemetry-bench] failed to prune temporary worktree metadata:", error); + } + } +} diff --git a/test/README.md b/test/README.md index 7d659b023..c2701a8b1 100644 --- a/test/README.md +++ b/test/README.md @@ -15,13 +15,20 @@ bun run test bun test test/games/shared/parser.test.ts --timeout 30000 bun run test:ai bun run bench +bun run bench:telemetry +bun run bench:telemetry:current --no-enforce --output=telemetry-benchmark-results.json ``` `bun run test` is standard suite. Focused command runs one final-path file; Bun preload still isolates `DATA_DIR` in `.data-test`. `bun run test:ai` runs `test/ai/evals/ai-quality.ai-eval.ts` with its longer timeout. `bun run bench` -runs `test/benchmarks/pipeline.bench.ts`; benchmarks are explicit scripts, not -ordinary tests. +runs parser/pipeline microbenchmarks. `bun run bench:telemetry` checks the current +worktree against `main` (or `--base=`) on the same machine. It replays +20,000 committed AC Evo frames and enforces relative throughput/memory plus hard +memory budgets. When the base predates this benchmark, the command explicitly +bootstraps hard memory budgets without fabricating a throughput comparison. +`bun run bench:telemetry:current` produces one report without a base comparison. +Benchmarks are explicit scripts, not ordinary tests. ## Top-level map @@ -61,7 +68,8 @@ ordinary tests. and output boundaries. Put recording-driven suites in `e2e/`; telemetry catalog E2E suites stay in `telemetry/catalog/` with that domain. - **Benchmark:** performance measurement only. Keep setup and input stable; run - through `bun run bench`, never as part of standard discovery. + through `bun run bench` or `bun run bench:telemetry`, never as part of standard + discovery. - **AI eval:** model-backed quality checks under `ai/evals/`; use explicit `*.ai-eval.ts` entry points and curated `ai-fixtures/` data. diff --git a/test/benchmarks/replay-parser-worker.ts b/test/benchmarks/replay-parser-worker.ts new file mode 100644 index 000000000..9334c1261 --- /dev/null +++ b/test/benchmarks/replay-parser-worker.ts @@ -0,0 +1,125 @@ +#!/usr/bin/env bun + +import { memoryUsage } from "bun:jsc"; +import { client, initDb } from "../../server/db"; +import { insertLap } from "../../server/db/lap-mutation-queries"; +import { deleteSession, insertSession, updateSessionRawFile } from "../../server/db/session-queries"; +import { parseRawLapFrames } from "../../server/db/telemetry-replay-storage"; +import { initServerGameAdapters } from "../../server/games/init"; +import { queryLapTelemetryBySemanticId } from "../../server/telemetry/replay"; +import { initGameAdapters } from "../../shared/games/init"; +import type { ReplayParserBenchmarkMeasurement } from "./replay-parser.bench"; + +const FRAME_COUNT = Number(process.env.TELEMETRY_BENCHMARK_FRAMES ?? 20_000); +const FIXTURE = "test/artifacts/sessions/session-ac-evo-mid-2026-04-21T20-24-34-810Z.bin.gz"; +const SEMANTIC_IDS = [ + "motion.speed", + "inputs.accel", + "inputs.brake", + "inputs.gear", + "inputs.clutch-percent", + "timing.current-lap", + "timing.lap-number", + "timing.distance-traveled", +] as const; + +type ScenarioName = ReplayParserBenchmarkMeasurement["name"]; + +interface MemoryBaseline { + readonly rssBytes: number; + readonly heapBytes: number; +} + +function argumentValue(name: string): string | undefined { + const prefix = `${name}=`; + return process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length); +} + +function settleMemory(): MemoryBaseline { + Bun.gc(true); + const heap = memoryUsage(); + return { + rssBytes: process.memoryUsage.rss(), + heapBytes: heap.current, + }; +} + +function measuredResult( + name: ScenarioName, + baseline: MemoryBaseline, + startedAt: number, + outputItems: number, +): ReplayParserBenchmarkMeasurement { + const durationMs = performance.now() - startedAt; + const heap = memoryUsage(); + const peakRssBytes = Math.max(process.memoryUsage.rss(), process.resourceUsage().maxRSS * 1024); + const peakHeapBytes = Math.max(heap.current, heap.peak); + return { + name, + fixture: FIXTURE, + inputFrames: FRAME_COUNT, + outputItems, + semanticCount: name === "replay" ? SEMANTIC_IDS.length : 0, + durationMs, + throughputPerSecond: (FRAME_COUNT * 1000) / durationMs, + baselineRssBytes: baseline.rssBytes, + peakRssBytes, + incrementalPeakRssBytes: Math.max(0, peakRssBytes - baseline.rssBytes), + baselineHeapBytes: baseline.heapBytes, + peakHeapBytes, + incrementalPeakHeapBytes: Math.max(0, peakHeapBytes - baseline.heapBytes), + }; +} + +async function runParserScenario(): Promise { + const baseline = settleMemory(); + const startedAt = performance.now(); + const packets = await parseRawLapFrames(FIXTURE, 12, FRAME_COUNT, "ac-evo"); + const result = measuredResult("parser", baseline, startedAt, packets.length); + if (packets.length < FRAME_COUNT || packets.length > FRAME_COUNT + 1) { + throw new Error(`Parser benchmark expected ${FRAME_COUNT}-${FRAME_COUNT + 1} packets, received ${packets.length}`); + } + return result; +} + +async function runReplayScenario(): Promise { + await initDb(); + const sessionId = await insertSession(1, 1, "ac-evo"); + try { + await updateSessionRawFile(sessionId, FIXTURE, "replay-parser-benchmark"); + const lapId = await insertLap(sessionId, 1, 90, true, 12, FRAME_COUNT); + const baseline = settleMemory(); + const startedAt = performance.now(); + const replay = await queryLapTelemetryBySemanticId(lapId, SEMANTIC_IDS); + const result = measuredResult("replay", baseline, startedAt, replay?.envelopes.length ?? 0); + if (!replay || replay.envelopes.length < FRAME_COUNT || replay.envelopes.length > FRAME_COUNT + 1) { + throw new Error(`Replay benchmark expected ${FRAME_COUNT}-${FRAME_COUNT + 1} envelopes, received ${replay?.envelopes.length ?? 0}`); + } + return result; + } finally { + await deleteSession(sessionId); + client.close(); + } +} + +if (!Number.isInteger(FRAME_COUNT) || FRAME_COUNT <= 0 || FRAME_COUNT > 20_000) { + throw new Error(`TELEMETRY_BENCHMARK_FRAMES must be an integer between 1 and 20000, received ${FRAME_COUNT}`); +} + +const scenario = argumentValue("--scenario") as ScenarioName | undefined; +const resultPath = argumentValue("--result"); +if ((scenario !== "parser" && scenario !== "replay") || !resultPath) { + throw new Error("Benchmark worker requires --scenario=parser|replay and --result="); +} + +initGameAdapters(); +initServerGameAdapters(); +const result = scenario === "parser" ? await runParserScenario() : await runReplayScenario(); +await Bun.write(resultPath, JSON.stringify(result, null, 2)); +console.log( + `[telemetry-bench] ${result.name}: ${result.throughputPerSecond.toFixed(0)}/s, ` + + `peak RSS ${(result.peakRssBytes / 1024 / 1024).toFixed(1)} MiB, ` + + `incremental RSS ${(result.incrementalPeakRssBytes / 1024 / 1024).toFixed(1)} MiB, ` + + `incremental heap ${(result.incrementalPeakHeapBytes / 1024 / 1024).toFixed(1)} MiB`, +); +process.exit(0); diff --git a/test/benchmarks/replay-parser.bench.ts b/test/benchmarks/replay-parser.bench.ts new file mode 100644 index 000000000..d2e547e23 --- /dev/null +++ b/test/benchmarks/replay-parser.bench.ts @@ -0,0 +1,159 @@ +#!/usr/bin/env bun + +import { mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { cpus, tmpdir, totalmem } from "node:os"; +import { join, resolve } from "node:path"; + +const MIB = 1024 * 1024; + +const BUDGETS = { + parser: { + maxPeakRssBytes: 768 * MIB, + maxIncrementalRssBytes: 512 * MIB, + maxIncrementalHeapBytes: 512 * MIB, + }, + replay: { + maxPeakRssBytes: 1_536 * MIB, + maxIncrementalRssBytes: 1_024 * MIB, + maxIncrementalHeapBytes: 1_024 * MIB, + }, +} as const; + +type ScenarioName = keyof typeof BUDGETS; + +export interface BenchmarkBudget { + readonly maxPeakRssBytes: number; + readonly maxIncrementalRssBytes: number; + readonly maxIncrementalHeapBytes: number; +} + + +export interface ReplayParserBenchmarkMeasurement { + readonly name: ScenarioName; + readonly fixture: string; + readonly inputFrames: number; + readonly outputItems: number; + readonly semanticCount: number; + readonly durationMs: number; + readonly throughputPerSecond: number; + readonly baselineRssBytes: number; + readonly peakRssBytes: number; + readonly incrementalPeakRssBytes: number; + readonly baselineHeapBytes: number; + readonly peakHeapBytes: number; + readonly incrementalPeakHeapBytes: number; +} + +export interface ReplayParserBenchmarkResult extends ReplayParserBenchmarkMeasurement { + readonly budget: BenchmarkBudget; +} + +export interface ReplayParserBenchmarkReport { + readonly schemaVersion: 2; + readonly runtime: string; + readonly platform: string; + readonly architecture: string; + readonly machine: { + readonly cpuModel: string; + readonly logicalCpuCount: number; + readonly totalMemoryBytes: number; + }; + readonly results: readonly ReplayParserBenchmarkResult[]; +} + +function argumentValue(name: string): string | undefined { + const prefix = `${name}=`; + return process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length); +} + + +function budgetFailures(results: readonly ReplayParserBenchmarkResult[]): string[] { + const failures: string[] = []; + for (const result of results) { + const budget = result.budget; + if (result.peakRssBytes > budget.maxPeakRssBytes) { + failures.push(`${result.name} peak RSS ${formatBytes(result.peakRssBytes)} exceeds ${formatBytes(budget.maxPeakRssBytes)}`); + } + if (result.incrementalPeakRssBytes > budget.maxIncrementalRssBytes) { + failures.push(`${result.name} incremental RSS ${formatBytes(result.incrementalPeakRssBytes)} exceeds ${formatBytes(budget.maxIncrementalRssBytes)}`); + } + if (result.incrementalPeakHeapBytes > budget.maxIncrementalHeapBytes) { + failures.push(`${result.name} incremental heap ${formatBytes(result.incrementalPeakHeapBytes)} exceeds ${formatBytes(budget.maxIncrementalHeapBytes)}`); + } + } + return failures; +} + +function formatBytes(bytes: number): string { + return `${(bytes / MIB).toFixed(1)} MiB`; +} + +function printResults(results: readonly ReplayParserBenchmarkResult[]): void { + for (const result of results) { + console.log( + `[telemetry-bench] ${result.name}: ${result.throughputPerSecond.toFixed(0)}/s, ` + + `peak RSS ${formatBytes(result.peakRssBytes)}, incremental RSS ${formatBytes(result.incrementalPeakRssBytes)}, ` + + `incremental heap ${formatBytes(result.incrementalPeakHeapBytes)}`, + ); + } +} + + +async function runController(): Promise { + + const outputPath = resolve(argumentValue("--output") ?? "telemetry-benchmark-results.json"); + const tempRoot = mkdtempSync(join(tmpdir(), "raceiq-telemetry-benchmark-")); + const results: ReplayParserBenchmarkResult[] = []; + try { + for (const scenario of ["parser", "replay"] as const) { + const dataDir = join(tempRoot, scenario, "data"); + const resultPath = join(tempRoot, `${scenario}.json`); + mkdirSync(dataDir, { recursive: true }); + const child = Bun.spawn({ + cmd: [process.execPath, resolve("test/benchmarks/replay-parser-worker.ts"), `--scenario=${scenario}`, `--result=${resultPath}`], + cwd: process.cwd(), + env: { + ...process.env, + DATA_DIR: dataDir, + RACEIQ_TEST_MODE: "1", + NODE_ENV: "test", + }, + stdout: "inherit", + stderr: "inherit", + }); + const exitCode = await child.exited; + if (exitCode !== 0) throw new Error(`${scenario} benchmark child exited with code ${exitCode}`); + const measurement = JSON.parse(readFileSync(resultPath, "utf8")) as ReplayParserBenchmarkMeasurement; + results.push({ ...measurement, budget: BUDGETS[scenario] }); + } + + const cpuInfo = cpus(); + const report: ReplayParserBenchmarkReport = { + schemaVersion: 2, + runtime: `bun ${Bun.version}`, + platform: process.platform, + architecture: process.arch, + machine: { + cpuModel: cpuInfo[0]?.model ?? "unknown", + logicalCpuCount: cpuInfo.length, + totalMemoryBytes: totalmem(), + }, + results, + }; + await Bun.write(outputPath, JSON.stringify(report, null, 2)); + printResults(results); + console.log(`[telemetry-bench] report written to ${outputPath}`); + + if (!process.argv.includes("--no-enforce")) { + const failures = budgetFailures(results); + if (failures.length > 0) { + for (const failure of failures) console.error(`[telemetry-bench] budget failed: ${failure}`); + process.exitCode = 1; + } + } + } finally { + rmSync(tempRoot, { recursive: true, force: true }); + } +} + +await runController(); diff --git a/test/tooling/replay-parser-bench-compare.test.ts b/test/tooling/replay-parser-bench-compare.test.ts new file mode 100644 index 000000000..7ca0c427a --- /dev/null +++ b/test/tooling/replay-parser-bench-compare.test.ts @@ -0,0 +1,164 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { + ReplayParserBenchmarkReport, + ReplayParserBenchmarkResult, +} from "../benchmarks/replay-parser.bench"; + +const MIB = 1024 * 1024; +const tempDirs: string[] = []; + +interface ReportOptions { + readonly parserFixture?: string; + readonly parserMaxPeakRssBytes?: number; + readonly parserThroughputPerSecond?: number; +} + +function makeTempDir(): string { + const dir = mkdtempSync(join(tmpdir(), "raceiq-replay-parser-compare-")); + tempDirs.push(dir); + return dir; +} + +function makeResult( + name: ReplayParserBenchmarkResult["name"], + options: ReportOptions = {}, +): ReplayParserBenchmarkResult { + return { + name, + fixture: name === "parser" ? (options.parserFixture ?? "test/fixture.bin") : "test/fixture.bin", + inputFrames: 100, + outputItems: 100, + semanticCount: name === "replay" ? 8 : 0, + durationMs: 100, + throughputPerSecond: name === "parser" ? (options.parserThroughputPerSecond ?? 1_000) : 1_000, + baselineRssBytes: 16 * MIB, + peakRssBytes: 32 * MIB, + incrementalPeakRssBytes: 16 * MIB, + baselineHeapBytes: 8 * MIB, + peakHeapBytes: 24 * MIB, + incrementalPeakHeapBytes: 16 * MIB, + budget: { + maxPeakRssBytes: name === "parser" ? (options.parserMaxPeakRssBytes ?? 128 * MIB) : 256 * MIB, + maxIncrementalRssBytes: 64 * MIB, + maxIncrementalHeapBytes: 64 * MIB, + }, + }; +} + +function makeReport(options: ReportOptions = {}): ReplayParserBenchmarkReport { + return { + schemaVersion: 2, + runtime: "bun 1.3.14", + platform: "test-platform", + architecture: "x64", + machine: { + cpuModel: "Test CPU", + logicalCpuCount: 8, + totalMemoryBytes: 16 * 1024 * MIB, + }, + results: [makeResult("parser", options), makeResult("replay", options)], + }; +} + +async function runComparator( + baseline: string, + current: string, + failOnRegression = false, +): Promise<{ code: number; output: string }> { + const dir = makeTempDir(); + const baselinePath = join(dir, "baseline.json"); + const currentPath = join(dir, "current.json"); + await Promise.all([Bun.write(baselinePath, baseline), Bun.write(currentPath, current)]); + + const args = [ + process.execPath, + "scripts/quality/replay-parser-bench-compare.ts", + baselinePath, + currentPath, + ]; + if (failOnRegression) args.push("--fail-on-regression"); + const proc = Bun.spawn(args, { + cwd: process.cwd(), + stdout: "pipe", + stderr: "pipe", + }); + const [code, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { code, output: `${stdout}\n${stderr}` }; +} + +afterEach(() => { + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("replay/parser benchmark comparison", () => { + test("accepts identical compatible reports", async () => { + const report = JSON.stringify(makeReport()); + const result = await runComparator(report, report, true); + + expect(result.code, result.output).toBe(0); + expect(result.output).toContain("All hard memory and same-machine relative budgets passed."); + }); + + test("enforces throughput regressions only when requested", async () => { + const baseline = JSON.stringify(makeReport()); + const current = JSON.stringify(makeReport({ parserThroughputPerSecond: 800 })); + + const enforced = await runComparator(baseline, current, true); + expect(enforced.code, enforced.output).toBe(1); + expect(enforced.output).toContain("parser throughput regressed 20.0%, limit is 10%"); + + const reportOnly = await runComparator(baseline, current); + expect(reportOnly.code, reportOnly.output).toBe(0); + expect(reportOnly.output).toContain("parser throughput regressed 20.0%, limit is 10%"); + }); + + test("rejects changed fixtures", async () => { + const result = await runComparator( + JSON.stringify(makeReport()), + JSON.stringify(makeReport({ parserFixture: "test/easier-fixture.bin" })), + true, + ); + + expect(result.code, result.output).toBe(1); + expect(result.output).toContain( + "parser workload changed: test/fixture.bin, 100 frames/0 semantics → test/easier-fixture.bin, 100 frames/0 semantics", + ); + }); + + test("rejects relaxed hard budgets", async () => { + const result = await runComparator( + JSON.stringify(makeReport()), + JSON.stringify(makeReport({ parserMaxPeakRssBytes: 129 * MIB })), + true, + ); + + expect(result.code, result.output).toBe(1); + expect(result.output).toContain("parser maxPeakRssBytes budget relaxed: 128.0 MiB → 129.0 MiB"); + }); + + test("always rejects malformed and non-finite reports", async () => { + const valid = JSON.stringify(makeReport({ parserThroughputPerSecond: 1_234 })); + const duplicateParser = JSON.stringify({ + ...makeReport(), + results: [makeResult("parser"), makeResult("parser")], + }); + const missingReplay = JSON.stringify({ + ...makeReport(), + results: [makeResult("parser")], + }); + const nonFinite = valid.replace('"throughputPerSecond":1234', '"throughputPerSecond":1e999'); + + for (const malformed of ["not json", "{}", duplicateParser, missingReplay, nonFinite]) { + const result = await runComparator(valid, malformed); + expect(result.code, result.output).not.toBe(0); + expect(result.output).toContain("is not a valid replay/parser benchmark report"); + } + }); +});