diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 3ab3c54181..b60764a45d 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -76,44 +76,85 @@ jobs: CNPMCORE_DATABASE_NAME=cnpmcore bash ./prepare-database-mysql.sh EGG_TS_ENABLE=false npx eggctl start --port=7001 --env=unittest --daemon - HEALTH_URL="http://127.0.0.1:7001/" - START_TIME=$(date +%s) - TIMEOUT=120 - STATUS="" - READY=0 - - echo "Waiting for application to become healthy at ${HEALTH_URL} (timeout: ${TIMEOUT}s)..." - while true; do - # Capture response body and status code - STATUS=$(curl -s -o /tmp/cnpmcore-health-response -w "%{http_code}" "${HEALTH_URL}" || echo "000") - echo "Health check attempt at $(date): status=${STATUS}" - - if [ "${STATUS}" = "200" ]; then - echo "Health check succeeded with status 200" - READY=1 - break - fi - - NOW=$(date +%s) - ELAPSED=$((NOW - START_TIME)) - if [ "${ELAPSED}" -ge "${TIMEOUT}" ]; then - echo "Health check timed out after ${ELAPSED}s with last status ${STATUS}" - break - fi - - sleep 5 - done - - npx eggctl stop - - if [ "${READY}" != "1" ]; then - echo "Health check failed; last HTTP status: ${STATUS}" - echo "Last health endpoint response body (if any):" - cat /tmp/cnpmcore-health-response 2>/dev/null || echo "" + + if bash ../wait-health.sh "http://127.0.0.1:7001/" /tmp/cnpmcore-health-response 120 5; then + npx eggctl stop + else + npx eggctl stop || true echo "Recent error logs (if any):" cat logs/cnpmcore/common-error.log 2>/dev/null || true exit 1 fi + - name: cnpmcore-snapshot + node-version: 24 + command: | + # V8 startup snapshot regression: build a snapshot of cnpmcore, restore + # it via `egg-scripts start --snapshot-blob`, and assert it serves + # `/-/ping` with HTTP 200. Restoring a V8 snapshot requires Node.js >= 24. + export EGG_TS_ENABLE=false + # Snapshot under the production env: it loads only config.default (DB + # comes from CNPMCORE_DATABASE_* env vars) and does NOT pull in test-only + # plugins like @eggjs/mock, so the snapshot stays lean and production-like. + # (unittest would bake the mock plugin + config.unittest into the blob.) + export EGG_SERVER_ENV=prod + export CNPMCORE_DATABASE_NAME=cnpmcore + # cnpmcore's NFSClientAdapter throws under prod when the store is local fs + # unless this is set. The snapshot e2e only needs a working /-/ping, so allow + # the local-fs store instead of wiring up OSS/S3. + export CNPMCORE_FORCE_LOCAL_FS=true + + npm install --legacy-peer-deps + + # Compile to JS and overlay onto source so the egg loader and tegg module + # scanner resolve .js at the expected paths (same approach as the cnpmcore + # job); the snapshot is built from the compiled app. + npm run tsc:prod + cp -r dist/* . + rm -rf dist + # Remove the now-redundant .ts sources so the bundler scans only .js. With + # both present, tegg's globby picks up the .ts AND .js of the same class as + # two modules (the "duplicate proto" hazard): a decorated class is bundled + # twice and only one copy's filePath is corrected, which breaks AOP advice + # resolution on restore (e.g. "Aop Advice(AsyncTimer) not found in loadUnits"). + find app config -name '*.ts' ! -name '*.d.ts' -delete + + # Under EGG_SERVER_ENV=prod the DB name comes from CNPMCORE_DATABASE_NAME + # (set to cnpmcore above), so prepare that database. + echo "Preparing database..." + mysql -h 127.0.0.1 -u root -e "CREATE DATABASE IF NOT EXISTS cnpmcore" + bash ./prepare-database-mysql.sh + + # Build the V8 startup snapshot. Building works on Node.js >= 22 (CI runs + # on 24 here). leoric is auto-externalized (native optional DB drivers); + # @cnpmjs/packument / koa-onerror / undici / urllib are kept external. The + # network stack stays lazy-external (no manual stubs) so the blob is + # restorable on Node.js >= 24. + npx egg-bin snapshot build \ + --output ./dist-bundle \ + --force-external @cnpmjs/packument \ + --force-external koa-onerror \ + --force-external undici \ + --force-external urllib + + # Restore the snapshot and verify it serves traffic. --no-sourcemap keeps + # `--import source-map-support` out of the `node --snapshot-blob` process + # (cnpmcore sets egg.typescript, which would otherwise auto-enable it). + npx egg-scripts start \ + --snapshot-blob ./dist-bundle/snapshot.blob \ + --port 7001 \ + --no-sourcemap \ + --daemon \ + --title=egg-server-cnpmcore-snapshot + + if bash ../wait-health.sh "http://127.0.0.1:7001/-/ping" /tmp/cnpmcore-snapshot-health-response 120 3; then + npx egg-scripts stop --title=egg-server-cnpmcore-snapshot + else + npx egg-scripts stop --title=egg-server-cnpmcore-snapshot || true + echo "Recent snapshot error logs (if any):" + cat dist-bundle/logs/cnpmcore/common-error.log 2>/dev/null || true + cat "${HOME}/logs/master-stderr.log" 2>/dev/null || true + exit 1 + fi - name: examples node-version: 24 command: | diff --git a/.gitignore b/.gitignore index 29f937357d..c66be1a469 100644 --- a/.gitignore +++ b/.gitignore @@ -119,6 +119,7 @@ tegg/plugin/tegg/test/fixtures/apps/**/*.js *.tgz ecosystem-ci/cnpmcore +ecosystem-ci/cnpmcore-snapshot ecosystem-ci/examples pnpm-lock.yaml .utoo.toml diff --git a/ecosystem-ci/patch-project.ts b/ecosystem-ci/patch-project.ts index 90ff4997e0..60ec9dc72a 100644 --- a/ecosystem-ci/patch-project.ts +++ b/ecosystem-ci/patch-project.ts @@ -91,8 +91,8 @@ async function patchPackageJSON(filePath: string, overrides: Record): Promise { - const packageJsonPath = join(projectDir, 'cnpmcore', 'package.json'); +async function patchProjectRoot(projectName: string, overrides: Record): Promise { + const packageJsonPath = join(projectDir, projectName, 'package.json'); await patchPackageJSON(packageJsonPath, overrides); } @@ -111,7 +111,10 @@ async function main(): Promise { switch (project) { case 'cnpmcore': - await patchCnpmcore(overrides); + // cnpmcore-snapshot is a second checkout of the same cnpmcore repo used by + // the V8 snapshot e2e job; it patches its own package.json the same way. + case 'cnpmcore-snapshot': + await patchProjectRoot(project, overrides); break; case 'examples': await patchExamples(overrides); diff --git a/ecosystem-ci/repo.json b/ecosystem-ci/repo.json index 7a59ab891e..f914061cc2 100644 --- a/ecosystem-ci/repo.json +++ b/ecosystem-ci/repo.json @@ -4,6 +4,11 @@ "branch": "master", "hash": "98463c33188bbd32513a74e6c3a2c4cc559ef5da" }, + "cnpmcore-snapshot": { + "repository": "https://github.com/cnpm/cnpmcore.git", + "branch": "master", + "hash": "98463c33188bbd32513a74e6c3a2c4cc559ef5da" + }, "examples": { "repository": "https://github.com/eggjs/examples.git", "branch": "master", diff --git a/ecosystem-ci/wait-health.sh b/ecosystem-ci/wait-health.sh new file mode 100755 index 0000000000..916a5b9a03 --- /dev/null +++ b/ecosystem-ci/wait-health.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Poll an HTTP endpoint until it returns 200, or time out. +# +# Usage: wait-health.sh [timeout-seconds] [sleep-seconds] +# Exits 0 on the first HTTP 200; exits 1 on timeout (dumping the last response body). +# +# Shared by the cnpmcore and cnpmcore-snapshot e2e jobs in +# .github/workflows/e2e-test.yml. Deliberately does not `set -e` so the +# curl-poll loop controls flow; callers wrap it in `if wait-health.sh ...; then`. +set -uo pipefail + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 [timeout-seconds] [sleep-seconds]" >&2 + exit 1 +fi + +URL="$1" +RESPONSE_FILE="$2" +TIMEOUT="${3:-120}" +SLEEP="${4:-5}" + +START_TIME=$(date +%s) +echo "Waiting for ${URL} to return HTTP 200 (timeout: ${TIMEOUT}s)..." +while true; do + STATUS=$(curl -s -o "${RESPONSE_FILE}" -w "%{http_code}" "${URL}" || echo "000") + echo "Health check at $(date): status=${STATUS}" + + if [ "${STATUS}" = "200" ]; then + echo "Health check succeeded with status 200" + exit 0 + fi + + NOW=$(date +%s) + ELAPSED=$((NOW - START_TIME)) + if [ "${ELAPSED}" -ge "${TIMEOUT}" ]; then + echo "Health check timed out after ${ELAPSED}s with last status ${STATUS}" + echo "Last response body (if any):" + cat "${RESPONSE_FILE}" 2>/dev/null || echo "" + exit 1 + fi + + sleep "${SLEEP}" +done diff --git a/site/.vitepress/config.mts b/site/.vitepress/config.mts index 850e442fdf..29e4f41830 100644 --- a/site/.vitepress/config.mts +++ b/site/.vitepress/config.mts @@ -291,6 +291,7 @@ function sidebarAdvanced(): DefaultTheme.SidebarItem[] { { text: 'Loader', link: 'loader' }, { text: 'Plugin Development', link: 'plugin' }, { text: 'Framework Development', link: 'framework' }, + { text: 'V8 Startup Snapshot', link: 'snapshot' }, ], }, ]; @@ -404,6 +405,7 @@ function sidebarAdvancedZhCN(): DefaultTheme.SidebarItem[] { { text: 'View 插件开发', link: 'view-plugin' }, { text: '升级你的生命周期事件函数', link: 'loader-update' }, { text: '对象生命周期', link: 'lifecycle' }, + { text: 'V8 启动快照', link: 'snapshot' }, ], }, ]; diff --git a/site/docs/advanced/snapshot.md b/site/docs/advanced/snapshot.md index 9f15bc671d..e437a163e9 100644 --- a/site/docs/advanced/snapshot.md +++ b/site/docs/advanced/snapshot.md @@ -5,22 +5,92 @@ order: 7 # V8 Startup Snapshot -Egg provides built-in helpers for building and restoring a V8 startup snapshot. This can reduce cold-start overhead for applications that need to preload framework metadata, plugins, services, routers, and tegg modules before serving traffic. +Egg can freeze a fully-loaded application into a [V8 startup snapshot](https://nodejs.org/api/cli.html#--build-snapshot) +so that cold start skips most of the boot work. Building a snapshot loads the +entire module graph — framework metadata, plugins, services, routers, and tegg +modules — and runs the lifecycle up to `configWillLoad`, then serializes that +heap into a blob. Restoring the blob resumes the remaining lifecycle (`didReady`) +and starts listening, so the process is ready to serve traffic almost +immediately. + +This builds on [Bundle Deployment](../core/bundle.md): the snapshot is produced +from a single self-contained bundle, so a snapshot is essentially a pre-booted +bundle. + +## Node.js version requirements + +| Phase | Command | Node.js | +| -------------------- | ----------------------------------- | ------- | +| **Build** a snapshot | `egg-bin snapshot build` | >= 22 | +| **Restore** (run) | `egg-scripts start --snapshot-blob` | >= 24 | + +::: warning Restoring requires Node.js >= 24 +A snapshot can be **built** on Node.js >= 22, but **restoring** a non-trivial Egg +heap on Node.js 22 aborts the process during deserialization with a native fatal +error (`Check failed: current == end_slot_index`, a V8 bug). Always restore on +Node.js **>= 24**. + +The supported launcher enforces this: `egg-scripts start --snapshot-blob` refuses +to launch on Node.js < 24 with a clear error before spawning anything. If you +bypass it and run `node --snapshot-blob` directly on Node.js 22, the process still +aborts mid-deserialization with the native fatal above — the snapshot's own +runtime guard can only print a friendly message on a runtime new enough to finish +deserializing but still below 24. So always restore through `egg-scripts` on +Node.js >= 24. +::: + +## Build and restore with the CLI + +The recommended workflow uses the bundler CLI. There is no `egg-bin snapshot +start`: building is a build-time concern (`egg-bin`), and restoring is a +production-runtime concern (`egg-scripts`). + +### Build the blob -## Public APIs +```bash +# bundles the app in snapshot mode (single self-contained worker.js + prelude) +# and runs `node --snapshot-blob --build-snapshot worker.js` for you +$ egg-bin snapshot build +``` + +By default this writes the bundle to `./dist-bundle` and the blob to +`./dist-bundle/snapshot.blob`. Useful options: + +| Option | Description | +| ------------------ | -------------------------------------------------------------- | +| `--output ` | Bundle output directory (also where `worker.js` lives). | +| `--blob ` | Snapshot blob path. Defaults to `/snapshot.blob`. | +| `--force-external` | Package to always keep external, repeatable (see Limitations). | +| `--skip-bundle` | Build the blob from an existing `worker.js` (skip bundling). | + +### Restore and serve + +Start a process directly from the blob with `egg-scripts` (Node.js >= 24): + +```bash +$ egg-scripts start --snapshot-blob ./dist-bundle/snapshot.blob --port 7001 +``` -Egg exports two public helpers from `egg`: +This launches a single self-contained `node --snapshot-blob ` process (no +egg-cluster, no framework resolution). The snapshot main reads the listen port +from `PORT` (or `--port`), runs `snapshotDidDeserialize` hooks, and calls +`app.listen()`. + +## Programmatic API + +For custom entry files, Egg also exports two helpers from `egg`: ```ts import { buildSnapshot, restoreSnapshot } from 'egg'; ``` -- `buildSnapshot()` starts Egg in snapshot mode, loads metadata, triggers cleanup hooks for non-serializable resources, and registers the application object into the V8 snapshot payload. -- `restoreSnapshot()` restores the application from that payload, recreates runtime-only resources, and resumes the remaining Egg lifecycle. - -## Build a Snapshot +- `buildSnapshot()` starts Egg in snapshot mode, loads metadata, triggers cleanup + hooks for non-serializable resources, and registers the application object into + the V8 snapshot payload. +- `restoreSnapshot()` restores the application from that payload, recreates + runtime-only resources, and resumes the remaining Egg lifecycle. -Create a snapshot entry file: +Build entry: ```ts import { buildSnapshot } from 'egg'; @@ -30,17 +100,11 @@ await buildSnapshot({ }); ``` -Then build the blob with Node.js: - ```bash node --snapshot-blob=snapshot.blob --build-snapshot snapshot-entry.mjs ``` -During snapshot build, Egg runs with `snapshot: true`. In this mode Egg loads application metadata but stops after `configWillLoad`, so hooks such as `configDidLoad`, `didLoad`, `willReady`, `didReady`, and `serverDidReady` are deferred until restore time. - -## Restore a Snapshot - -Start a process from the generated blob, then restore the application: +Restore entry (Node.js >= 24): ```ts import { restoreSnapshot } from 'egg'; @@ -49,11 +113,30 @@ const app = await restoreSnapshot(); await app.listen(7001); ``` -`restoreSnapshot()` resumes the normal startup flow from `configDidLoad` through `didReady`, so the returned app is ready for runtime setup such as creating servers or opening external connections. +`restoreSnapshot()` resumes the normal startup flow from `configDidLoad` through +`didReady`, so the returned app is ready for runtime setup such as creating +servers or opening external connections. + +## How it works + +During snapshot build, Egg runs with `snapshot: true`. In this mode Egg loads +application metadata but **stops after `configWillLoad`**, so hooks such as +`configDidLoad`, `didLoad`, `willReady`, `didReady`, and `serverDidReady` are +deferred until restore time. Before the heap is serialized, Egg runs +`snapshotWillSerialize` hooks to release non-serializable resources (timers, +sockets, native handles, logger streams). The network stack (`node:http`, +`node:https`, TLS/DNS, the HTTP client) is kept **external and lazy** so its +non-serializable native bindings are never captured in the blob; they are +reconnected on first use after restore. + +At restore time V8 deserializes the heap, then the snapshot main runs +`snapshotDidDeserialize` hooks to recreate those runtime-only resources, finishes +the deferred lifecycle through `didReady`, and starts listening. -## Snapshot Lifecycle Hooks +## Snapshot lifecycle hooks -If your `app.js` or `agent.js` boot class manages resources that cannot be serialized into a V8 snapshot, implement these hooks: +If your `app.js` or `agent.js` boot class manages resources that cannot be +serialized into a V8 snapshot, implement these hooks: ```js class AppBootHook { @@ -72,4 +155,43 @@ module.exports = AppBootHook; - `snapshotWillSerialize()` runs before the snapshot is written. - `snapshotDidDeserialize()` runs after the process starts from the snapshot. -Use these hooks for resources such as timers, sockets, process listeners, loggers, or other handles that must be recreated in a live runtime. +Use these hooks for resources such as timers, sockets, process listeners, +loggers, or other handles that must be recreated in a live runtime. + +## Performance + +Because the module graph is already loaded and the app is booted up to +`configWillLoad`, restore only pays for `didReady` and connecting/listening. +Measured on [cnpmcore](https://github.com/cnpm/cnpmcore): + +| Boot mode | Restore → listening | +| ------------------ | -------------------- | +| Normal bundle boot | ~942 ms | +| Snapshot restore | ~233 ms (~4x faster) | + +The win grows with the size of the module graph (plugins, tegg modules, routers), +which is exactly the cost a snapshot front-loads into build time. + +## Known limitations + +- **Restore requires Node.js >= 24** (see above). +- **Single process only**: the snapshot runs as one self-contained process + (`mode: 'single'`), like a bundle. Cluster mode is not supported. +- **Native addons are external** and must be present in the deploy target. +- **Third-party dependencies are constrained**: any dependency that opens live + resources or captures non-serializable state at module-evaluation time (open + sockets, native HTTP/2 bindings, background timers, file handles) must either + be kept external (`--force-external`) or implement the snapshot lifecycle hooks + so its state is released before serialization and rebuilt after restore. Not + every package is snapshot-safe out of the box. +- **Web globals are no-op stubs after restore**: at build the prelude replaces the + undici-backed globals (`fetch`/`Headers`/`Request`/`Response`/`FormData`/`WebSocket`, + plus `Blob`/`File`) with no-op stubs, because touching them at build time pulls in + Node's built-in undici and its native http/http2 bindings (which a snapshot cannot + serialize). Node's native lazy getters are themselves not snapshot-serializable, so + they cannot be reinstated on restore. **In the restored process those globals stay + stubs** — a snapshotted app should use a lazy-loaded HTTP client (e.g. `urllib`/`undici` + via an external, loaded for real on restore) rather than `globalThis.fetch` directly. + +The supported surface is still evolving; the full list of known limitations and +the design rationale are tracked in the project's V8 snapshot RFC. diff --git a/site/docs/zh-CN/advanced/snapshot.md b/site/docs/zh-CN/advanced/snapshot.md index d7d5ec1a89..f5daaeb3d8 100644 --- a/site/docs/zh-CN/advanced/snapshot.md +++ b/site/docs/zh-CN/advanced/snapshot.md @@ -5,22 +5,81 @@ order: 8 # V8 启动快照 -Egg 内置了 V8 startup snapshot 的构建与恢复能力。对于需要在启动前预加载框架元数据、插件、Service、Router 和 tegg 模块的应用，这可以减少冷启动阶段的开销。 +Egg 可以把一个完全加载好的应用固化成 [V8 启动快照](https://nodejs.org/api/cli.html#--build-snapshot)， +从而让冷启动跳过绝大部分启动开销。构建快照时会加载整个模块图——框架元数据、 +插件、Service、Router 和 tegg 模块——并把生命周期跑到 `configWillLoad`，再把此时的 +堆序列化成一个 blob。恢复 blob 时只需继续执行剩余生命周期（`didReady`）并开始监听， +进程几乎可以立即对外服务。 + +它构建在 [Bundle 部署](../core/bundle.md) 之上：快照是从单文件自包含 bundle 产出的， +因此可以把快照理解为「预启动好的 bundle」。 + +## Node.js 版本要求 + +| 阶段 | 命令 | Node.js | +| ---------------- | ----------------------------------- | ------- | +| **构建**快照 | `egg-bin snapshot build` | >= 22 | +| **恢复**（运行） | `egg-scripts start --snapshot-blob` | >= 24 | + +::: warning 恢复必须使用 Node.js >= 24 +快照可以在 Node.js >= 22 上**构建**，但在 Node.js 22 上**恢复**一个非平凡的 Egg 堆时， +进程会在反序列化阶段以原生 fatal 错误崩溃（`Check failed: current == end_slot_index`， +属于 V8 的 bug）。请始终在 Node.js **>= 24** 上恢复。 + +受支持的启动方式会强制拦截：`egg-scripts start --snapshot-blob` 在 Node.js < 24 时会在 +启动任何进程前直接报清晰错误并拒绝启动。如果你绕过它、在 Node.js 22 上直接运行 +`node --snapshot-blob`，进程仍会在反序列化阶段以上面的原生 fatal 崩溃——快照自带的运行时 +拦截只能在「能完成反序列化但仍低于 24」的版本上打印友好提示。因此请始终通过 `egg-scripts` +在 Node.js >= 24 上恢复。 +::: + +## 使用 CLI 构建与恢复 + +推荐使用 bundler CLI。注意没有 `egg-bin snapshot start`：构建属于构建期（`egg-bin`）， +恢复属于生产运行期（`egg-scripts`）。 + +### 构建 blob + +```bash +# 以快照模式打包（单文件自包含 worker.js + prelude），并自动执行 +# `node --snapshot-blob --build-snapshot worker.js` +$ egg-bin snapshot build +``` + +默认会把 bundle 写到 `./dist-bundle`，blob 写到 `./dist-bundle/snapshot.blob`。常用参数： + +| 参数 | 说明 | +| ------------------ | -------------------------------------------------- | +| `--output ` | bundle 输出目录（`worker.js` 所在目录）。 | +| `--blob ` | 快照 blob 路径，默认 `/snapshot.blob`。 | +| `--force-external` | 始终保持 external 的包，可重复（见「已知限制」）。 | +| `--skip-bundle` | 从已有的 `worker.js` 构建 blob（跳过打包）。 | + +### 恢复并提供服务 + +直接用 `egg-scripts` 从 blob 启动进程（Node.js >= 24）： + +```bash +$ egg-scripts start --snapshot-blob ./dist-bundle/snapshot.blob --port 7001 +``` + +它会启动一个单文件自包含的 `node --snapshot-blob ` 进程（没有 egg-cluster，也 +不做框架解析）。快照主函数从 `PORT`（或 `--port`）读取监听端口，执行 +`snapshotDidDeserialize` 钩子，然后调用 `app.listen()`。 ## 对外 API -Egg 从 `egg` 导出两个公开方法： +如果需要自定义入口文件，Egg 也从 `egg` 导出两个方法： ```ts import { buildSnapshot, restoreSnapshot } from 'egg'; ``` -- `buildSnapshot()` 会以快照模式启动 Egg，加载元数据，触发非可序列化资源的清理钩子，并把应用对象写入 V8 快照负载。 +- `buildSnapshot()` 会以快照模式启动 Egg，加载元数据，触发非可序列化资源的清理钩子， + 并把应用对象写入 V8 快照负载。 - `restoreSnapshot()` 会从快照中恢复应用，重建运行期资源，并继续执行剩余的 Egg 生命周期。 -## 构建快照 - -先创建一个构建入口文件： +构建入口： ```ts import { buildSnapshot } from 'egg'; @@ -30,17 +89,11 @@ await buildSnapshot({ }); ``` -然后使用 Node.js 生成快照 blob： - ```bash node --snapshot-blob=snapshot.blob --build-snapshot snapshot-entry.mjs ``` -在构建阶段，Egg 会以 `snapshot: true` 运行。此时 Egg 会加载应用元数据，但在 `configWillLoad` 之后停止，因此 `configDidLoad`、`didLoad`、`willReady`、`didReady` 和 `serverDidReady` 等钩子都会延后到恢复阶段执行。 - -## 恢复快照 - -从生成的 blob 启动进程后，调用恢复方法： +恢复入口（Node.js >= 24）： ```ts import { restoreSnapshot } from 'egg'; @@ -49,11 +102,25 @@ const app = await restoreSnapshot(); await app.listen(7001); ``` -`restoreSnapshot()` 会从 `configDidLoad` 继续执行正常启动流程直到 `didReady`，因此返回的 `app` 已经可以继续执行运行期初始化逻辑，例如启动服务或建立外部连接。 +`restoreSnapshot()` 会从 `configDidLoad` 继续执行正常启动流程直到 `didReady`，因此返回的 +`app` 已经可以继续执行运行期初始化逻辑，例如启动服务或建立外部连接。 + +## 工作原理 + +在构建阶段，Egg 会以 `snapshot: true` 运行。此时 Egg 会加载应用元数据，但在 +`configWillLoad` **之后停止**，因此 `configDidLoad`、`didLoad`、`willReady`、`didReady` +和 `serverDidReady` 等钩子都会延后到恢复阶段执行。在序列化堆之前，Egg 会运行 +`snapshotWillSerialize` 钩子，释放不可序列化的资源（timer、socket、原生句柄、logger +流）。网络栈（`node:http`、`node:https`、TLS/DNS、HTTP client）会保持 **external 且 +惰性加载**，因此它们不可序列化的原生绑定不会被写进 blob，而是在恢复后首次使用时重建。 + +恢复阶段，V8 先反序列化堆，随后快照主函数运行 `snapshotDidDeserialize` 钩子重建这些 +运行期资源，跑完延后的生命周期直到 `didReady`，然后开始监听。 ## 快照生命周期钩子 -如果你的 `app.js` 或 `agent.js` Boot 类管理了不能直接写入 V8 快照的资源，可以实现下面两个钩子： +如果你的 `app.js` 或 `agent.js` Boot 类管理了不能直接写入 V8 快照的资源，可以实现下面 +两个钩子： ```js class AppBootHook { @@ -72,4 +139,37 @@ module.exports = AppBootHook; - `snapshotWillSerialize()` 会在写入快照前执行。 - `snapshotDidDeserialize()` 会在进程从快照启动后执行。 -这两个钩子适合处理 timer、socket、process listener、logger 等需要在真实运行期重新建立的资源。 +这两个钩子适合处理 timer、socket、process listener、logger 等需要在真实运行期重新建立 +的资源。 + +## 性能 + +由于模块图已经加载、应用也已启动到 `configWillLoad`，恢复阶段只需要付出 `didReady` 与 +连接/监听的成本。在 [cnpmcore](https://github.com/cnpm/cnpmcore) 上实测： + +| 启动方式 | 恢复 → 监听 | +| ---------------- | -------------------- | +| 普通 bundle 启动 | ~942 ms | +| 快照恢复 | ~233 ms（快约 4 倍） | + +模块图越大（插件、tegg 模块、Router 越多），收益越明显——这正是快照在构建期提前承担的 +开销。 + +## 已知限制 + +- **恢复需要 Node.js >= 24**（见上文）。 +- **仅单进程**：快照以单个自包含进程运行（`mode: 'single'`），与 bundle 一致，不支持 + cluster 模式。 +- **原生 addon 为 external**，必须在部署目标上存在。 +- **第三方依赖受限**：任何在模块求值阶段就打开活跃资源或捕获不可序列化状态的依赖 + （打开的 socket、原生 HTTP/2 绑定、后台 timer、文件句柄）都必须要么保持 external + （`--force-external`），要么实现快照生命周期钩子，在序列化前释放、在恢复后重建。并不是 + 每个包都开箱即可被快照化。 +- **Web 全局对象在恢复后是惰性桩**：构建期 prelude 会把 undici 支撑的全局对象 + （`fetch`/`Headers`/`Request`/`Response`/`FormData`/`WebSocket`、以及 `Blob`/`File`）替换为 + 空操作的桩，否则在构建期触碰它们会拉起 Node 内建 undici 的原生 http/http2 绑定（无法被快照 + 序列化）。Node 的原生惰性 getter 本身不可被快照序列化，因此恢复后无法把它们还原。**恢复后的 + 进程里这些全局对象仍是桩**——快照化的应用应使用按需懒加载的 HTTP 客户端（如 `urllib`/`undici`， + 通过 external 在恢复期加载真实模块），而不要直接依赖 `globalThis.fetch`。 + +支持范围仍在演进中；完整的已知限制与设计取舍记录在项目的 V8 快照 RFC 中。 diff --git a/tools/egg-bin/src/commands/snapshot.ts b/tools/egg-bin/src/commands/snapshot.ts index d743e5b516..8ec282b0ad 100644 --- a/tools/egg-bin/src/commands/snapshot.ts +++ b/tools/egg-bin/src/commands/snapshot.ts @@ -142,6 +142,10 @@ export default class Snapshot extends BaseCommand throw new Error(`snapshot build finished but no blob was written at ${blobPath}`); } this.log(`snapshot blob written to ${blobPath}`); + // Building works on Node.js >= 22, but restoring the blob requires Node.js + // >= 24 (Node.js 22 aborts while deserializing a non-trivial egg heap). + // Surface that here so the requirement is visible at build time. + this.log('note: restoring this snapshot requires Node.js >= 24 (e.g. `egg-scripts start --snapshot-blob`)'); } async #spawnNode(nodeArgs: readonly string[], extraEnv: NodeJS.ProcessEnv = {}): Promise { diff --git a/tools/egg-bundler/src/lib/Bundler.ts b/tools/egg-bundler/src/lib/Bundler.ts index a711190ca3..b77d83ca43 100644 --- a/tools/egg-bundler/src/lib/Bundler.ts +++ b/tools/egg-bundler/src/lib/Bundler.ts @@ -10,7 +10,12 @@ import { ExternalsResolver } from './ExternalsResolver.ts'; import { assertFrameworkPackageSpecifier } from './frameworkSpecifier.ts'; import { ManifestLoader } from './ManifestLoader.ts'; import { PackRunner } from './PackRunner.ts'; -import { injectExternalRequireLazyHook, prependSnapshotPrelude, resolveSnapshotLazyModules } from './prelude.ts'; +import { + injectExternalRequireLazyHook, + prependSnapshotPrelude, + readExternalExports, + resolveSnapshotLazyModules, +} from './prelude.ts'; const debug = debuglog('egg/bundler/bundler'); @@ -420,8 +425,19 @@ export class Bundler { // externalRequire and prepend the prelude to each entry's worker.js so it runs // before the bundle IIFE (and therefore before any bundled module loads). if (snapshot) { + // Read each external's export names from the bundler process so the prelude's + // member proxy can present a full ESM namespace (`import { X } from 'pkg'`). + const externalExports = await wrapStep('read external exports', async () => + readExternalExports(absBaseDir, Object.keys(externalsMap)), + ); const applied = await wrapStep('apply snapshot prelude', () => - this.#applySnapshotPrelude(absOutputDir, ['worker'], snapshotLazyModules, patchResult.outputFiles), + this.#applySnapshotPrelude( + absOutputDir, + ['worker'], + snapshotLazyModules, + patchResult.outputFiles, + externalExports, + ), ); debug( 'snapshot: prepended prelude to %d entry file(s), injected lazy hook into %d externalRequire(s)', @@ -479,6 +495,7 @@ export class Bundler { entryNames: readonly string[], lazyModules: readonly string[], outputFiles: readonly string[], + externalExports: Readonly> = {}, ): Promise<{ prependedEntries: readonly string[]; injectedCount: number }> { const entrySet = new Set(entryNames.map((name) => this.#sanitizeOutputRelativePath(`${name}.js`))); const seenEntries = new Set(); @@ -511,7 +528,7 @@ export class Bundler { let next = hooked.content; if (isEntry) { - next = prependSnapshotPrelude(next, lazyModules); + next = prependSnapshotPrelude(next, lazyModules, externalExports); seenEntries.add(rel); prependedEntries.push(rel); } diff --git a/tools/egg-bundler/src/lib/EntryGenerator.ts b/tools/egg-bundler/src/lib/EntryGenerator.ts index 55c11951cd..cad896b875 100644 --- a/tools/egg-bundler/src/lib/EntryGenerator.ts +++ b/tools/egg-bundler/src/lib/EntryGenerator.ts @@ -178,6 +178,28 @@ export class EntryGenerator { .replaceAll(/\/+/g, '/'); } + /** + * The baseDir-relative keys of every tegg decorated file, used at runtime to + * re-stamp the correct `filePath` on decorated classes (whose decorator-captured + * path is unreliable in a bundle — see the worker entry comment). + */ + #collectDecoratedFileKeys(manifest: StartupManifest): string[] { + const tegg = manifest.extensions?.tegg as TeggManifestExtension | undefined; + const keys: string[] = []; + const seen = new Set(); + for (const desc of tegg?.moduleDescriptors ?? []) { + for (const rel of desc.decoratedFiles ?? []) { + const relKey = this.#teggRelKey(desc.unitPath, rel); + const normalized = relKey?.replaceAll(path.sep, '/'); + if (normalized && !seen.has(normalized)) { + seen.add(normalized); + keys.push(normalized); + } + } + } + return keys; + } + #collectResolveCacheAliases(manifest: StartupManifest): Array<[string, string]> { const aliases: Array<[string, string]> = []; for (const [requestRel, targetRel] of Object.entries(manifest.resolveCache)) { @@ -238,6 +260,7 @@ export class EntryGenerator { ), ); const appResolveCacheAliases = JSON.stringify(this.#collectResolveCacheAliases(manifest)); + const decoratedFileKeys = JSON.stringify(this.#collectDecoratedFileKeys(manifest)); const frameworkSpec = JSON.stringify(this.#framework); const externalBlock = @@ -316,6 +339,47 @@ for (const [appAbsRequest, targetRel] of __APP_RESOLVE_CACHE_ALIASES) { } } +// ── decorator file-path correction ────────────────────────────────────────── +// tegg decorators (@SingletonProto/@HTTPController/@Advice/…) capture a class's +// source path from the CALL STACK at module evaluation, via +// StackUtil.getCalleeFromStack(depth) with a hardcoded frame index. In a bundle +// every user frame collapses onto worker.js plus turbopack module wrappers, so a +// decorator that uses a deeper index than the norm — notably @Advice's depth-5 vs +// @SingletonProto's depth-4 — captures the runtime frame ("…/worker.js") instead +// of its own source file. tegg then cannot match that proto to its load unit and +// fails at restore with "Aop Advice(X) not found in loadUnits". +// +// The bundle DOES know each decorated file's real path (the manifest's tegg +// decoratedFiles, surfaced here as __DECORATED_FILE_KEYS). Re-stamp the correct +// path (outputDir + relKey, matching the format the well-behaved decorators get) +// on every decorated export, overriding whatever the decorator captured. This runs +// at build, so the corrected paths are baked into the snapshot the restore reads. +const __DECORATED_FILE_KEYS: string[] = ${decoratedFileKeys}; +{ + const __FILE_PATH_META = Symbol.for('EggPrototype.filePath'); + const __reflect = Reflect as unknown as { + hasOwnMetadata?: (key: symbol, target: unknown) => boolean; + defineMetadata?: (key: symbol, value: unknown, target: unknown) => void; + }; + if (typeof __reflect.hasOwnMetadata === 'function' && typeof __reflect.defineMetadata === 'function') { + for (const __rel of __DECORATED_FILE_KEYS) { + const __mod = __getBundleMap(__rel) as Record | undefined; + if (!__mod || (typeof __mod !== 'object' && typeof __mod !== 'function')) continue; + const __abs = path.resolve(__outputDir, __rel); + for (const __k of Object.keys(__mod)) { + const __exported = __mod[__k]; + if ( + __exported && + (typeof __exported === 'function' || typeof __exported === 'object') && + __reflect.hasOwnMetadata(__FILE_PATH_META, __exported) + ) { + __reflect.defineMetadata(__FILE_PATH_META, __abs, __exported); + } + } + } + } +} + // Tegg module reference / descriptor paths are stored relative to baseDir in the // manifest. Resolve them to absolute paths under the runtime output dir so every // tegg loader consumer (ModuleConfigUtil, EggAppLoader, LoaderFactory.loadApp @@ -361,6 +425,44 @@ if (process.env.EGG_BUNDLE_SNAPSHOT === 'build') { // no servers/timers/connections), run the snapshotWillSerialize hooks to release // non-serializable resources, then register the deserialize main function that // V8 invokes when restoring from the blob. + // + // Install the egg loader module importer for the BUILD phase too: the loader + // loads config/app files, and under --build-snapshot Node dynamic import() is + // unavailable, so route through the bundle map / require() instead. Do NOT set + // __RUNTIME_REQUIRE here — that would make externals load for real and defeat + // the lazy snapshot stubs; __EGG_MODULE_IMPORTER__ only feeds app/config files. + { + // getBuiltinModule needs Node >= 22.3; fall back to an eval'd require for + // 22.0–22.2 (same opaque-specifier trick the restore branch uses below). + const { createRequire: __cr } = + typeof process.getBuiltinModule === 'function' + ? process.getBuiltinModule('node:module') + : (0, eval)('require')('node:module'); + const __buildReq = __cr(__outputDir + '/'); + globalThis.__EGG_MODULE_IMPORTER__ = async (fp: string) => { + const bundled = __getBundleMap(fp); + if (bundled !== undefined) return bundled; + // Only skip when fp ITSELF cannot be resolved (a manifest-miss config unit the + // loader tolerates). Resolving first separates that from a nested MODULE_NOT_FOUND + // raised while loading a resolved file, which is a genuine missing dependency that + // must surface rather than be silently swallowed. + let resolved: string; + try { + resolved = __buildReq.resolve(fp); + } catch { + return undefined; + } + try { + return __buildReq(resolved); + } catch (err) { + // A resolved ESM file cannot be require()'d under --build-snapshot (no ESM + // loader); skip it. Any other error — including a nested MODULE_NOT_FOUND from + // a real missing dependency — is genuine and propagates. + if ((err as { code?: string } | undefined)?.code === 'ERR_INTERNAL_ASSERTION') return undefined; + throw err; + } + }; + } startEgg({ ...__startOptions, snapshot: true }).then(async (app) => { if (app.agent) { await app.agent.triggerSnapshotWillSerialize(); @@ -372,6 +474,18 @@ if (process.env.EGG_BUNDLE_SNAPSHOT === 'build') { // V8 runs this callback synchronously right after deserialization, before // the Node ESM loader is ready, so all real work is deferred to the next // tick via setImmediate. + // + // Restoring a V8 snapshot requires Node.js >= 24: Node.js 22 aborts while + // deserializing a non-trivial egg heap (V8 bug). The supported launcher + // (egg-scripts start --snapshot-blob) already gates this before spawning, + // so this is a defense-in-depth guard for a direct \`node --snapshot-blob\` + // launch that managed to deserialize on an unsupported runtime. + const __nodeMajor = parseInt(process.versions.node, 10); + if (__nodeMajor < 24) { + // eslint-disable-next-line no-console + console.error('[egg-bundler] V8 snapshot restore requires Node.js >= 24, but this process is ' + process.version + '. Build works on Node.js >= 22; restore must run on Node.js >= 24.'); + process.exit(1); + } setImmediate(() => { // A restored snapshot process has no dynamic import() callback // (ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING). Route every egg loader import diff --git a/tools/egg-bundler/src/lib/PackRunner.ts b/tools/egg-bundler/src/lib/PackRunner.ts index d1743117e4..f7b443dd4f 100644 --- a/tools/egg-bundler/src/lib/PackRunner.ts +++ b/tools/egg-bundler/src/lib/PackRunner.ts @@ -139,13 +139,23 @@ export class PackRunner { await fs.mkdir(projectPath, { recursive: true }); await fs.writeFile(projectTsconfigPath, desiredTsconfig); - // UMD-form externals ({ commonjs, root }) make @utoo/pack's standalone - // output emit a `require(name)` branch under `typeof exports === 'object'`, - // which is what node picks. Plain string externals only emit the - // `globalThis[name]` branch, unusable for direct node execution. - const umdExternals: Record = {}; + // External-config form depends on the output type: + // + // - standalone: UMD form ({ commonjs, root }) emits a + // `typeof exports === 'object' ? require(name) : globalThis[name]` guard. + // The standalone loader runs module factories with a real CommonJS + // module/exports, so the guard picks `require(name)` — what node wants. + // + // - single-file (snapshot): the `export`/`library` output inlines every + // factory into one IIFE with NO CommonJS module/exports in scope, so the + // UMD guard falls through to `globalThis[name]` (undefined) and EVERY + // external breaks. ExternalType `commonjs` instead emits a direct + // `require(name)` (surfaced as the runtime `externalRequire` helper), which + // the snapshot prelude's lazy hook can intercept. See the snapshot + // lazy-external mechanism in prelude.ts. + const externalsConfig: Record = {}; for (const [k, v] of Object.entries(externals)) { - umdExternals[k] = { commonjs: v, root: v }; + externalsConfig[k] = singleFile ? { root: v, type: 'commonjs' } : { commonjs: v, root: v }; } const resolveConfig = this.#buildResolveConfig(resolve); @@ -169,7 +179,7 @@ export class PackRunner { path: outputDir, type: singleFile ? 'export' : 'standalone', }, - externals: umdExternals, + externals: externalsConfig, ...(resolveConfig ? { resolve: resolveConfig } : {}), optimization: { treeShaking: false, diff --git a/tools/egg-bundler/src/lib/prelude.ts b/tools/egg-bundler/src/lib/prelude.ts index 85616a02f1..d59a699173 100644 --- a/tools/egg-bundler/src/lib/prelude.ts +++ b/tools/egg-bundler/src/lib/prelude.ts @@ -4,16 +4,26 @@ * In snapshot mode the bundler prepends this prelude to the emitted single-file * `worker.js`, BEFORE the bundle IIFE (`((__UTOOPACK__)=>{...})([...modules])`), * so it runs before any bundled module is evaluated. It installs the - * lazy-external / native-binding stub mechanism that keeps a V8 startup snapshot - * serializable: the Node network stack (http/https/http2/tls/dns) is NOT loaded - * while the snapshot is built (its native bindings — HTTPParser, nghttp2 - * settingsBuffer, tls SecureContext, dns ChannelWrap — cannot be serialized, and - * `WebAssembly` is disabled under `--build-snapshot`), then forwarded to the real - * module at restore time via `globalThis.__RUNTIME_REQUIRE` (installed by the - * generated snapshot-restore entry). + * lazy-external / native-binding mechanism that keeps a V8 startup snapshot + * serializable: + * + * 1. Node's web globals (fetch/Headers/.../File/Blob) are replaced with plain JS + * stubs so a bundled module touching them at evaluation time neither crashes + * (delete → ReferenceError) nor pulls in Node's built-in undici (whose llhttp + * HTTPParser / nghttp2 native bindings cannot be V8-snapshot-serialized). + * 2. `node:buffer.File/Blob` getters are stubbed for the same reason (reading + * them lazily initializes the undici/http stack). + * 3. Every external require is routed through `__makeLazyExt`, a member-proxy + * that returns a build-time stub and, at restore time, forwards to the real + * module via `globalThis.__RUNTIME_REQUIRE` — recording the access path so + * `class X extends pkg.Klass` / `DataTypes.INTEGER(11).UNSIGNED` keep working. + * + * The real globals/modules are present in the restored (live) process. */ import { promises as fs } from 'node:fs'; +import http from 'node:http'; +import { createRequire } from 'node:module'; import path from 'node:path'; import { debuglog } from 'node:util'; @@ -26,17 +36,20 @@ const debug = debuglog('egg/bundler/snapshot-prelude'); export const SNAPSHOT_PRELUDE_MARKER = '@eggjs/egg-bundler:snapshot-prelude'; /** - * Node built-in network modules that produce non-serializable native bindings when - * loaded inside a V8 startup snapshot builder: - * - * - `node:http` / `node:https` create an `HTTPParser` (llhttp) C++ global handle. - * - `node:http2` creates `HTTPParser` + an `nghttp2` `settingsBuffer` Uint32Array. - * - `node:tls` creates a `SecureContext`. - * - `node:dns` creates a `ChannelWrap`. + * Node built-in modules that produce non-serializable native bindings when loaded + * inside a V8 startup snapshot builder. They are kept as lazy externals: build time + * returns a member-proxy stub; restore time forwards to the real module via + * `globalThis.__RUNTIME_REQUIRE`. * - * Egg's loader phase touches the HTTP/TLS/DNS stack (HttpClient, agents, etc.), so - * these are kept as lazy externals: build time returns a stub Proxy; restore time - * forwards to the real module via `globalThis.__RUNTIME_REQUIRE`. + * - network stack (HTTPParser, nghttp2, SecureContext, ChannelWrap): + * http/https/http2/tls/dns. + * - `inspector`: a builtin would normally load for real at build (it is not a + * business package, so the `!isBuiltin` lazy condition does not catch it), but + * `egg` core does `import inspector from 'node:inspector'` and evaluates + * `inspector.url()` while building config — which initializes the CDP stack + * (readline/repl + http2 nghttp2 native), making the heap unserializable. Keep + * it lazy so the build-time stub is used; the live process gets the real module + * on restore. */ export const DEFAULT_SNAPSHOT_LAZY_MODULES: readonly string[] = [ 'http', @@ -49,14 +62,17 @@ export const DEFAULT_SNAPSHOT_LAZY_MODULES: readonly string[] = [ 'node:tls', 'dns', 'node:dns', + 'inspector', + 'node:inspector', ]; /** - * Node's lazy web globals. Accessing any of them lazily initializes undici, whose - * llhttp parser allocates a `WebAssembly` instance + `HTTPParser` binding that the - * snapshot builder cannot serialize. They are removed with `delete` before any - * bundled module runs. `Object.defineProperty` is NOT usable here: redefining the - * lazy accessor triggers the very undici load we are avoiding. + * Node's web globals. They are backed by undici (fetch/Headers/Request/Response/ + * FormData/WebSocket/EventSource) or by node:buffer (File/Blob); touching the + * undici-backed ones lazily initializes undici (llhttp HTTPParser + WebAssembly), + * which the snapshot builder cannot serialize. They are replaced with build-time + * stubs (NOT `delete`d — a deleted global throws ReferenceError when a bundled + * module references it; a stub is referencable and harmless at build time). */ const WEB_GLOBALS: readonly string[] = [ 'fetch', @@ -72,113 +88,13 @@ const WEB_GLOBALS: readonly string[] = [ 'Blob', ]; -/** `http.METHODS` — hardcoded so a library's top-level `[...http.METHODS]` does not force a build-time load. */ -const HTTP_METHODS: readonly string[] = [ - 'ACL', - 'BIND', - 'CHECKOUT', - 'CONNECT', - 'COPY', - 'DELETE', - 'GET', - 'HEAD', - 'LINK', - 'LOCK', - 'M-SEARCH', - 'MERGE', - 'MKACTIVITY', - 'MKCALENDAR', - 'MKCOL', - 'MOVE', - 'NOTIFY', - 'OPTIONS', - 'PATCH', - 'POST', - 'PROPFIND', - 'PROPPATCH', - 'PURGE', - 'PUT', - 'QUERY', - 'REBIND', - 'REPORT', - 'SEARCH', - 'SOURCE', - 'SUBSCRIBE', - 'TRACE', - 'UNBIND', - 'UNLINK', - 'UNLOCK', - 'UNSUBSCRIBE', -]; - -/** `http.STATUS_CODES` — hardcoded for the same reason as METHODS. */ -const HTTP_STATUS_CODES: Readonly> = { - '100': 'Continue', - '101': 'Switching Protocols', - '102': 'Processing', - '103': 'Early Hints', - '200': 'OK', - '201': 'Created', - '202': 'Accepted', - '203': 'Non-Authoritative Information', - '204': 'No Content', - '205': 'Reset Content', - '206': 'Partial Content', - '207': 'Multi-Status', - '208': 'Already Reported', - '226': 'IM Used', - '300': 'Multiple Choices', - '301': 'Moved Permanently', - '302': 'Found', - '303': 'See Other', - '304': 'Not Modified', - '305': 'Use Proxy', - '307': 'Temporary Redirect', - '308': 'Permanent Redirect', - '400': 'Bad Request', - '401': 'Unauthorized', - '402': 'Payment Required', - '403': 'Forbidden', - '404': 'Not Found', - '405': 'Method Not Allowed', - '406': 'Not Acceptable', - '407': 'Proxy Authentication Required', - '408': 'Request Timeout', - '409': 'Conflict', - '410': 'Gone', - '411': 'Length Required', - '412': 'Precondition Failed', - '413': 'Payload Too Large', - '414': 'URI Too Long', - '415': 'Unsupported Media Type', - '416': 'Range Not Satisfiable', - '417': 'Expectation Failed', - '418': "I'm a Teapot", - '421': 'Misdirected Request', - '422': 'Unprocessable Entity', - '423': 'Locked', - '424': 'Failed Dependency', - '425': 'Too Early', - '426': 'Upgrade Required', - '428': 'Precondition Required', - '429': 'Too Many Requests', - '431': 'Request Header Fields Too Large', - '451': 'Unavailable For Legal Reasons', - '500': 'Internal Server Error', - '501': 'Not Implemented', - '502': 'Bad Gateway', - '503': 'Service Unavailable', - '504': 'Gateway Timeout', - '505': 'HTTP Version Not Supported', - '506': 'Variant Also Negotiates', - '507': 'Insufficient Storage', - '508': 'Loop Detected', - '509': 'Bandwidth Limit Exceeded', - '510': 'Not Extended', - '511': 'Network Authentication Required', -}; - -const HTTP_MAX_HEADER_SIZE = 16384; +/** + * `node:buffer` is a real, serializable builtin (Buffer is needed at build time), + * but reading its `File`/`Blob` getters lazily initializes Node's built-in undici + * (→ http/http2 native bindings). Those two property getters are stubbed at build; + * the live process re-exposes the real ones on restore. + */ +const BUFFER_DANGEROUS_PROPS: readonly string[] = ['File', 'Blob']; interface AppPackageJson { readonly egg?: { @@ -223,16 +139,23 @@ export async function resolveSnapshotLazyModules(baseDir: string): Promise> = {}, +): string { const lazyJson = JSON.stringify([...lazyModules]); const webJson = JSON.stringify([...WEB_GLOBALS]); - const methodsJson = JSON.stringify([...HTTP_METHODS]); - const statusJson = JSON.stringify(HTTP_STATUS_CODES); + const bufDangerJson = JSON.stringify([...BUFFER_DANGEROUS_PROPS]); + const externalExportsJson = JSON.stringify(externalExports); + const httpConstsJson = JSON.stringify({ + METHODS: http.METHODS, + STATUS_CODES: http.STATUS_CODES, + maxHeaderSize: http.maxHeaderSize, + }); return `// ⚠️ auto-generated by @eggjs/egg-bundler — snapshot prelude (do not edit) // marker: ${SNAPSHOT_PRELUDE_MARKER} @@ -240,110 +163,158 @@ export function renderSnapshotPrelude(lazyModules: readonly string[] = DEFAULT_S /* eslint-disable */ (function eggBundlerSnapshotPrelude() { 'use strict'; - // Drop Node's lazy web globals before any bundled module touches them. These - // getters lazily initialize undici, whose llhttp HTTPParser / WebAssembly cannot - // be V8-snapshot-serialized. MUST use \`delete\`: redefining the lazy accessor - // would trigger the very load we are avoiding. + // Neutralize Node's undici-backed web globals (fetch/Headers/Request/...) so they + // never lazily initialize Node's undici stack (llhttp HTTPParser + nghttp2), whose + // native bindings a V8 startup snapshot cannot serialize. + // + // This MUST be done in two passes: + // 1. delete the global first. Node defines these as lazy accessor properties; a + // plain redefine via Object.defineProperty (e.g. of \`Headers\`) makes Node load + // undici → http2 native eagerly — the exact thing we must avoid. \`delete\` + // removes the lazy getter WITHOUT triggering it. + // 2. then install a no-op stub CLASS as a plain data property. It is now safe (no + // lazy getter remains to trigger) and constructable, so a bundled package doing + // \`class X extends globalThis.Request {}\` still works (\`delete\`/\`undefined\` + // alone would throw "Class extends value undefined" at class definition). + // The live process re-exposes the real globals on restore. var __WEB_GLOBALS = ${webJson}; for (var __i = 0; __i < __WEB_GLOBALS.length; __i++) { try { delete globalThis[__WEB_GLOBALS[__i]]; } catch (e) {} } + for (var __k = 0; __k < __WEB_GLOBALS.length; __k++) { + try { Object.defineProperty(globalThis, __WEB_GLOBALS[__k], { value: function WebGlobalStub(){}, configurable: true, writable: true }); } catch (e) {} + } if (globalThis.__LAZY_EXT) return; - // Set of module ids treated as lazy externals. The patched externalRequire - // forwards these here instead of requiring the real module at build time. + // Set of module ids treated as lazy externals (the patched externalRequire + // forwards these to __makeLazyExt instead of loading the real module at build). globalThis.__LAZY_EXT = new Set(${lazyJson}); - // Hardcoded http constants so top-level \`[...http.METHODS]\` or - // \`Object.keys(http.STATUS_CODES)\` in a bundled library does not force a - // build-time load of the real (non-serializable) http module. - var __HTTP_METHODS = ${methodsJson}; - var __HTTP_STATUS_CODES = ${statusJson}; - var __HTTP_MAX_HEADER_SIZE = ${HTTP_MAX_HEADER_SIZE}; + // http constants captured from the build Node (no hand-written tables). + globalThis.__HTTP_CONSTS = ${httpConstsJson}; + + // External-package export names, read at build by the bundler. The member proxy + // exposes these via ownKeys so @utoo/pack's interopEsm builds a full ESM + // namespace and \`import { X } from 'pkg'\` resolves to a member-proxy (not undefined). + globalThis.__EXTERNAL_EXPORTS = ${externalExportsJson}; + + // node:buffer is real at build (Buffer is needed) but its File/Blob getters + // trigger Node's built-in undici. Stub just those two; restore re-exposes them. + (function () { + try { + var __buf = process.getBuiltinModule('node:buffer'); + var __bd = ${bufDangerJson}; + for (var __j = 0; __j < __bd.length; __j++) { + try { Object.defineProperty(__buf, __bd[__j], { value: function BufStub(){}, configurable: true, writable: true }); } catch (e) {} + } + } catch (e) {} + })(); + + // isBuiltin: tells builtin "tool" modules (path/fs/module — load for real at + // build) apart from non-builtin packages (lazy-stubbed at build). + globalThis.__isBuiltin = (function () { + try { return process.getBuiltinModule('node:module').isBuiltin; } catch (e) { return function () { return false; }; } + })(); globalThis.__makeLazyExt = function (id, thunk) { + var realMod = function () { var rt = globalThis.__RUNTIME_REQUIRE; return rt ? rt(id) : undefined; }; + var EXPORTS = (globalThis.__EXTERNAL_EXPORTS && globalThis.__EXTERNAL_EXPORTS[id]) || []; var isHttp = id === 'http' || id === 'node:http' || id === 'https' || id === 'node:https'; - // Build time: globalThis.__RUNTIME_REQUIRE is unset -> returns undefined, the - // real module is never loaded. Restore time: __RUNTIME_REQUIRE (installed by the - // generated snapshot-restore entry) really requires the module, so - // http.createServer is the genuine builtin and the app truly listens. - var realModule = function () { - var rt = globalThis.__RUNTIME_REQUIRE; - return rt ? rt(id) : undefined; - }; - var buildConst = function (prop) { - if (!isHttp) return undefined; - if (prop === 'METHODS') return __HTTP_METHODS; - if (prop === 'STATUS_CODES') return __HTTP_STATUS_CODES; - if (prop === 'maxHeaderSize') return __HTTP_MAX_HEADER_SIZE; - return undefined; + + // Call args captured at build can themselves be member-proxies — e.g. a model column + // \`DataTypes.TEXT(LENGTH_VARIANTS.long)\` where LENGTH_VARIANTS is also a lazy export. + // They must be resolved to their real values before reaching the real callee, or the + // callee receives a Proxy and mishandles it (leoric coerces it to a string in an error + // template and throws "String.prototype.toString requires that 'this' be a String"). + // A member-proxy returns its resolved value via the __MR symbol; anything else passes + // through unchanged. + var __MR = globalThis.__MEMBER_RESOLVE || (globalThis.__MEMBER_RESOLVE = Symbol.for('@eggjs/egg-bundler:memberResolve')); + var resolveArg = function (a) { + if (a && (typeof a === 'object' || typeof a === 'function')) { + try { var rv = a[__MR]; if (rv !== undefined) return rv; } catch (e) {} + } + return a; }; + var resolveArgs = function (args) { var out = []; for (var i = 0; i < args.length; i++) out.push(resolveArg(args[i])); return out; }; + + // A member-proxy records the access path (get/apply/construct + call args) + // taken at build time and replays it against the real module on restore, so + // e.g. \`class X extends urllib.HttpClient\` (build: stub superclass; restore: + // real super()/methods) and \`DataTypes.INTEGER(11).UNSIGNED\` keep working. + function makeMember(ops) { + var resolve = function () { + var v = realMod(), prev; + for (var i = 0; i < ops.length; i++) { + if (v == null) return undefined; + var op = ops[i]; + if (op.t === 'g') { prev = v; v = v[op.k]; } + // Use Reflect.apply/construct (invoke [[Call]]/[[Construct]] directly) rather + // than v.apply(...): when an earlier step resolved to a member-proxy (a callable + // Proxy), \`typeof v === 'function'\` is true but \`v.apply\` is undefined, so + // \`v.apply(...)\` throws "v.apply is not a function" (hit by e.g. leoric's + // createType introspecting a proxied DataType). Reflect.apply works on any callable. + else if (op.t === 'a') { v = (typeof v === 'function') ? Reflect.apply(v, prev, resolveArgs(op.args)) : undefined; prev = undefined; } + else if (op.t === 'c') { v = (typeof v === 'function') ? Reflect.construct(v, resolveArgs(op.args)) : undefined; prev = undefined; } + } + return v; + }; + var protoProxy = new Proxy({}, { get: function (t, p) { var r = resolve(); return r && r.prototype ? r.prototype[p] : undefined; } }); + // Pick the proxy target so \`typeof member\` matches what the resolved value will be: + // a call/construct RESULT is normally an instance (typeof 'object'), while a plain + // member access is usually a class/function (typeof 'function'). Libraries branch on + // \`typeof x === 'function'\` (e.g. leoric's createType: function ⇒ DataType class, + // object ⇒ DataType instance) — a uniformly-function proxy would take the wrong path. + // (A non-callable target makes the apply/construct traps dead, which is correct: a + // resolved instance is not meant to be called.) + var __lastOp = ops.length ? ops[ops.length - 1] : null; + var __target = __lastOp && (__lastOp.t === 'a' || __lastOp.t === 'c') ? {} : function () {}; + var member = new Proxy(__target, { + get: function (t, p) { if (p === __MR) return resolve(); if (p === 'prototype') return protoProxy; var r = resolve(); if (r != null) return r[p]; if (p === 'then') return undefined; if (typeof p === 'symbol') return undefined; return makeMember(ops.concat([{ t: 'g', k: p }])); }, + apply: function (t, thisArg, args) { var r = resolve(); if (typeof r === 'function') return Reflect.apply(r, thisArg, resolveArgs(args)); return makeMember(ops.concat([{ t: 'a', args: args }])); }, + construct: function (t, args, nt) { var r = resolve(); if (typeof r === 'function') return Reflect.construct(r, resolveArgs(args), nt || r); return makeMember(ops.concat([{ t: 'c', args: args }])); } + }); + return member; + } + var proxy = new Proxy(function () {}, { get: function (target, prop) { + if (prop === __MR) return realMod(); if (prop === 'default') return proxy; - var real = realModule(); - if (real !== undefined && real !== null) { - return real[prop]; - } - // --- build time only below --- - if (typeof prop === 'string') { - var c = buildConst(prop); - if (c !== undefined) return c; - } + var real = realMod(); + if (real != null) return real[prop]; + // Build-time http constants so a library iterating http.METHODS etc. + // (e.g. \`for (const m of http.METHODS)\`) does not crash. + if (isHttp && typeof prop === 'string' && globalThis.__HTTP_CONSTS && Object.prototype.hasOwnProperty.call(globalThis.__HTTP_CONSTS, prop)) return globalThis.__HTTP_CONSTS[prop]; if (prop === '__esModule') return undefined; - if (typeof prop === 'symbol') return undefined; - // Never expose a build-time \`then\`: returning the callable proxy would make - // the stub a thenable, so \`await require(id)\` / Promise.resolve(stub) hangs - // (apply never resolves). At restore the real module forwards \`then\` above. if (prop === 'then') return undefined; - // Satisfy Proxy invariants: the function target's own non-configurable - // props (prototype/length/name) must be reported faithfully. - if (prop === 'prototype' || prop === 'name' || prop === 'length') { - return Reflect.get(target, prop); - } - return proxy; // chainable build-time stub - }, - apply: function (target, thisArg, args) { - var real = realModule(); - if (typeof real === 'function') return Reflect.apply(real, thisArg, args); - return undefined; - }, - construct: function (target, args) { - var real = realModule(); - if (typeof real === 'function') return Reflect.construct(real, args); - return proxy; // build time: keep \`new Stub().method()\` chainable - }, - has: function (target, prop) { - var real = realModule(); - if (real !== undefined && real !== null) return prop in real; - return true; + if (typeof prop === 'symbol') return undefined; + if (prop === 'prototype' || prop === 'name' || prop === 'length') return Reflect.get(target, prop); + return makeMember([{ t: 'g', k: prop }]); }, - // Structural traps. At restore time reflect the REAL module's keys so - // Object.keys / spread / destructuring-rest over e.g. require('http') see the - // genuine exports; at build time fall back to the target. Either way the - // function target's own non-configurable keys (prototype) stay reported so the - // Proxy invariants hold. Real descriptors are forced configurable to avoid the - // "report a non-configurable prop absent from target" invariant violation. + apply: function (target, thisArg, args) { var real = realMod(); if (typeof real === 'function') return Reflect.apply(real, thisArg, resolveArgs(args)); return undefined; }, + construct: function (target, args, nt) { var real = realMod(); if (typeof real === 'function') return Reflect.construct(real, resolveArgs(args), nt || real); return Object.create((nt && nt.prototype) || target.prototype); }, + has: function (target, prop) { var real = realMod(); if (real != null) return prop in real; return true; }, + // Expose the external's real export names at build time (from + // __EXTERNAL_EXPORTS, injected by the entry) so @utoo/pack's interopEsm — + // which enumerates Object.getOwnPropertyNames(raw) — builds a full ESM + // namespace whose named bindings each point at a member-proxy. Without this + // \`import { HttpClient } from 'urllib'\` resolves to undefined. ownKeys: function (target) { - var real = realModule(); - if (real === undefined || real === null) return Reflect.ownKeys(target); - var keys = Reflect.ownKeys(real); - var targetKeys = Reflect.ownKeys(target); - for (var i = 0; i < targetKeys.length; i++) { - if (keys.indexOf(targetKeys[i]) === -1) keys.push(targetKeys[i]); - } - return keys; + var real = realMod(); + var base = (real != null) ? Reflect.ownKeys(real) : EXPORTS.slice(); + var tk = Reflect.ownKeys(target); + for (var i = 0; i < tk.length; i++) if (base.indexOf(tk[i]) === -1) base.push(tk[i]); + return base; }, getOwnPropertyDescriptor: function (target, prop) { - var targetDesc = Reflect.getOwnPropertyDescriptor(target, prop); - if (targetDesc && !targetDesc.configurable) return targetDesc; - var real = realModule(); - if (real === undefined || real === null) return targetDesc; - var desc = Reflect.getOwnPropertyDescriptor(real, prop); - if (desc) desc.configurable = true; - return desc; - }, + var td = Reflect.getOwnPropertyDescriptor(target, prop); + if (td && !td.configurable) return td; + var real = realMod(); + if (real != null) { var d = Reflect.getOwnPropertyDescriptor(real, prop); if (d) d.configurable = true; return d; } + if (typeof prop === 'string' && EXPORTS.indexOf(prop) !== -1) return { value: makeMember([{ t: 'g', k: prop }]), configurable: true, enumerable: true, writable: true }; + return td; + } }); return proxy; }; @@ -360,6 +331,7 @@ export function renderSnapshotPrelude(lazyModules: readonly string[] = DEFAULT_S export function prependSnapshotPrelude( source: string, lazyModules: readonly string[] = DEFAULT_SNAPSHOT_LAZY_MODULES, + externalExports: Readonly> = {}, ): string { // Only look for the marker in the file head (the prelude is short and always // sits at the very top). A whole-file `includes` would false-positive if any @@ -369,7 +341,7 @@ export function prependSnapshotPrelude( return source; } - const prelude = renderSnapshotPrelude(lazyModules); + const prelude = renderSnapshotPrelude(lazyModules, externalExports); const lines = source.split('\n'); let insertAt = 0; @@ -391,6 +363,57 @@ export function prependSnapshotPrelude( return `${head}\n${prelude}${tail}`; } +/** + * Read the export names of every external id (lazy network builtins + external + * packages) from the BUILD process, so {@link renderSnapshotPrelude}'s member + * proxy can present a full ESM namespace for `import { X } from 'pkg'`. Runs in + * the bundler process (NOT the snapshot), so requiring a package here is harmless; + * ids that cannot be required (e.g. a missing optional native binary) are skipped. + */ +export function readExternalExports(baseDir: string, ids: Iterable): Record { + const req = createRequire(path.join(baseDir, 'package.json')); + let isBuiltin: ((id: string) => boolean) | undefined; + try { + isBuiltin = (process.getBuiltinModule('node:module') as { isBuiltin?: (id: string) => boolean }).isBuiltin; + } catch { + isBuiltin = undefined; + } + // Function/class internals that are own-properties but never real exports. + const FN_INTERNALS = new Set(['length', 'name', 'prototype', 'arguments', 'caller']); + const collect = (mod: unknown): string[] => { + const set = new Set(); + const add = (o: unknown) => { + if (!o || (typeof o !== 'object' && typeof o !== 'function')) return; + // getOwnPropertyNames (not Object.keys) so NON-ENUMERABLE named exports are + // included — @utoo/pack's interopEsm enumerates getOwnPropertyNames(raw) to build + // the ESM namespace, so EXPORTS must match or `import { X }` would resolve to + // undefined for a non-enumerable X. + for (const k of Object.getOwnPropertyNames(o)) { + if (typeof o === 'function' && FN_INTERNALS.has(k)) continue; + set.add(k); + } + }; + add(mod); + // CJS packages required as ESM expose named exports on `default`; merge them + // (e.g. leoric's `DataTypes`/`Bone` only show up under default via import). + if (mod && typeof mod === 'object') add((mod as Record).default); + return [...set]; + }; + const out: Record = {}; + for (const id of ids) { + try { + const mod = isBuiltin?.(id) + ? process.getBuiltinModule(id as Parameters[0]) + : req(id); + const names = collect(mod); + if (names.length > 0) out[id] = names; + } catch (err) { + debug('skip external exports for %s: %s', id, err instanceof Error ? err.message : err); + } + } + return out; +} + /** * The source `@utoo/pack` (Turbopack) emits for its external-require helper. The * lazy hook is injected right after the opening brace, using the helper's own @@ -408,11 +431,17 @@ export interface ExternalRequireInjectionResult { * Inject the lazy dispatch at the start of every `externalRequire` body: * * ```js - * if (globalThis.__LAZY_EXT && globalThis.__LAZY_EXT.has(id)) return globalThis.__makeLazyExt(id, thunk); + * if (globalThis.__makeLazyExt && !globalThis.__RUNTIME_REQUIRE && + * (globalThis.__LAZY_EXT.has(id) || !globalThis.__isBuiltin(id))) + * return globalThis.__makeLazyExt(id, thunk); * ``` * - * so a require of a lazy module id is rerouted to {@link renderSnapshotPrelude}'s - * `__makeLazyExt` instead of loading the real (non-serializable) module. + * so at BUILD time a require is rerouted to {@link renderSnapshotPrelude}'s + * `__makeLazyExt` when the id is a blacklisted network builtin OR a non-builtin + * package (business deps must not be loaded/required for real at build — they may + * be missing platform binaries or open connections). Builtin "tool" modules + * (path/fs/module) stay real. At restore (`__RUNTIME_REQUIRE` installed) the hook + * is a no-op and the real module is required. */ export function injectExternalRequireLazyHook(content: string): ExternalRequireInjectionResult { // Idempotent: a re-run over already-injected output must not double-inject the @@ -425,7 +454,7 @@ export function injectExternalRequireLazyHook(content: string): ExternalRequireI let injected = 0; const next = content.replace(EXTERNAL_REQUIRE_SIGNATURE, (match, idParam: string, thunkParam: string) => { injected++; - return `${match} if (globalThis.__LAZY_EXT && globalThis.__LAZY_EXT.has(${idParam})) return globalThis.__makeLazyExt(${idParam}, ${thunkParam});`; + return `${match} if (globalThis.__makeLazyExt && !globalThis.__RUNTIME_REQUIRE && (globalThis.__LAZY_EXT.has(${idParam}) || !globalThis.__isBuiltin(${idParam}))) return globalThis.__makeLazyExt(${idParam}, ${thunkParam});`; }); return { content: next, injected }; } diff --git a/tools/egg-bundler/test/PackRunner.test.ts b/tools/egg-bundler/test/PackRunner.test.ts index 3e11a800a8..1c73dbcaa4 100644 --- a/tools/egg-bundler/test/PackRunner.test.ts +++ b/tools/egg-bundler/test/PackRunner.test.ts @@ -109,7 +109,7 @@ describe('PackRunner', () => { await expect(fs.stat(deepOut)).resolves.toBeTruthy(); }); - it('defaults to single-file export output with a per-entry library, target node 22, platform node, and UMD-form externals through to buildFunc', async () => { + it('defaults to single-file export output with a per-entry library, target node 22, platform node, and commonjs-type externals through to buildFunc', async () => { const buildFunc = vi.fn(async () => {}); const entries: PackEntry[] = [ { name: 'worker', filepath: '/abs/worker.entry.ts' }, @@ -132,10 +132,13 @@ describe('PackRunner', () => { expect(config.target).toBe('node 22'); expect(config.platform).toBe('node'); expect(config.output).toEqual({ path: path.join(tmpDir, 'out'), type: 'export' }); - // Externals must be UMD-form ({ commonjs, root }) so @utoo/pack output emits - // `require(name)` for the CJS runtime (not globalThis[name]). + // Single-file: externals are ExternalType `commonjs` ({ root, type }) so + // @utoo/pack emits a direct require(name) (surfaced as externalRequire, which + // the snapshot lazy hook intercepts). UMD form would fall through to + // globalThis[name] = undefined inside the single-file IIFE (no CommonJS + // module/exports in scope there). expect(config.externals).toEqual({ - '@eggjs/core': { commonjs: '@eggjs/core', root: '@eggjs/core' }, + '@eggjs/core': { root: '@eggjs/core', type: 'commonjs' }, }); expect(config.resolve).toBeUndefined(); expect(projectPath).toBe(tmpDir); diff --git a/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap b/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap index 50d0e5cbfb..32354b4ef7 100644 --- a/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap +++ b/tools/egg-bundler/test/__snapshots__/EntryGenerator.worker.canonical.snap @@ -100,6 +100,47 @@ for (const [appAbsRequest, targetRel] of __APP_RESOLVE_CACHE_ALIASES) { } } +// ── decorator file-path correction ────────────────────────────────────────── +// tegg decorators (@SingletonProto/@HTTPController/@Advice/…) capture a class's +// source path from the CALL STACK at module evaluation, via +// StackUtil.getCalleeFromStack(depth) with a hardcoded frame index. In a bundle +// every user frame collapses onto worker.js plus turbopack module wrappers, so a +// decorator that uses a deeper index than the norm — notably @Advice's depth-5 vs +// @SingletonProto's depth-4 — captures the runtime frame ("…/worker.js") instead +// of its own source file. tegg then cannot match that proto to its load unit and +// fails at restore with "Aop Advice(X) not found in loadUnits". +// +// The bundle DOES know each decorated file's real path (the manifest's tegg +// decoratedFiles, surfaced here as __DECORATED_FILE_KEYS). Re-stamp the correct +// path (outputDir + relKey, matching the format the well-behaved decorators get) +// on every decorated export, overriding whatever the decorator captured. This runs +// at build, so the corrected paths are baked into the snapshot the restore reads. +const __DECORATED_FILE_KEYS: string[] = ["node_modules/@eggjs/fake-module/app/service/UserService.ts"]; +{ + const __FILE_PATH_META = Symbol.for('EggPrototype.filePath'); + const __reflect = Reflect as unknown as { + hasOwnMetadata?: (key: symbol, target: unknown) => boolean; + defineMetadata?: (key: symbol, value: unknown, target: unknown) => void; + }; + if (typeof __reflect.hasOwnMetadata === 'function' && typeof __reflect.defineMetadata === 'function') { + for (const __rel of __DECORATED_FILE_KEYS) { + const __mod = __getBundleMap(__rel) as Record | undefined; + if (!__mod || (typeof __mod !== 'object' && typeof __mod !== 'function')) continue; + const __abs = path.resolve(__outputDir, __rel); + for (const __k of Object.keys(__mod)) { + const __exported = __mod[__k]; + if ( + __exported && + (typeof __exported === 'function' || typeof __exported === 'object') && + __reflect.hasOwnMetadata(__FILE_PATH_META, __exported) + ) { + __reflect.defineMetadata(__FILE_PATH_META, __abs, __exported); + } + } + } + } +} + // Tegg module reference / descriptor paths are stored relative to baseDir in the // manifest. Resolve them to absolute paths under the runtime output dir so every // tegg loader consumer (ModuleConfigUtil, EggAppLoader, LoaderFactory.loadApp @@ -145,6 +186,44 @@ if (process.env.EGG_BUNDLE_SNAPSHOT === 'build') { // no servers/timers/connections), run the snapshotWillSerialize hooks to release // non-serializable resources, then register the deserialize main function that // V8 invokes when restoring from the blob. + // + // Install the egg loader module importer for the BUILD phase too: the loader + // loads config/app files, and under --build-snapshot Node dynamic import() is + // unavailable, so route through the bundle map / require() instead. Do NOT set + // __RUNTIME_REQUIRE here — that would make externals load for real and defeat + // the lazy snapshot stubs; __EGG_MODULE_IMPORTER__ only feeds app/config files. + { + // getBuiltinModule needs Node >= 22.3; fall back to an eval'd require for + // 22.0–22.2 (same opaque-specifier trick the restore branch uses below). + const { createRequire: __cr } = + typeof process.getBuiltinModule === 'function' + ? process.getBuiltinModule('node:module') + : (0, eval)('require')('node:module'); + const __buildReq = __cr(__outputDir + '/'); + globalThis.__EGG_MODULE_IMPORTER__ = async (fp: string) => { + const bundled = __getBundleMap(fp); + if (bundled !== undefined) return bundled; + // Only skip when fp ITSELF cannot be resolved (a manifest-miss config unit the + // loader tolerates). Resolving first separates that from a nested MODULE_NOT_FOUND + // raised while loading a resolved file, which is a genuine missing dependency that + // must surface rather than be silently swallowed. + let resolved: string; + try { + resolved = __buildReq.resolve(fp); + } catch { + return undefined; + } + try { + return __buildReq(resolved); + } catch (err) { + // A resolved ESM file cannot be require()'d under --build-snapshot (no ESM + // loader); skip it. Any other error — including a nested MODULE_NOT_FOUND from + // a real missing dependency — is genuine and propagates. + if ((err as { code?: string } | undefined)?.code === 'ERR_INTERNAL_ASSERTION') return undefined; + throw err; + } + }; + } startEgg({ ...__startOptions, snapshot: true }).then(async (app) => { if (app.agent) { await app.agent.triggerSnapshotWillSerialize(); @@ -156,6 +235,18 @@ if (process.env.EGG_BUNDLE_SNAPSHOT === 'build') { // V8 runs this callback synchronously right after deserialization, before // the Node ESM loader is ready, so all real work is deferred to the next // tick via setImmediate. + // + // Restoring a V8 snapshot requires Node.js >= 24: Node.js 22 aborts while + // deserializing a non-trivial egg heap (V8 bug). The supported launcher + // (egg-scripts start --snapshot-blob) already gates this before spawning, + // so this is a defense-in-depth guard for a direct `node --snapshot-blob` + // launch that managed to deserialize on an unsupported runtime. + const __nodeMajor = parseInt(process.versions.node, 10); + if (__nodeMajor < 24) { + // eslint-disable-next-line no-console + console.error('[egg-bundler] V8 snapshot restore requires Node.js >= 24, but this process is ' + process.version + '. Build works on Node.js >= 22; restore must run on Node.js >= 24.'); + process.exit(1); + } setImmediate(() => { // A restored snapshot process has no dynamic import() callback // (ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING). Route every egg loader import diff --git a/tools/egg-bundler/test/snapshot-lazy-bundler.test.ts b/tools/egg-bundler/test/snapshot-lazy-bundler.test.ts index 7bf4c7ce1d..74ec0f293b 100644 --- a/tools/egg-bundler/test/snapshot-lazy-bundler.test.ts +++ b/tools/egg-bundler/test/snapshot-lazy-bundler.test.ts @@ -114,7 +114,7 @@ describe('Bundler snapshot lazy-external wiring', () => { const worker = await fs.readFile(path.join(tmpOutput, 'worker.js'), 'utf8'); expect(worker).toContain(SNAPSHOT_PRELUDE_MARKER); expect(worker).toContain( - 'if (globalThis.__LAZY_EXT && globalThis.__LAZY_EXT.has(id)) return globalThis.__makeLazyExt(id, thunk);', + 'if (globalThis.__makeLazyExt && !globalThis.__RUNTIME_REQUIRE && (globalThis.__LAZY_EXT.has(id) || !globalThis.__isBuiltin(id))) return globalThis.__makeLazyExt(id, thunk);', ); // prelude (with __LAZY_EXT) precedes the bundle IIFE — fail closed if either marker is absent const lazyExtIndex = worker.indexOf('globalThis.__LAZY_EXT = new Set('); @@ -132,7 +132,7 @@ describe('Bundler snapshot lazy-external wiring', () => { '"use strict";', '// marker: @eggjs/egg-bundler:snapshot-prelude', 'function externalRequire(id, thunk, esm = false) {', - ' if (globalThis.__LAZY_EXT && globalThis.__LAZY_EXT.has(id)) return globalThis.__makeLazyExt(id, thunk);', + ' if (globalThis.__makeLazyExt && !globalThis.__RUNTIME_REQUIRE && (globalThis.__LAZY_EXT.has(id) || !globalThis.__isBuiltin(id))) return globalThis.__makeLazyExt(id, thunk);', ' return thunk();', '}', '((__UTOOPACK__)=>{})([]);', diff --git a/tools/egg-bundler/test/snapshot-lazy-external.test.ts b/tools/egg-bundler/test/snapshot-lazy-external.test.ts index 2acd067ab1..700a027d18 100644 --- a/tools/egg-bundler/test/snapshot-lazy-external.test.ts +++ b/tools/egg-bundler/test/snapshot-lazy-external.test.ts @@ -74,15 +74,29 @@ describe('snapshot lazy-external', () => { }); describe('renderSnapshotPrelude (lazy body)', () => { - it('deletes Node lazy web globals with delete (not defineProperty)', () => { + it('neutralizes web globals by delete-then-stub (delete first, then a stub class)', () => { + // A plain Object.defineProperty over Node's lazy web globals (e.g. Headers) + // makes Node eagerly load undici → http2 native. So we delete first (removes + // the lazy getter without triggering it), THEN define a stub class (now a + // plain data property, constructable for `class extends globalThis.Request`). const prelude = renderSnapshotPrelude(); - expect(prelude).toContain('delete globalThis['); - expect(prelude).not.toContain('Object.defineProperty'); + expect(prelude).toContain('delete globalThis[__WEB_GLOBALS[__i]]'); + expect(prelude).toContain('Object.defineProperty(globalThis, __WEB_GLOBALS[__k]'); + // delete must come before the stub-define so no lazy getter remains to trigger. + expect(prelude.indexOf('delete globalThis[__WEB_GLOBALS[__i]]')).toBeLessThan( + prelude.indexOf('Object.defineProperty(globalThis, __WEB_GLOBALS[__k]'), + ); for (const g of ['fetch', 'Headers', 'Request', 'Response', 'FormData', 'WebSocket', 'File', 'Blob']) { expect(prelude).toContain(JSON.stringify(g)); } }); + it('stubs node:buffer File/Blob (which would otherwise pull in undici)', () => { + const prelude = renderSnapshotPrelude(); + expect(prelude).toContain("process.getBuiltinModule('node:buffer')"); + expect(prelude).toContain('["File","Blob"]'); + }); + it('installs __LAZY_EXT with every lazy id and the __makeLazyExt factory', () => { const prelude = renderSnapshotPrelude(['http', 'node:http', 'custom-pkg']); expect(prelude).toContain('globalThis.__LAZY_EXT = new Set('); @@ -91,13 +105,20 @@ describe('snapshot lazy-external', () => { expect(prelude).toContain('globalThis.__RUNTIME_REQUIRE'); }); - it('hardcodes http METHODS / STATUS_CODES / maxHeaderSize', () => { + it('inlines http METHODS / STATUS_CODES / maxHeaderSize read from the build Node', () => { const prelude = renderSnapshotPrelude(); + expect(prelude).toContain('globalThis.__HTTP_CONSTS'); expect(prelude).toContain('"GET"'); expect(prelude).toContain('"POST"'); expect(prelude).toContain('"200"'); expect(prelude).toContain('16384'); }); + + it('injects __EXTERNAL_EXPORTS so the member proxy can build a full ESM namespace', () => { + const prelude = renderSnapshotPrelude(['urllib'], { urllib: ['HttpClient', 'request'] }); + expect(prelude).toContain('globalThis.__EXTERNAL_EXPORTS'); + expect(prelude).toContain('"HttpClient"'); + }); }); describe('injectExternalRequireLazyHook', () => { @@ -106,7 +127,7 @@ describe('snapshot lazy-external', () => { const { content, injected } = injectExternalRequireLazyHook(src); expect(injected).toBe(1); expect(content).toContain( - 'if (globalThis.__LAZY_EXT && globalThis.__LAZY_EXT.has(id)) return globalThis.__makeLazyExt(id, thunk);', + 'if (globalThis.__makeLazyExt && !globalThis.__RUNTIME_REQUIRE && (globalThis.__LAZY_EXT.has(id) || !globalThis.__isBuiltin(id))) return globalThis.__makeLazyExt(id, thunk);', ); expect(content.indexOf('__makeLazyExt')).toBeLessThan(content.indexOf('return thunk()')); }); @@ -115,7 +136,9 @@ describe('snapshot lazy-external', () => { const src = 'function externalRequire(a,b,c=false){return b();}'; const { content, injected } = injectExternalRequireLazyHook(src); expect(injected).toBe(1); - expect(content).toContain('globalThis.__LAZY_EXT.has(a)) return globalThis.__makeLazyExt(a, b);'); + expect(content).toContain( + '(globalThis.__LAZY_EXT.has(a) || !globalThis.__isBuiltin(a))) return globalThis.__makeLazyExt(a, b);', + ); }); it('injects into every externalRequire occurrence', () => { diff --git a/tools/egg-bundler/test/snapshot-lazy.realbuild.test.ts b/tools/egg-bundler/test/snapshot-lazy.realbuild.test.ts index acccd647d0..54db919d55 100644 --- a/tools/egg-bundler/test/snapshot-lazy.realbuild.test.ts +++ b/tools/egg-bundler/test/snapshot-lazy.realbuild.test.ts @@ -124,7 +124,9 @@ describe('snapshot lazy-external — real @utoo/pack build', () => { expect(worker).toContain(SNAPSHOT_PRELUDE_MARKER); // @utoo/pack really emitted externalRequire and the hook landed inside it. expect(worker).toMatch(/function\s+externalRequire\s*\(/); - expect(worker).toMatch(/globalThis\.__LAZY_EXT\.has\([A-Za-z_$][\w$]*\)\) return globalThis\.__makeLazyExt\(/); + expect(worker).toMatch( + /globalThis\.__LAZY_EXT\.has\([A-Za-z_$][\w$]*\) \|\| !globalThis\.__isBuiltin\([A-Za-z_$][\w$]*\)\)\) return globalThis\.__makeLazyExt\(/, + ); // The injected + prepended source is still valid JS. await execFileAsync(process.execPath, ['--check', workerPath]); @@ -132,9 +134,9 @@ describe('snapshot lazy-external — real @utoo/pack build', () => { // BUILD context: run worker.js directly. http must be the stub. const built = await execFileAsync(process.execPath, [workerPath], { cwd: outputDir }); expect(JSON.parse(built.stdout)).toEqual({ - methodsHasGet: true, // hardcoded METHODS, real http never loaded - maxHeaderSize: 16384, // hardcoded constant - createServerCall: 'undefined', // apply trap no-ops at build time + methodsHasGet: true, // __HTTP_CONSTS.METHODS (read from build Node), real http never loaded + maxHeaderSize: 16384, // __HTTP_CONSTS.maxHeaderSize, real http never loaded + createServerCall: 'object', // build: a call-result member-proxy uses an object target (typeof 'object', mirroring the real instance); still chainable for x.y(z).w restored: false, }); diff --git a/tools/scripts/src/commands/start.ts b/tools/scripts/src/commands/start.ts index 493b925340..a4aa7fbb0e 100644 --- a/tools/scripts/src/commands/start.ts +++ b/tools/scripts/src/commands/start.ts @@ -85,6 +85,10 @@ export default class Start extends BaseCommand { sourcemap: Flags.boolean({ summary: 'whether enable sourcemap support, will load `source-map-support` etc', aliases: ['ts', 'typescript'], + // Allow the `--no-sourcemap` negation so callers can opt out of the + // source-map-support `--import` that `egg.typescript` auto-enables (e.g. a + // snapshot restore, where `--import` must not ride into `node --snapshot-blob`). + allowNo: true, }), }; @@ -114,6 +118,27 @@ export default class Start extends BaseCommand { return path.join(import.meta.dirname, '../../scripts', serverBinName); } + /** + * Resolve the major version of the node binary that will actually run the + * snapshot. For the default (`--node` unset → 'node') or the egg-scripts + * runtime itself, the spawned process shares this runtime's version; for an + * explicit custom `--node /path`, query that binary directly. Returns + * `undefined` when the version cannot be determined, so the gate fails open + * (proceeds) rather than blocking a launch whose version it cannot read. + */ + async #resolveSnapshotNodeMajor(command: string): Promise { + if (command === 'node' || command === process.execPath) { + return parseInt(process.versions.node, 10); + } + try { + const { stdout } = await execFile(command, ['--version']); + const match = /^v?(\d+)\./.exec(stdout.toString().trim()); + return match ? Number(match[1]) : undefined; + } catch { + return undefined; + } + } + public async run(): Promise { const { args, flags } = this; // context.execArgvObj = context.execArgvObj || {}; @@ -248,6 +273,24 @@ export default class Start extends BaseCommand { let eggArgs: string[]; let displayName: string; if (flags['snapshot-blob']) { + // Restoring a V8 startup snapshot requires Node.js >= 24. A snapshot can be + // *built* on Node.js >= 22, but restoring a non-trivial egg heap on Node.js + // 22 aborts the process during deserialization (V8 bug: + // `Check failed: current == end_slot_index`). Refuse early with a clear + // message here instead of letting the spawned `node --snapshot-blob` child + // die with a cryptic native fatal error. Check the version of the node binary + // that will actually run the snapshot (`command`/`--node`), not just the + // egg-scripts runtime, so a custom `--node` is gated against the real target. + const nodeMajor = await this.#resolveSnapshotNodeMajor(command); + if (nodeMajor !== undefined && nodeMajor < 24) { + this.error( + `egg-scripts start --snapshot-blob requires Node.js >= 24 to restore a V8 snapshot, ` + + `but ${command} is Node.js ${nodeMajor}.x. ` + + `Building a snapshot (egg-bin snapshot build) works on Node.js >= 22, ` + + `but restoring it must run on Node.js >= 24. Please upgrade Node.js to 24 or later.`, + { exit: 1 }, + ); + } // Snapshot boot: a single self-contained `node --snapshot-blob ` // process (no egg-cluster, no framework resolution). The snapshot entry // reads the listen port from PORT env, and `--title` is appended only so diff --git a/tools/scripts/test/snapshot-start.test.ts b/tools/scripts/test/snapshot-start.test.ts index eb25bd2d2b..e434418f0f 100644 --- a/tools/scripts/test/snapshot-start.test.ts +++ b/tools/scripts/test/snapshot-start.test.ts @@ -7,17 +7,43 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import Start from '../src/commands/start.ts'; const spawnMock = vi.hoisted(() => vi.fn()); +// Callback-style execFile mock (the gate promisifies it). A custom `--node` path +// is version-checked through it, so we can simulate any reported version. +const execFileMock = vi.hoisted(() => vi.fn()); vi.mock('node:child_process', async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, spawn: spawnMock }; + return { ...actual, spawn: spawnMock, execFile: execFileMock }; }); const __dirname = import.meta.dirname; +// Override process.versions.node so the snapshot Node >= 24 gate is exercised +// deterministically regardless of the host Node version (egg-scripts CI runs on +// both Node 22 and 24). Returns a restorer. +function pinNodeVersion(version: string): () => void { + // Redefine `process.versions` itself rather than its `node` property: in some + // runtimes `process.versions.node` is non-configurable and a direct + // defineProperty on it would throw. + const originalVersions = process.versions; + Object.defineProperty(process, 'versions', { + value: { ...originalVersions, node: version }, + configurable: true, + writable: true, + }); + return () => { + Object.defineProperty(process, 'versions', { + value: originalVersions, + configurable: true, + writable: true, + }); + }; +} + describe('test/snapshot-start.test.ts', () => { const baseDir = path.join(__dirname, 'fixtures/example'); let homeDir: string; + let restoreNodeVersion: (() => void) | undefined; beforeEach(async () => { // Mock HOME (node-homedir honors MOCK_HOME_DIR) so the shared start pipeline's @@ -33,9 +59,20 @@ describe('test/snapshot-start.test.ts', () => { kill: vi.fn(), pid: 4242, })); + // Default execFile to an empty success; individual tests override it to + // simulate a specific `--node --version` output (or a spawn failure). + execFileMock.mockReset(); + execFileMock.mockImplementation((_cmd: string, _args: string[], cb: (e: unknown, r?: { stdout: string }) => void) => + cb(null, { stdout: '' }), + ); + // Snapshot restore requires Node >= 24; pin a supported version so the + // argv-construction tests below do not trip the gate on a Node 22 host. + restoreNodeVersion = pinNodeVersion('24.18.0'); }); afterEach(async () => { + restoreNodeVersion?.(); + restoreNodeVersion = undefined; for (const signal of ['SIGINT', 'SIGQUIT', 'SIGTERM']) { process.removeAllListeners(signal); } @@ -71,4 +108,47 @@ describe('test/snapshot-start.test.ts', () => { expect(args).toContain('/abs/app.blob'); expect(options.env.PORT).toBe('8080'); }); + + it('accepts the --no-sourcemap negation (sourcemap flag has allowNo)', async () => { + // Regression guard: the cnpmcore-snapshot e2e job uses --no-sourcemap to keep + // `--import source-map-support` out of the `node --snapshot-blob` process. + // Without allowNo:true on the sourcemap flag, oclif rejects --no-sourcemap with + // NonExistentFlagsError and the start command never spawns. + await Start.run(['--snapshot-blob', './snapshot.blob', '--no-sourcemap', baseDir]); + const { args } = spawnArgs(); + expect(args).not.toContain('--import'); + }); + + it('refuses to restore on Node.js < 24 with a clear error', async () => { + // Override the supported-version pin from beforeEach. We can discard this + // restorer because afterEach's `restoreNodeVersion` (captured in beforeEach + // before any pin) restores the original host descriptor regardless. + pinNodeVersion('22.22.3'); + await expect(Start.run(['--snapshot-blob', './snapshot.blob', baseDir])).rejects.toThrow(/Node\.js >= 24/); + // Gated before spawning the doomed `node --snapshot-blob` child. + expect(spawnMock).not.toHaveBeenCalled(); + }); + + it('queries a custom --node binary and refuses an unsupported version', async () => { + // A custom `--node /path` is not this runtime, so the gate resolves its version + // by running ` --version` instead of reading process.versions. + execFileMock.mockImplementation((_cmd: string, _args: string[], cb: (e: unknown, r?: { stdout: string }) => void) => + cb(null, { stdout: 'v22.4.1\n' }), + ); + await expect( + Start.run(['--snapshot-blob', './snapshot.blob', '--node', '/opt/node22/bin/node', baseDir]), + ).rejects.toThrow(/Node\.js >= 24/); + expect(execFileMock).toHaveBeenCalledWith('/opt/node22/bin/node', ['--version'], expect.any(Function)); + expect(spawnMock).not.toHaveBeenCalled(); + }); + + it('proceeds (fails open) when a custom --node version cannot be determined', async () => { + // If ` --version` cannot run, the gate returns undefined and the launch + // proceeds rather than blocking on a version it could not read. + execFileMock.mockImplementation((_cmd: string, _args: string[], cb: (e: unknown) => void) => + cb(new Error('spawn ENOENT')), + ); + await Start.run(['--snapshot-blob', './snapshot.blob', '--node', '/no/such/node', baseDir]); + expect(spawnMock).toHaveBeenCalledTimes(1); + }); });