Skip to content
Open
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,7 @@ function parseArgs(argv: string[]): CliArgs {
"webkit",
"buildDir",
"cacheDir",
"downloadCacheDir",
"nodejsVersion",
"nodejsAbiVersion",
"zigCommit",
Expand Down
2 changes: 1 addition & 1 deletion scripts/build/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ Why not auto-register in emit functions? Some rules are shared (`dep_configure`

**cmd.exe quoting is partial.** `shell.ts` quote() handles spaces/special chars but NOT `%VAR%` expansion, `^` escape, `&|>` redirection. If an arg contains those, switch to powershell.

**`rm -rf build/` doesn't clear the cache locally.** `cfg.cacheDir` is machine-shared at `$BUN_INSTALL/build-cache` for non-CI builds (ccache, zig, tarballs, prebuilt WebKit). Everything there is content-addressed or version-stamped, so a stale entry can't be hit — don't reach for `bun run clean cache` as a debugging step. If a build misbehaves, the bug is in the inputs or the graph, not the cache; nuking it just costs you a cold rebuild. CI keeps `<buildDir>/cache` so `rm -rf build/` is still a full reset there.
**`rm -rf build/` doesn't clear the cache locally.** `cfg.cacheDir` is machine-shared at `$BUN_INSTALL/build-cache` for non-CI builds (ccache, zig-cache); `cfg.downloadCacheDir` defaults to the same place and holds dep tarballs / prebuilt WebKit / nodejs-headers. Everything there is content-addressed or version-stamped, so a stale entry can't be hit — don't reach for `bun run clean cache` as a debugging step. If a build misbehaves, the bug is in the inputs or the graph, not the cache; nuking it just costs you a cold rebuild. CI keeps `cacheDir` at `<buildDir>/cache` (so `rm -rf build/` is a full ccache reset there); CI agents that set `BUN_DEPS_CACHE_PATH` redirect only `downloadCacheDir` outside the build tree so fetches survive across jobs.

## Node compatibility

Expand Down
18 changes: 17 additions & 1 deletion scripts/build/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,14 @@ export interface Config {
buildDir: string;
/** Generated code output, e.g. buildDir/codegen/. */
codegenDir: string;
/** Persistent cache for dep tarballs and builds. */
/** ccache/zig-cache. Local: shared across checkouts. CI: per-build (ephemeral). */
cacheDir: string;
/**
* Downloaded artifacts (dep tarballs, prebuilt webkit, nodejs-headers).
* Defaults to cacheDir; CI agents that want fetches to survive across
* ephemeral runners point this elsewhere via BUN_DEPS_CACHE_PATH.
*/
downloadCacheDir: string;
/** Vendored dependencies (gitignored). */
vendorDir: string;

Expand Down Expand Up @@ -231,6 +237,7 @@ export interface PartialConfig {
webkit?: WebKitMode;
buildDir?: string;
cacheDir?: string;
downloadCacheDir?: string;
// Version pins (defaults in versions.ts).
nodejsVersion?: string;
nodejsAbiVersion?: string;
Expand Down Expand Up @@ -451,6 +458,14 @@ export function resolveConfig(partial: PartialConfig, toolchain: Toolchain): Con
: ci
? resolve(buildDir, "cache")
: resolve(bunInstall, "build-cache");
// Downloaded artifacts only — content-addressed/version-stamped, so safe
// to share across builds. Anchored to repo root for the same regen-rule
// reason as bunInstall above.
const downloadCacheDir = partial.downloadCacheDir
? resolve(cwd, partial.downloadCacheDir)
: process.env.BUN_DEPS_CACHE_PATH
? resolve(cwd, process.env.BUN_DEPS_CACHE_PATH)
: cacheDir;
const vendorDir = resolve(cwd, "vendor");

// ─── Validation ───
Expand Down Expand Up @@ -524,6 +539,7 @@ export function resolveConfig(partial: PartialConfig, toolchain: Toolchain): Con
buildDir,
codegenDir,
cacheDir,
downloadCacheDir,
vendorDir,
cc: toolchain.cc,
cxx: toolchain.cxx,
Expand Down
2 changes: 1 addition & 1 deletion scripts/build/deps/nodejs-headers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export const nodejsHeaders: Dependency = {
// Delete headers that conflict with BoringSSL / our libuv.
// Tarball top-level is `node-v<version>/` (hoisted), inside is `include/node/`.
rmAfterExtract: ["include/node/openssl", "include/node/uv", "include/node/uv.h"],
destDir: resolve(cfg.cacheDir, `nodejs-headers-${cfg.nodejsVersion}`),
destDir: resolve(cfg.downloadCacheDir, `nodejs-headers-${cfg.nodejsVersion}`),
}),

build: () => ({ kind: "none" }),
Expand Down
2 changes: 1 addition & 1 deletion scripts/build/deps/webkit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ function prebuiltUrl(cfg: Config): string {
*/
function prebuiltDestDir(cfg: Config): string {
const version16 = cfg.webkitVersion.slice(0, 16);
return resolve(cfg.cacheDir, `webkit-${version16}${prebuiltSuffix(cfg)}`);
return resolve(cfg.downloadCacheDir, `webkit-${version16}${prebuiltSuffix(cfg)}`);
}

// ───────────────────────────────────────────────────────────────────────────
Expand Down
4 changes: 4 additions & 0 deletions scripts/build/flags.ts
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,10 @@ export const globalFlags: Flag[] = [
`-ffile-prefix-map=${c.cwd}=.`,
`-ffile-prefix-map=${c.vendorDir}=vendor`,
`-ffile-prefix-map=${c.cacheDir}=cache`,
// WebKit/nodejs-headers live under downloadCacheDir; when that diverges
// from cacheDir (BUN_DEPS_CACHE_PATH set) the cacheDir map doesn't cover
// them and their absolute paths would leak into DWARF.
...(c.downloadCacheDir !== c.cacheDir ? [`-ffile-prefix-map=${c.downloadCacheDir}=dlcache`] : []),
],
when: c => c.unix && c.ci,
desc: "Remap source paths in debug info (reproducible builds)",
Expand Down
6 changes: 3 additions & 3 deletions scripts/build/source.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* `buildDir/deps/<name>/`. This supports "local" dep mode where the user edits
* vendored source directly — the fetch step is skipped and no .ref is written.
*
* Tarballs are cached in `cacheDir/tarballs/<identity-hash>.tar.gz` so
* Tarballs are cached in `downloadCacheDir/tarballs/<identity-hash>.tar.gz` so
* re-extraction after a failed patch doesn't re-download.
*/

Expand Down Expand Up @@ -152,7 +152,7 @@ export type Source =
rmAfterExtract?: string[];
/**
* Where extracted files land. Default: `vendor/<name>/`. Prebuilt deps
* (WebKit, nodejs-headers) override to `cacheDir/<name>-<version>/`.
* (WebKit, nodejs-headers) override to `downloadCacheDir/<name>-<version>/`.
*/
destDir?: string;
};
Expand Down Expand Up @@ -868,7 +868,7 @@ function emitFetch(
repo: source.repo,
commit: source.commit,
dest: srcDir,
cache: resolve(cfg.cacheDir, "tarballs"),
cache: resolve(cfg.downloadCacheDir, "tarballs"),
// Pass patches space-separated. Shell-safe because patch paths are
// under our control (no spaces in repo paths per convention).
patches: patchPaths.join(" "),
Expand Down
64 changes: 3 additions & 61 deletions scripts/machine.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ import {
sha256,
spawn,
spawnSafe,
spawnScp,
spawnSsh,
spawnSshSafe,
spawnSyncSafe,
startGroup,
waitForPort,
which,
writeFile,
} from "./utils.mjs";
Comment thread
alii marked this conversation as resolved.
Expand Down Expand Up @@ -937,64 +937,6 @@ async function getGithubOrgSshKeys(organization) {
* @property {number} [retries]
*/

/**
* @typedef ScpOptions
* @property {string} hostname
* @property {string} source
* @property {string} destination
* @property {string[]} [identityPaths]
* @property {string} [port]
* @property {string} [username]
* @property {number} [retries]
*/

/**
* @param {ScpOptions} options
* @returns {Promise<void>}
*/
async function spawnScp(options) {
const { hostname, port, username, identityPaths, password, source, destination, retries = 3 } = options;
await waitForPort({ hostname, port: port || 22 });

const command = ["scp", "-o", "StrictHostKeyChecking=no"];
command.push("-O"); // use SCP instead of SFTP
if (!password) {
command.push("-o", "BatchMode=yes");
}
if (port) {
command.push("-P", port);
}
if (password) {
const sshPass = which("sshpass", { required: true });
command.unshift(sshPass, "-p", password);
} else if (identityPaths) {
command.push(...identityPaths.flatMap(path => ["-i", path]));
}
command.push(resolve(source));
if (username) {
command.push(`${username}@${hostname}:${destination}`);
} else {
command.push(`${hostname}:${destination}`);
}

let cause;
for (let i = 0; i < retries; i++) {
const result = await spawn(command, { stdio: "inherit" });
const { exitCode, stderr } = result;
if (exitCode === 0) {
return;
}

cause = stderr.trim() || undefined;
if (/(bad configuration option)|(no such file or directory)/i.test(stderr)) {
break;
}
await new Promise(resolve => setTimeout(resolve, Math.pow(2, i) * 1000));
}

throw new Error(`SCP failed: ${source} -> ${username}@${hostname}:${destination}`, { cause });
}

/**
* @param {string} passwordData
* @param {string} privateKeyPath
Expand Down Expand Up @@ -1098,7 +1040,7 @@ function getCloud(name) {
* @property {(source: string, destination: string) => Promise<void>} upload
* @property {() => Promise<RdpCredentials>} [rdp]
* @property {() => Promise<void>} attach
* @property {() => Promise<string>} snapshot
* @property {(label?: string) => Promise<string>} snapshot
Comment thread
robobun marked this conversation as resolved.
* @property {() => Promise<void>} close
*/

Expand Down Expand Up @@ -1484,7 +1426,7 @@ async function main() {
}

try {
if (options.rdp) {
if (options.rdp && typeof machine.rdp === "function") {
await startGroup("Connecting with RDP...", async () => {
const { hostname, username, password } = await machine.rdp();

Expand Down
40 changes: 30 additions & 10 deletions scripts/tart.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { inspect } from "node:util";
import { isPrivileged, spawnSafe, which } from "./utils.mjs";
import { isPrivileged, spawnSafe, spawnScp, spawnSsh, spawnSshSafe, which } from "./utils.mjs";

/**
* @link https://tart.run/
Expand Down Expand Up @@ -33,8 +33,8 @@ export const tart = {
},

/**
* @typedef {"sequoia" | "sonoma" | "ventura" | "monterey"} TartDistro
* @typedef {`ghcr.io/cirruslabs/macos-${TartDistro}-xcode`} TartImage
* @typedef {"tahoe" | "sequoia" | "sonoma" | "ventura" | "monterey"} TartDistro
* @typedef {`ghcr.io/cirruslabs/macos-${TartDistro}-base`} TartImage
* @link https://github.com/orgs/cirruslabs/packages?repo_name=macos-image-templates
*/

Expand All @@ -48,16 +48,17 @@ export const tart = {
throw new Error(`Unsupported platform: ${inspect(platform)}`);
}
const distros = {
"26": "tahoe",
"15": "sequoia",
"14": "sonoma",
"13": "ventura",
"12": "monterey",
};
const distro = distros[release];
if (!distro) {
throw new Error(`Unsupported macOS release: ${distro}`);
throw new Error(`Unsupported macOS release: ${release}`);
}
return `ghcr.io/cirruslabs/macos-${distro}-xcode`;
return `ghcr.io/cirruslabs/macos-${distro}-base`;
},

/**
Expand Down Expand Up @@ -87,6 +88,9 @@ export const tart = {
json: true,
throwOnError: error => !/does not exist/i.test(inspect(error)),
});
if (!result) {
return undefined;
}
return {
Name: name,
...result,
Comment thread
robobun marked this conversation as resolved.
Expand Down Expand Up @@ -184,8 +188,10 @@ export const tart = {
);
}

// This command is blocking, so it needs to be detached and not awaited
this.spawn(["run", name, ...args], { detached: true });
// `tart run` blocks for the VM's lifetime, so it's detached and not
// awaited. stopVm() makes it exit non-zero; without throwOnError:false
// that becomes an unhandled rejection and Node exits 1.
this.spawn(["run", name, ...args], { detached: true, throwOnError: false });
},

/**
Expand All @@ -206,6 +212,12 @@ export const tart = {
*/
async createMachine(options) {
const { name, imageName, cpuCount, memoryGb, diskSizeGb, rdp } = options;
// cirruslabs base images use password auth (admin/admin); spawnSsh shells
// out to sshpass for that, which isn't on stock macOS. Check before
// cloneVm/runVm so a missing dep doesn't orphan a running VM.
if (!which("sshpass")) {
throw new Error("tart machine ops need sshpass: brew install hudochenkov/sshpass/sshpass");
}

const image = imageName || this.getImage(options);
const machineId = name || `i-${Math.random().toString(36).slice(2, 11)}`;
Comment thread
claude[bot] marked this conversation as resolved.
Comment thread
alii marked this conversation as resolved.
Expand Down Expand Up @@ -260,9 +272,16 @@ export const tart = {
await spawnScp({ ...connectOptions, source, destination });
};

const rdp = async () => {
const connectOptions = await connect();
await spawnRdp({ ...connectOptions });
const snapshot = async label => {
if (!label) throw new Error("tart snapshot() requires a label");
// tart can't push a running VM — stop first, then push to ghcr. Auth via
// TART_REGISTRY_USERNAME / TART_REGISTRY_PASSWORD (set by the image-build
// pipeline; tart reads them directly, no `tart login` needed).
await this.stopVm(name);
const remote = `ghcr.io/oven-sh/${label}`;
console.log(`Pushing ${name} to ${remote} (~25GB, this takes a while)...`);
Comment thread
alii marked this conversation as resolved.
await this.spawn(["push", name, remote], { stdio: "inherit" });
return remote;
Comment thread
alii marked this conversation as resolved.
};

const close = async () => {
Comment thread
alii marked this conversation as resolved.
Expand All @@ -276,6 +295,7 @@ export const tart = {
spawnSafe: execSafe,
attach,
upload,
snapshot,
close,
[Symbol.asyncDispose]: close,
};
Comment thread
robobun marked this conversation as resolved.
Expand Down
59 changes: 59 additions & 0 deletions scripts/utils.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3146,6 +3146,65 @@ export async function spawnSsh(options, spawnOptions = {}) {
return result;
}

/**
* @typedef ScpOptions
* @property {string} hostname
* @property {string} source
* @property {string} destination
* @property {string[]} [identityPaths]
* @property {string} [port]
* @property {string} [username]
* @property {string} [password]
* @property {number} [retries]
*/

/**
* @param {ScpOptions} options
* @returns {Promise<void>}
*/
export async function spawnScp(options) {
const { hostname, port, username, identityPaths, password, source, destination, retries = 3 } = options;
await waitForPort({ hostname, port: port || 22 });

const command = ["scp", "-o", "StrictHostKeyChecking=no"];
command.push("-O"); // use SCP instead of SFTP
if (!password) {
command.push("-o", "BatchMode=yes");
}
if (port) {
command.push("-P", port);
}
if (password) {
const sshPass = which("sshpass", { required: true });
command.unshift(sshPass, "-p", password);
} else if (identityPaths) {
command.push(...identityPaths.flatMap(path => ["-i", path]));
}
command.push(resolve(source));
if (username) {
command.push(`${username}@${hostname}:${destination}`);
} else {
command.push(`${hostname}:${destination}`);
}

let cause;
for (let i = 0; i < retries; i++) {
const result = await spawn(command, { stdio: "inherit" });
const { exitCode, stderr } = result;
if (exitCode === 0) {
return;
}

cause = stderr.trim() || undefined;
if (/(bad configuration option)|(no such file or directory)/i.test(stderr)) {
break;
}
await new Promise(resolve => setTimeout(resolve, Math.pow(2, i) * 1000));
}

throw new Error(`SCP failed: ${source} -> ${username}@${hostname}:${destination}`, { cause });
Comment thread
alii marked this conversation as resolved.
}

Comment thread
alii marked this conversation as resolved.
/**
* @param {MachineOptions} options
* @returns {Promise<Machine>}
Expand Down
Loading