From 7ad6591dba0ba32ee9d20ae57e3032b06428a77a Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:25:25 +0000 Subject: [PATCH 1/4] ci: print the CPU model and cloud instance type in the job log's Machine header CI requests one instance type per lane (.buildkite/ci.mjs), but under capacity pressure a job can land on a different, slower type, and the job log had nothing that identified the machine class. The CPU line comes from os.cpus(); the instance type is read from the EC2 or Azure metadata service, selected by the agent's cloud tag, so off-cloud agents skip the request. --- scripts/utils.mjs | 67 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/scripts/utils.mjs b/scripts/utils.mjs index a1a204c09e7b..28cc5a5c49cf 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -15,7 +15,15 @@ import { writeFileSync, } from "node:fs"; import { connect } from "node:net"; -import { hostname, homedir as nodeHomedir, tmpdir as nodeTmpdir, release, userInfo } from "node:os"; +import { + availableParallelism, + cpus, + hostname, + homedir as nodeHomedir, + tmpdir as nodeTmpdir, + release, + userInfo, +} from "node:os"; import { basename, dirname, join, relative, resolve } from "node:path"; import { normalize as normalizeWindows } from "node:path/win32"; @@ -1870,6 +1878,61 @@ export function getPublicIp() { } } +/** + * The CPU model and the number of CPUs this process may run on, e.g. + * "Intel(R) Xeon(R) Platinum 8488C x8" or "Apple M2 Ultra x24". + * @returns {string | undefined} + */ +export function getCpuDescription() { + const [cpu] = cpus(); + const model = cpu?.model?.replace(/\s+/g, " ").trim(); + if (!model) { + return; + } + return `${model} x${availableParallelism()}`; +} + +/** + * The instance type the cloud provider actually launched for this machine, + * e.g. "r7i.2xlarge" on EC2 or "Standard_D8s_v5" on Azure. CI asks for one + * type per lane (.buildkite/ci.mjs), but under capacity pressure the + * machine may be a different, slower type, and nothing else in a job log + * records which one it was. + * @returns {string | undefined} + */ +export function getCloudInstanceType() { + // Set by scripts/agent.mjs on the cloud agents; absent on the darwin fleet + // and GitHub Actions, where there is no metadata service to ask. + const cloud = getEnv("BUILDKITE_AGENT_META_DATA_CLOUD", false); + let request; + if (cloud === "aws") { + request = ["http://169.254.169.254/latest/meta-data/instance-type"]; + } else if (cloud === "azure") { + request = [ + "-H", + "Metadata: true", + "http://169.254.169.254/metadata/instance/compute/vmSize?api-version=2021-02-01&format=text", + ]; + } else { + return; + } + + const { error, stdout } = spawnSync([ + "curl", + "-sf", + "--noproxy", + "*", + "--connect-timeout", + "1", + "--max-time", + "3", + ...request, + ]); + if (!error) { + return stdout.trim() || undefined; + } +} + /** * @returns {string} */ @@ -3095,8 +3158,10 @@ export function printEnvironment() { } console.log("Distro:", getDistro()); console.log("Distro Version:", getDistroVersion()); + console.log("CPU:", getCpuDescription()); console.log("Hostname:", getHostname()); if (isCI) { + console.log("Instance Type:", getCloudInstanceType()); console.log("Tailscale IP:", getTailscaleIp()); console.log("Public IP:", getPublicIp()); } From c3d88e9d9ccf7daf3b59623c4f46820a06d3b344 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:19:49 +0000 Subject: [PATCH 2/4] ci: say why the instance type lookup is keyed on the agent's cloud tag --- scripts/utils.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 28cc5a5c49cf..9b9f6fa72976 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -1901,8 +1901,9 @@ export function getCpuDescription() { * @returns {string | undefined} */ export function getCloudInstanceType() { - // Set by scripts/agent.mjs on the cloud agents; absent on the darwin fleet - // and GitHub Actions, where there is no metadata service to ask. + // The `cloud` tag scripts/agent.mjs puts on the agents we launch ourselves. + // Anything without it (the darwin fleet, GitHub Actions) is not a machine + // whose type we chose, so there is nothing to compare against and no request. const cloud = getEnv("BUILDKITE_AGENT_META_DATA_CLOUD", false); let request; if (cloud === "aws") { From 8f8ce349cdcfaa15b0a61180dbf6fa22781092ec Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:44:47 +0000 Subject: [PATCH 3/4] ci: name the instance type in test failure annotations too The failure annotations are what `bun run ci:errors` and the triage tooling read, so a timeout on fallback hardware now reads "on :debian: 13 x64-asan (r6i.2xlarge)" there without opening the job log. The metadata lookup is memoized since it now runs once per failure as well as once for the header. --- scripts/runner.node.mjs | 14 ++++++++++++-- scripts/utils.mjs | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/scripts/runner.node.mjs b/scripts/runner.node.mjs index 9ef779b2ad96..2fc852e14f1e 100755 --- a/scripts/runner.node.mjs +++ b/scripts/runner.node.mjs @@ -43,6 +43,7 @@ import { getBuildLabel, getBuildMetadata, getBuildUrl, + getCloudInstanceType, getCommit, getDistro, getDistroVersion, @@ -1131,7 +1132,7 @@ async function runTests() { context: "flaky", label: title, style: "warning", - content: `
${title} - ${reason} (in the parallel batch on ${getBuildLabel()}; passed alone)${detail}
`, + content: `
${title} - ${reason} (in the parallel batch on ${getTestLabel()}; passed alone)${detail}
`, }); } } @@ -2789,10 +2790,19 @@ function addPath(...paths) { } /** + * The lane a failure is reported against, plus the machine type the job + * actually ran on when that is known, e.g. ":debian: 13 x64-asan (r7i.2xlarge)". + * The type is what tells a timeout on fallback hardware apart from one on the + * hardware the lane asks for, so it goes in the annotation next to the lane. * @returns {string | undefined} */ function getTestLabel() { - return getBuildLabel()?.replace(" - test-bun", ""); + const label = getBuildLabel()?.replace(" - test-bun", ""); + if (!label) { + return; + } + const instanceType = getCloudInstanceType(); + return instanceType ? `${label} (${instanceType})` : label; } /** diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 9b9f6fa72976..40014994f6ec 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -1892,15 +1892,28 @@ export function getCpuDescription() { return `${model} x${availableParallelism()}`; } +/** @type {string | undefined} "" once looked up and not available */ +let cloudInstanceType; + /** * The instance type the cloud provider actually launched for this machine, * e.g. "r7i.2xlarge" on EC2 or "Standard_D8s_v5" on Azure. CI asks for one * type per lane (.buildkite/ci.mjs), but under capacity pressure the * machine may be a different, slower type, and nothing else in a job log - * records which one it was. + * records which one it was. Looked up once per process. * @returns {string | undefined} */ export function getCloudInstanceType() { + if (typeof cloudInstanceType !== "string") { + cloudInstanceType = fetchCloudInstanceType() || ""; + } + return cloudInstanceType || undefined; +} + +/** + * @returns {string | undefined} + */ +function fetchCloudInstanceType() { // The `cloud` tag scripts/agent.mjs puts on the agents we launch ourselves. // Anything without it (the darwin fleet, GitHub Actions) is not a machine // whose type we chose, so there is nothing to compare against and no request. From 931510e75e0e39007c14bcbf3ee4a93c120e5b7b Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 13 Aug 2026 20:28:24 +0000 Subject: [PATCH 4/4] ci: tag agents with the instance type they were launched as The launched type is a boot-time fact of the machine like kernel or abi-version, so agent.mjs records it as a launched-instance-type tag via the existing metadata helpers. That puts it in the Buildkite API for every job, and the runner reads it back from the tag's environment variable; images baked before the tag existed fall through to the direct metadata request. The key is not instance-type, which is the type the job asked for in its agent query. --- scripts/agent.mjs | 7 +++++++ scripts/utils.mjs | 44 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/scripts/agent.mjs b/scripts/agent.mjs index ca4e7bda731c..10cf71d866f9 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -13,6 +13,7 @@ import { getAwsSecret, getAzureSecret, getCloud, + getCloudLaunchedInstanceType, getCloudMetadataTag, getDistro, getDistroVersion, @@ -383,6 +384,12 @@ async function doBuildkiteAgent(action, cliOptions = {}) { tags[tag] = value; } } + // Deliberately not `instance-type`: that key is the type the job asked + // for (.buildkite/ci.mjs), and under capacity pressure this machine may + // be a different one. The runner reads this back from + // BUILDKITE_AGENT_META_DATA_LAUNCHED_INSTANCE_TYPE for the job log header + // and failure annotations. + tags["launched-instance-type"] = await getCloudLaunchedInstanceType(cloud); } options["tags"] = Object.entries(tags) diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 40014994f6ec..1134170f2480 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -1899,24 +1899,28 @@ let cloudInstanceType; * The instance type the cloud provider actually launched for this machine, * e.g. "r7i.2xlarge" on EC2 or "Standard_D8s_v5" on Azure. CI asks for one * type per lane (.buildkite/ci.mjs), but under capacity pressure the - * machine may be a different, slower type, and nothing else in a job log - * records which one it was. Looked up once per process. + * machine may be a different, slower type. scripts/agent.mjs records the + * launched type as the agent's `launched-instance-type` tag, which Buildkite + * exposes to the job as an environment variable. Looked up once per process. * @returns {string | undefined} */ export function getCloudInstanceType() { if (typeof cloudInstanceType !== "string") { - cloudInstanceType = fetchCloudInstanceType() || ""; + cloudInstanceType = + getEnv("BUILDKITE_AGENT_META_DATA_LAUNCHED_INSTANCE_TYPE", false) || fetchCloudInstanceType() || ""; } return cloudInstanceType || undefined; } /** + * Images baked before agent.mjs set the tag only have the `cloud` tag, so + * ask the metadata service directly. Synchronous and bounded, since this + * runs while printing a log header. * @returns {string | undefined} */ function fetchCloudInstanceType() { - // The `cloud` tag scripts/agent.mjs puts on the agents we launch ourselves. - // Anything without it (the darwin fleet, GitHub Actions) is not a machine - // whose type we chose, so there is nothing to compare against and no request. + // Without a `cloud` tag (the darwin fleet, GitHub Actions) this is not a + // machine whose type we chose, so there is nothing to compare against. const cloud = getEnv("BUILDKITE_AGENT_META_DATA_CLOUD", false); let request; if (cloud === "aws") { @@ -2343,6 +2347,34 @@ export async function getCloudMetadataTag(tag, cloud) { return getCloudMetadata(metadata, cloud); } +/** + * The instance type this machine was launched as, e.g. "r7i.2xlarge" or + * "Standard_D8s_v5". agent.mjs tags the agent with it at start; jobs read it + * back through getCloudInstanceType(). + * @param {Cloud} [cloud] + * @returns {Promise} + */ +export async function getCloudLaunchedInstanceType(cloud) { + cloud ??= await getCloud(); + + if (cloud === "azure") { + const body = await getCloudMetadata("", cloud); + if (!body) return; + try { + return JSON.parse(body)?.compute?.vmSize || undefined; + } catch {} + return; + } + + const metadata = { + "aws": "instance-type", + // "projects//machineTypes/" + "google": "machine-type", + }; + + return (await getCloudMetadata(metadata, cloud))?.split("/").pop() || undefined; +} + /** * @typedef {Object} AwsCredentials * @property {string} AccessKeyId