test: verify exact pixel output for every codec + fail CI on silently-skipped suites

BENCHMARKING.md

@@ -5,8 +5,9 @@ what the numbers mean, why they don't match real wall-clock time, and how
 to read warnings from the CodSpeed dashboard.
 
 Bench files live under `packages/*/bench/*.bench.js` and are driven by
-`vitest bench` + `@codspeed/vitest-plugin@^4`. The full pipeline is in
-`.github/workflows/pr-checks.yml` (job: `codspeed-bench`).
+`vitest bench` + `@codspeed/vitest-plugin@^5`. The full pipeline is in
+`.github/workflows/pr-checks.yml` (jobs: `codspeed-bench` and
+`codspeed-walltime`).
 
 ## TL;DR
 
@@ -21,10 +22,10 @@ Bench files live under `packages/*/bench/*.bench.js` and are driven by
   inflates 30–100× vs production V8; for wasm decode kernels it's ~5–15×;
   for pure compute it's roughly 1×.
 
-## Why simulation, not walltime
+## Two instruments: simulation is the gate, walltime is the reality check
 
 CodSpeed has two instruments: `simulation` (Cachegrind) and `walltime`
-(real CPU, statistical sampling on macro-runners).
+(real CPU, statistical sampling on macro-runners). We run **both**:
 
 | | Simulation | Walltime |
 |---|---|---|
@@ -36,9 +37,22 @@ CodSpeed has two instruments: `simulation` (Cachegrind) and `walltime`
 | Honest about JIT tier-up | No (depends on tier at bench time) | Yes |
 | Good for | Regression detection on a PR | Absolute production-like timing |
 
-We chose simulation because regression detection is the primary goal —
-catching "this PR slowed openjpeg by 5%" matters more than knowing the
-exact ms a user's browser will take.
+Simulation is the **blocking gate** because regression detection is the
+primary goal — catching "this PR slowed openjpeg by 5%" matters more than
+knowing the exact ms a user's browser will take, and simulation's <1%
+determinism flags small algorithmic slips that walltime noise would hide.
+
+Walltime runs as a **second, advisory instrument** (`codspeed-walltime`
+job, `continue-on-error: true`) on CodSpeed macro runners — 16-core ARM64
+bare-metal machines. It covers simulation's two blind spots: real-time
+effects (branch prediction, actual caches) that instruction counting
+models away, and the pure-JS packages (`little-endian`/`big-endian`)
+where the no-JIT simulation model is furthest from production V8.
+Walltime benches run packages sequentially (`--concurrency 1`) because
+parallel processes contend for cores and add noise; simulation is immune
+to contention so it keeps `--parallel`. Note the macro runners are ARM64:
+walltime numbers are real milliseconds, but on different silicon than
+most x86 production traffic.
 
 ## How the numbers get inflated
 

package.json

@@ -5,17 +5,19 @@
     "packages/*"
   ],
   "devDependencies": {
-    "@codspeed/vitest-plugin": "^4.0.1",
-    "@vitest/coverage-v8": "^2.1.8",
+    "@codspeed/vitest-plugin": "^5.7.1",
+    "@vitest/coverage-v8": "^3.2.4",
     "dotenv": "^14.1.0",
     "lerna": "^8.0.0",
-    "vitest": "^2.1.8"
+    "vitest": "^3.2.4",
+    "playwright-core": "^1.49.0"
   },
   "engines": {
     "node": ">=18"
   },
   "scripts": {
     "build:all": "lerna run build:ci --parallel --stream",
+    "test": "vitest run",
     "test:all": "lerna run test:ci --parallel --stream",
     "bench": "lerna run bench --parallel --stream",
     "bench:ci": "lerna run bench --parallel --stream",

packages/big-endian/bench/decode.bench.js

@@ -20,8 +20,14 @@ describe("big-endian decode (byte-swap)", () => {
     decode({ bitsAllocated: 16, pixelRepresentation: 1 }, data16)
   })
 
+  // The 16-bit benches above do real per-pixel swap work and are left
+  // unbatched. The 8-bit path is a bare property assignment, so a single
+  // call is swamped by fixed harness overhead and cache-model variation
+  // across runner CPUs; batch x100 so the body is measurable.
   const data8 = makeBuffer(SIZE_512x512)
-  bench("8-bit passthrough, 512x512", () => {
-    decode({ bitsAllocated: 8 }, data8)
+  bench("8-bit passthrough, 512x512 x100", () => {
+    for (let i = 0; i < 100; i++) {
+      decode({ bitsAllocated: 8 }, data8)
+    }
   })
 })

packages/big-endian/src/index.js

@@ -2,18 +2,28 @@
 function swap16(val) {
   return ((val & 0xff) << 8) | ((val >> 8) & 0xff);
 }
-
+
+function swap32(val) {
+  return (
+    ((val & 0xff) << 24) |
+    ((val & 0xff00) << 8) |
+    ((val >> 8) & 0xff00) |
+    ((val >> 24) & 0xff)
+  );
+}
+
 /**
  * Decodes the provided pixelData and sets the `pixelData` property
  * of the imageFrame object to the decoded representation.
- * 
+ *
  * Set pixelData will be `Uint16Array` if `pixelRepresentation` is 0,
- * otherwise it will be an `Int16Array`
- * 
+ * otherwise it will be an `Int16Array`. 32-bit data is byte-swapped
+ * into a `Float32Array`, mirroring the little-endian package.
+ *
  * @param {object} imageFrame
- * @param {number} imageFrame.bitsAllocated - 16 or 8
+ * @param {number} imageFrame.bitsAllocated - 32, 16, 8 or 1
  * @param {number} imageFrame.pixelRepresentation - 0 or 1
- * @param {*} pixelData 
+ * @param {*} pixelData
  */
 function decode(imageFrame, pixelData) {
   if (imageFrame.bitsAllocated === 16) {
@@ -38,8 +48,25 @@ function decode(imageFrame, pixelData) {
     for (let i = 0; i < imageFrame.pixelData.length; i++) {
       imageFrame.pixelData[i] = swap16(imageFrame.pixelData[i]);
     }
-  } else if (imageFrame.bitsAllocated === 8) {
+  } else if (imageFrame.bitsAllocated === 8 || imageFrame.bitsAllocated === 1) {
     imageFrame.pixelData = pixelData;
+  } else if (imageFrame.bitsAllocated === 32) {
+    let arrayBuffer = pixelData.buffer;
+
+    let offset = pixelData.byteOffset;
+    const length = pixelData.length;
+    // Float32Array views must be 4-byte aligned; shift unaligned data
+    if (offset % 4) {
+      arrayBuffer = arrayBuffer.slice(offset);
+      offset = 0;
+    }
+
+    const swapView = new Uint32Array(arrayBuffer, offset, length / 4);
+    for (let i = 0; i < swapView.length; i++) {
+      swapView[i] = swap32(swapView[i]);
+    }
+
+    imageFrame.pixelData = new Float32Array(arrayBuffer, offset, length / 4);
   }
 
   return imageFrame;

packages/big-endian/test/decode.test.js

@@ -43,6 +43,43 @@ describe("big-endian decode", () => {
     expect(Array.from(imageFrame.pixelData)).toEqual([1, 2])
   })
 
+  it("passes 1-bit pixel data through unchanged", () => {
+    const pixelData = new Uint8Array([0b10101010])
+    const imageFrame = { bitsAllocated: 1 }
+
+    decode(imageFrame, pixelData)
+
+    expect(imageFrame.pixelData).toBe(pixelData)
+  })
+
+  it("byte-swaps 32-bit pixel data into Float32Array", () => {
+    const source = new Float32Array([1.5, -2.25, 3.75])
+    // Build the big-endian byte stream for those values
+    const bigEndianBytes = new Uint8Array(source.length * 4)
+    const view = new DataView(bigEndianBytes.buffer)
+    source.forEach((value, i) => view.setFloat32(i * 4, value, false))
+    const imageFrame = { bitsAllocated: 32 }
+
+    decode(imageFrame, bigEndianBytes)
+
+    expect(imageFrame.pixelData).toBeInstanceOf(Float32Array)
+    expect(Array.from(imageFrame.pixelData)).toEqual([1.5, -2.25, 3.75])
+  })
+
+  it("realigns 32-bit pixel data when byteOffset is not 4-byte aligned", () => {
+    const source = new Float32Array([1.5, -2.25])
+    const padded = new Uint8Array(2 + source.length * 4)
+    const view = new DataView(padded.buffer)
+    source.forEach((value, i) => view.setFloat32(2 + i * 4, value, false))
+    const pixelData = new Uint8Array(padded.buffer, 2, source.length * 4)
+    const imageFrame = { bitsAllocated: 32 }
+
+    decode(imageFrame, pixelData)
+
+    expect(imageFrame.pixelData).toBeInstanceOf(Float32Array)
+    expect(Array.from(imageFrame.pixelData)).toEqual([1.5, -2.25])
+  })
+
   it("returns the same imageFrame object", () => {
     const imageFrame = { bitsAllocated: 8 }
     const result = decode(imageFrame, new Uint8Array([0]))

packages/big-endian/vitest.config.mjs

@@ -4,6 +4,18 @@ import codspeedPlugin from "@codspeed/vitest-plugin"
 export default defineConfig({
   plugins: [codspeedPlugin()],
   test: {
+    // Under the CodSpeed simulation instrument the entire process runs ~60x
+    // slower under valgrind while vitest's hard-coded 60s worker-RPC timer
+    // counts real seconds, so large bench suites structurally hit "Timeout
+    // calling onTaskUpdate" AFTER their benches complete and upload. Ignore
+    // that exit-code noise in simulation only; walltime and test runs stay
+    // strict.
+    // (CODSPEED_ENV is set whenever the CodSpeed runner is active; the
+    // mode string is "instrumentation" on older runners and "simulation"
+    // on newer ones, so match anything except walltime.)
+    dangerouslyIgnoreUnhandledErrors:
+      process.env.CODSPEED_ENV !== undefined &&
+      process.env.CODSPEED_RUNNER_MODE !== "walltime",
     name: "big-endian",
     include: ["test/**/*.test.js"],
     benchmark: {

packages/charls/bench/decode.bench.js

@@ -59,7 +59,9 @@ let coldEnc
 let warmEnc
 if (!skip) {
   const factory = (await import(distPath)).default ?? (await import(distPath))
-  codec = await factory()
+  // Silence wasm stdout/stderr so vitest's console interception never runs
+  // inside a measured bench body (see openjphjs bench for details).
+  codec = await factory({ print: () => {}, printErr: () => {} })
 
   // Cold instances: one per fixture, constructed but never decoded.
   // The bench body will be the first decode call on this instance.
@@ -86,14 +88,21 @@ if (!skip) {
 }
 
 describe.skipIf(skip)("charls JPEG-LS (wasm)", () => {
-  bench("instantiate+destroy JpegLSDecoder", () => {
-    const d = new codec.JpegLSDecoder()
-    d.delete()
+  // Batched x50: a single instantiate+destroy is ~60 µs, where fixed harness
+  // overhead and cache-model variation across runner CPUs dominate; the loop
+  // puts the body in the ms range so the codec work is the signal.
+  bench("instantiate+destroy JpegLSDecoder x50", () => {
+    for (let i = 0; i < 50; i++) {
+      const d = new codec.JpegLSDecoder()
+      d.delete()
+    }
   })
 
-  bench("instantiate+destroy JpegLSEncoder", () => {
-    const e = new codec.JpegLSEncoder()
-    e.delete()
+  bench("instantiate+destroy JpegLSEncoder x50", () => {
+    for (let i = 0; i < 50; i++) {
+      const e = new codec.JpegLSEncoder()
+      e.delete()
+    }
   })
 
   bench("decode CT1.JLS (.80 lossless, 512x512x16bit) — cold", () => {

packages/charls/test/decode.test.js

@@ -8,14 +8,24 @@ const distDir = resolve(__dirname, "../dist")
 const fixturesDir = resolve(__dirname, "fixtures")
 
 const ct1Encoded = readFileSync(resolve(fixturesDir, "CT1.JLS"))
+// CT1.RAW is the lossless reference for CT1.JLS. The same slice ships as
+// openjpeg's CT1.RAW and openjphjs's CT1.RAW; charls, openjpeg and openjph
+// all decode their CT1 fixtures to these exact bytes, so the three codecs
+// cross-validate each other.
+const ct1Raw = readFileSync(resolve(fixturesDir, "CT1.RAW"))
 const ct2Encoded = readFileSync(resolve(fixturesDir, "CT2.JLS"))
 const ct2Raw = readFileSync(resolve(fixturesDir, "CT2.RAW"))
 // CT-512x512-near-lossless.JLS is a real .81 (JPEG-LS Lossy / Near-Lossless)
 // payload extracted from a Cornerstone3D test DICOM. Decoding exercises the
 // same charls codec but verifies the near-lossless code path (NEAR > 0).
+// Near-lossless decoding is deterministic, so the decoder's own output is
+// pinned as a regression golden (identical across all three build variants).
 const ctNearLossless = readFileSync(
   resolve(fixturesDir, "CT-512x512-near-lossless.JLS")
 )
+const ctNearLosslessRaw = readFileSync(
+  resolve(fixturesDir, "CT-512x512-near-lossless.RAW")
+)
 
 async function loadModule(modulePath) {
   const mod = await import(modulePath)
@@ -37,11 +47,17 @@ describe.each(buildVariants)("charls JPEG-LS decode — $name", ({ path, dist })
     if (isBuilt) codec = await loadModule(path)
   })
 
+  // In CI a missing dist means the build/artifact pipeline broke; fail loudly
+  // instead of letting every skipIf() below silently skip the suite.
+  it.runIf(process.env.CI)("dist is present in CI", () => {
+    expect(isBuilt, `${dist} missing — build artifact was not replayed`).toBe(true)
+  })
+
   it.skipIf(!isBuilt)("exposes a version string", () => {
     expect(typeof codec.getVersion()).toBe("string")
   })
 
-  it.skipIf(!isBuilt)("decodes CT1.JLS to a 512x512 16-bit monochrome frame", () => {
+  it.skipIf(!isBuilt)("decodes CT1.JLS to bytes matching CT1.RAW (lossless)", () => {
     const decoder = new codec.JpegLSDecoder()
     decoder.getEncodedBuffer(ct1Encoded.length).set(ct1Encoded)
     decoder.decode()
@@ -53,7 +69,8 @@ describe.each(buildVariants)("charls JPEG-LS decode — $name", ({ path, dist })
     expect(frameInfo.componentCount).toBe(1)
 
     const decoded = decoder.getDecodedBuffer()
-    expect(decoded.length).toBe(512 * 512 * 2)
+    expect(decoded.length).toBe(ct1Raw.length)
+    expect(Buffer.from(decoded).equals(ct1Raw)).toBe(true)
 
     decoder.delete()
   })
@@ -71,7 +88,7 @@ describe.each(buildVariants)("charls JPEG-LS decode — $name", ({ path, dist })
   })
 
   it.skipIf(!isBuilt)(
-    "decodes a near-lossless CT JLS (transfer syntax .81)",
+    "decodes a near-lossless CT JLS (transfer syntax .81) matching the pinned golden output",
     () => {
       const decoder = new codec.JpegLSDecoder()
       decoder.getEncodedBuffer(ctNearLossless.length).set(ctNearLossless)
@@ -84,7 +101,8 @@ describe.each(buildVariants)("charls JPEG-LS decode — $name", ({ path, dist })
       expect(frameInfo.componentCount).toBe(1)
 
       const decoded = decoder.getDecodedBuffer()
-      expect(decoded.length).toBe(512 * 512 * 2)
+      expect(decoded.length).toBe(ctNearLosslessRaw.length)
+      expect(Buffer.from(decoded).equals(ctNearLosslessRaw)).toBe(true)
       decoder.delete()
     }
   )
@@ -114,8 +132,11 @@ describe.each(encoderVariants)(
       encoder.setNearLossless(0)
       encoder.encode()
       const encoded = encoder.getEncodedBuffer()
-      expect(encoded.length).toBeGreaterThan(0)
-      expect(encoded.length).toBeLessThan(ct2Raw.length)
+      // Compression-quality regression bounds. Measured 115504 bytes on
+      // 2026-07-07 (emsdk 3.1.74): the ceiling catches an encoder that
+      // regresses compression ratio; the floor catches silent truncation.
+      expect(encoded.length).toBeGreaterThan(115504 * 0.5)
+      expect(encoded.length).toBeLessThan(115504 * 1.10)
 
       const decoder = new codec.JpegLSDecoder()
       decoder.getEncodedBuffer(encoded.length).set(encoded)