From 699bd73c0df5a477bab0b980115f786f473cf6b4 Mon Sep 17 00:00:00 2001 From: MP GOWTHAM Date: Thu, 13 Aug 2026 22:28:12 +0530 Subject: [PATCH 1/2] Add XPRESS (MS-XCA) decompression operations --- src/core/config/Categories.json | 4 +- src/core/lib/XPRESS.mjs | 263 ++++++++++++++++++ src/core/operations/XPRESSDecompress.mjs | 41 +++ .../operations/XPRESSHuffmanDecompress.mjs | 48 ++++ tests/operations/tests/XPRESS.mjs | 79 ++++++ 5 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 src/core/lib/XPRESS.mjs create mode 100644 src/core/operations/XPRESSDecompress.mjs create mode 100644 src/core/operations/XPRESSHuffmanDecompress.mjs create mode 100644 tests/operations/tests/XPRESS.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 6ad3adb776..1a8288c6cd 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -433,7 +433,9 @@ "LZMA Compress", "LZ4 Decompress", "LZ4 Compress", - "LZNT1 Decompress" + "LZNT1 Decompress", + "XPRESS Decompress", + "XPRESS LZ77+Huffman Decompress" ] }, { diff --git a/src/core/lib/XPRESS.mjs b/src/core/lib/XPRESS.mjs new file mode 100644 index 0000000000..9cfc748fde --- /dev/null +++ b/src/core/lib/XPRESS.mjs @@ -0,0 +1,263 @@ +/** + * XPRESS (MS-XCA) decompression. + * + * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + * + * Implements the two XPRESS variants from: + * https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/ + * (2.1 XPRESS Algorithm Details, 2.2 LZ77+Huffman Algorithm Details) + * + * Cross-validated against the reference decoder in The Sleuth Kit + * (tsk/fs/xpress.c) and go-ntfs (parser/xpress.go). + */ + +import OperationError from "../errors/OperationError.mjs"; + +/** Maximum output per call (Windows limits XPRESS blocks to 1 MiB). */ +const MAX_DECOMPRESSED = 1000000; + +/** + * Decompress an XPRESS plain-LZ77 stream. + * + * The stream is self-terminating: a sequence of 32-bit flag groups + * tested from bit 31 down. A clear bit is a literal byte. A set bit is + * a match described by an LE16 word, (offset-1) in the top 13 bits and + * (length-3) in the low 3 bits. A match whose low 3 bits are 7 uses the + * shared-nibble form: the low nibble of the next stream byte extends + * the length, and its high nibble extends the next match that also uses + * this form. A nibble of 15 selects a raw length: a byte, an LE16 if + * the byte is 255, or an LE32 if the LE16 is 0. The final flag group + * is padded with set bits; a match flag with no input left is the + * end-of-data marker. + * + * @param {byteArray} input + * @returns {byteArray} decompressed data + */ +export function decompress(input) { + const out = []; + let pending = -1; // offset of the shared-nibble byte, -1 when none pending + let flags = 0; + let flagsLeft = 0; + let i = 0; + + while (true) { + if (flagsLeft === 0) { + if (input.length - i < 4) + throw new OperationError("XPRESS: truncated flag group"); + flags = (input[i] | (input[i + 1] << 8) | + (input[i + 2] << 16) | (input[i + 3] << 24)) >>> 0; + i += 4; + flagsLeft = 32; + } + flagsLeft--; + if (((flags >>> flagsLeft) & 1) === 0) { + if (i >= input.length) + throw new OperationError("XPRESS: truncated literal"); + out.push(input[i++]); + continue; + } + + // A set flag with no input left is the end-of-data marker. + if (i >= input.length) + return out; + if (input.length - i < 2) + throw new OperationError("XPRESS: truncated match"); + const mb = input[i] | (input[i + 1] << 8); + i += 2; + const moff = (mb >>> 3) + 1; + let mlen = (mb & 7) + 3; + + if ((mb & 7) === 7) { + let nib; + if (pending === -1) { + if (i >= input.length) + throw new OperationError("XPRESS: truncated shared nibble"); + nib = input[i] & 0x0f; + pending = i++; + } else { + nib = input[pending] >>> 4; + pending = -1; + } + if (nib === 15) { + let v = 0; + if (i >= input.length) + throw new OperationError("XPRESS: truncated raw length"); + v = input[i++]; + if (v === 255) { + if (input.length - i < 2) + throw new OperationError("XPRESS: truncated raw length"); + v = input[i] | (input[i + 1] << 8); + i += 2; + if (v === 0) { + if (input.length - i < 4) + throw new OperationError("XPRESS: truncated raw length"); + v = (input[i] | (input[i + 1] << 8) | + (input[i + 2] << 16) | (input[i + 3] << 24)) >>> 0; + i += 4; + } + } + if (v < 22) + throw new OperationError("XPRESS: invalid match length"); + mlen = v + 3; + } else { + mlen = nib + 3; + } + } + + if (moff > 8192 || moff > out.length) + throw new OperationError("XPRESS: match offset out of range"); + if (out.length + mlen > MAX_DECOMPRESSED) + throw new OperationError("XPRESS: decompression ratio too large"); + + const start = out.length - moff; + for (let j = 0; j < mlen; j++) + out.push(out[start + j]); + } +} + +/** + * Decompress an XPRESS LZ77+Huffman stream into exactly + * decompressedSize bytes. + * + * The first 256 bytes hold 512 4-bit code lengths, the even symbol in + * the low nibble and the odd in the high. Canonical codes are assigned + * in (length, symbol) order, most-significant bit first. The bit stream + * follows as LE16 words, MSB first, read through a 32-bit register + * refilled while fewer than 15 bits remain. Symbols 0..255 are + * literals. Symbol 256 is end-of-data; mid-stream it decodes as a match + * of length 3 at offset 1. Symbols 257..511 are matches: ((s-256)>>4) + * selects the offset bit width, ((s-256)&15) the base length, with a + * nibble of 15 selecting a raw length byte (an LE16 if 255, an LE32 if + * the LE16 is 0). + * + * @param {byteArray} input + * @param {number} decompressedSize + * @returns {byteArray} decompressed data + */ +export function decompressHuffman(input, decompressedSize) { + if (decompressedSize <= 0 || decompressedSize > MAX_DECOMPRESSED) + throw new OperationError("XPRESS: invalid decompressed size"); + if (input.length < 256) + throw new OperationError("XPRESS: truncated Huffman table"); + + const lens = new Array(512); + for (let l = 0; l < 256; l++) { + lens[l * 2] = input[l] & 0x0f; + lens[l * 2 + 1] = input[l] >>> 4; + } + + // Decode table in canonical (length, symbol) order, MSB first. + const TABLE_BITS = 15; + const TABLE_SIZE = 1 << TABLE_BITS; + const table = new Array(TABLE_SIZE); + let e = 0; + for (let l = 1; l <= TABLE_BITS; l++) { + for (let s = 0; s < 512; s++) { + if (lens[s] === l) { + const n = 1 << (TABLE_BITS - l); + for (let k = 0; k < n; k++) + table[e++] = s; + } + } + } + if (e !== TABLE_SIZE) + throw new OperationError("XPRESS: invalid Huffman code lengths"); + + // Preload two LE16 words, most-significant bit first. + let bits = 0; + let nbits = 0; + let i = 256; + while (nbits < 32) { + if (input.length - i < 2) + throw new OperationError("XPRESS: truncated bit stream"); + bits = ((bits >>> 0) | (input[i] | (input[i + 1] << 8)) << (16 - nbits)) >>> 0; + i += 2; + nbits += 16; + } + + const out = []; + while (out.length < decompressedSize) { + while (nbits < 15) { + if (input.length - i < 2) + throw new OperationError("XPRESS: truncated bit stream"); + bits = ((bits >>> 0) | (input[i] | (input[i + 1] << 8)) << (16 - nbits)) >>> 0; + i += 2; + nbits += 16; + } + const sym = table[(bits >>> 17) & 0x7fff]; + const clen = lens[sym]; + bits = (bits >>> 0) << clen; + nbits -= clen; + + if (sym < 256) { + out.push(sym); + continue; + } + + if (sym === 256) { + // End of data; mid-stream it decodes as a match(3, 1). + if (out.length === decompressedSize) + break; + if (decompressedSize - out.length < 3) + throw new OperationError("XPRESS: corrupt end-of-data marker"); + const start = out.length - 1; + for (let j = 0; j < 3; j++) + out.push(out[start + j]); + continue; + } + + const hb = (sym - 256) >>> 4; + let mlen = (sym - 256) & 15; + if (mlen === 15) { + let v = 0; + if (i >= input.length) + throw new OperationError("XPRESS: truncated raw length"); + v = input[i++]; + if (v === 255) { + if (input.length - i < 2) + throw new OperationError("XPRESS: truncated raw length"); + v = input[i] | (input[i + 1] << 8); + i += 2; + if (v === 0) { + if (input.length - i < 4) + throw new OperationError("XPRESS: truncated raw length"); + v = (input[i] | (input[i + 1] << 8) | + (input[i + 2] << 16) | (input[i + 3] << 24)) >>> 0; + i += 4; + } + mlen = v + 3; + } else { + mlen = v + 18; + } + } else { + mlen += 3; + } + + while (nbits < hb) { + if (input.length - i < 2) + throw new OperationError("XPRESS: truncated bit stream"); + bits = ((bits >>> 0) | (input[i] | (input[i + 1] << 8)) << (16 - nbits)) >>> 0; + i += 2; + nbits += 16; + } + let moff = 0; + if (hb > 0) { + moff = (bits >>> (32 - hb)) & ((1 << hb) - 1); + bits = (bits >>> 0) << hb; + nbits -= hb; + } + moff += 1 << hb; + + if (moff > out.length) + throw new OperationError("XPRESS: match offset out of range"); + if (out.length + mlen > MAX_DECOMPRESSED) + throw new OperationError("XPRESS: decompression ratio too large"); + + const start = out.length - moff; + for (let j = 0; j < mlen; j++) + out.push(out[start + j]); + } + return out; +} diff --git a/src/core/operations/XPRESSDecompress.mjs b/src/core/operations/XPRESSDecompress.mjs new file mode 100644 index 0000000000..6b94fc1a4e --- /dev/null +++ b/src/core/operations/XPRESSDecompress.mjs @@ -0,0 +1,41 @@ +/** + * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import {decompress} from "../lib/XPRESS.mjs"; + +/** + * XPRESS Decompress operation + */ +class XPRESSDecompress extends Operation { + + /** + * XPRESS Decompress constructor + */ + constructor() { + super(); + + this.name = "XPRESS Decompress"; + this.module = "Compression"; + this.description = "Decompresses data using the XPRESS plain LZ77 algorithm (MS-XCA section 2.1).

Similar to the Windows API RtlDecompressBuffer with COMPRESSION_FORMAT_XPRESS."; + this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15"; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.args = []; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + return decompress(input); + } + +} + +export default XPRESSDecompress; diff --git a/src/core/operations/XPRESSHuffmanDecompress.mjs b/src/core/operations/XPRESSHuffmanDecompress.mjs new file mode 100644 index 0000000000..34b938e937 --- /dev/null +++ b/src/core/operations/XPRESSHuffmanDecompress.mjs @@ -0,0 +1,48 @@ +/** + * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import {decompressHuffman} from "../lib/XPRESS.mjs"; + +/** + * XPRESS LZ77+Huffman Decompress operation + */ +class XPRESSHuffmanDecompress extends Operation { + + /** + * XPRESS LZ77+Huffman Decompress constructor + */ + constructor() { + super(); + + this.name = "XPRESS LZ77+Huffman Decompress"; + this.module = "Compression"; + this.description = "Decompresses data using the XPRESS LZ77+Huffman algorithm (MS-XCA section 2.2).

The uncompressed size must be known in advance, as it is from the WOF chunk table or WIM header, so it is taken as an argument."; + this.infoURL = "https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15"; + this.inputType = "byteArray"; + this.outputType = "byteArray"; + this.args = [ + { + "name": "Decompressed size", + "type": "number", + "value": 4096 + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + const size = args[0]; + return decompressHuffman(input, size); + } + +} + +export default XPRESSHuffmanDecompress; diff --git a/tests/operations/tests/XPRESS.mjs b/tests/operations/tests/XPRESS.mjs new file mode 100644 index 0000000000..0df3b62921 --- /dev/null +++ b/tests/operations/tests/XPRESS.mjs @@ -0,0 +1,79 @@ +/** + * XPRESS tests. + * + * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + + // MS-XCA section 3.1 worked example (plain LZ77). + { + name: "XPRESS Decompress: worked example", + input: "0000000047484f53542f2f5245434f5645522064617461207265636f7665727920656e67ffffff07696e652e0a", + expectedOutput: "GHOST//RECOVER data recovery engine.\n", + recipeConfig: [ + { + "op": "From Hex", + "args": ["Space"] + }, + { + "op": "XPRESS Decompress", + "args": [] + } + ] + }, + + // Matches with shared nibbles, raw lengths and 64 MiB of output. + { + name: "XPRESS Decompress: repeated string compression", + input: "ffffff1f61626317000fff2601", + expectedOutput: "abc".repeat(100), + recipeConfig: [ + { + "op": "From Hex", + "args": ["Space"] + }, + { + "op": "XPRESS Decompress", + "args": [] + } + ] + }, + + // Literals plus 65536-byte offset matches (anchor example). + { + name: "XPRESS Decompress: anchor example", + input: "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 30 23 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 a8 dc 00 00 ff 26 01", + expectedOutput: "abc".repeat(100), + recipeConfig: [ + { + "op": "From Hex", + "args": ["Space"] + }, + { + "op": "XPRESS LZ77+Huffman Decompress", + "args": [300] + } + ] + }, + + // MS-XCA section 3.1 worked example (LZ77+Huffman), ten repetitions. + { + name: "XPRESS LZ77+Huffman Decompress: worked example", + input: "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 50 66 55 55 66 65 55 45 65 55 55 65 55 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 b4 e3 a9 8f 5e e7 62 8e bc 5f ac 28 47 19 40 42 98 aa eb 89 7c da 20 5c 61 96 e4 b6 ff 38 01 00 00", + expectedOutput: "The quick brown fox jumps over the lazy dog. ".repeat(8), + recipeConfig: [ + { + "op": "From Hex", + "args": ["Space"] + }, + { + "op": "XPRESS LZ77+Huffman Decompress", + "args": [360] + } + ] + } +]); \ No newline at end of file From 98ec000e621f11b39773ff0cc654e3947f12545e Mon Sep 17 00:00:00 2001 From: MP GOWTHAM Date: Fri, 14 Aug 2026 18:54:45 +0530 Subject: [PATCH 2/2] XPRESS: fix empty-output EOD underflow, raise output cap to 32 MiB The mid-stream end-of-data-as-match(3,1) fallback indexed out[-1] when the output was still empty, pushing undefined into the result; guard it the same way the Go and C ports now do. Also raise MAX_DECOMPRESSED from 1 MiB to 32 MiB (WIM chunks can be up to 32 MiB; WOF is 1 MiB) and update the author tags to the real email. Found by the FuzzXpress target added to klauspost/compress. --- src/core/lib/XPRESS.mjs | 9 +++++---- src/core/operations/XPRESSDecompress.mjs | 2 +- src/core/operations/XPRESSHuffmanDecompress.mjs | 2 +- tests/operations/tests/XPRESS.mjs | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/core/lib/XPRESS.mjs b/src/core/lib/XPRESS.mjs index 9cfc748fde..00a34a822a 100644 --- a/src/core/lib/XPRESS.mjs +++ b/src/core/lib/XPRESS.mjs @@ -1,7 +1,7 @@ /** * XPRESS (MS-XCA) decompression. * - * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @author MP Gowtham [gowthamrockerzzz@gmail.com] * @copyright Crown Copyright 2026 * @license Apache-2.0 * @@ -15,8 +15,9 @@ import OperationError from "../errors/OperationError.mjs"; -/** Maximum output per call (Windows limits XPRESS blocks to 1 MiB). */ -const MAX_DECOMPRESSED = 1000000; +/** Maximum output per call (Windows sizes XPRESS blocks at up to + * 32 MiB for WIM chunks and up to 1 MiB for WOF chunks). */ +const MAX_DECOMPRESSED = 32 * 1024 * 1024; /** * Decompress an XPRESS plain-LZ77 stream. @@ -200,7 +201,7 @@ export function decompressHuffman(input, decompressedSize) { // End of data; mid-stream it decodes as a match(3, 1). if (out.length === decompressedSize) break; - if (decompressedSize - out.length < 3) + if (out.length === 0 || decompressedSize - out.length < 3) throw new OperationError("XPRESS: corrupt end-of-data marker"); const start = out.length - 1; for (let j = 0; j < 3; j++) diff --git a/src/core/operations/XPRESSDecompress.mjs b/src/core/operations/XPRESSDecompress.mjs index 6b94fc1a4e..a8167d37fc 100644 --- a/src/core/operations/XPRESSDecompress.mjs +++ b/src/core/operations/XPRESSDecompress.mjs @@ -1,5 +1,5 @@ /** - * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @author MP Gowtham [gowthamrockerzzz@gmail.com] * @copyright Crown Copyright 2026 * @license Apache-2.0 */ diff --git a/src/core/operations/XPRESSHuffmanDecompress.mjs b/src/core/operations/XPRESSHuffmanDecompress.mjs index 34b938e937..0b667dcb98 100644 --- a/src/core/operations/XPRESSHuffmanDecompress.mjs +++ b/src/core/operations/XPRESSHuffmanDecompress.mjs @@ -1,5 +1,5 @@ /** - * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @author MP Gowtham [gowthamrockerzzz@gmail.com] * @copyright Crown Copyright 2026 * @license Apache-2.0 */ diff --git a/tests/operations/tests/XPRESS.mjs b/tests/operations/tests/XPRESS.mjs index 0df3b62921..0da54e6d27 100644 --- a/tests/operations/tests/XPRESS.mjs +++ b/tests/operations/tests/XPRESS.mjs @@ -1,7 +1,7 @@ /** * XPRESS tests. * - * @author MP Gowtham [mpgowtham@users.noreply.github.com] + * @author MP Gowtham [gowthamrockerzzz@gmail.com] * @copyright Crown Copyright 2026 * @license Apache-2.0 */