diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 6ad3adb776..2164a51c7e 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -68,6 +68,8 @@ "Swap endianness", "To MessagePack", "From MessagePack", + "To Python Marshal", + "From Python Marshal", "To Braille", "From Braille", "Parse TLV", @@ -619,4 +621,4 @@ "Comment" ] } -] \ No newline at end of file +] diff --git a/src/core/lib/PythonMarshal.mjs b/src/core/lib/PythonMarshal.mjs new file mode 100644 index 0000000000..d2a027b9ca --- /dev/null +++ b/src/core/lib/PythonMarshal.mjs @@ -0,0 +1,505 @@ +/** + * @author GCHQ + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ + +import OperationError from "../errors/OperationError.mjs"; + +const FLAG_REF = 0x80; +const MAX_ITEMS = 1000000; +const MAX_DEPTH = 1000; +const NULL = Symbol("marshal null"); + +const TYPE = { + NULL: 0x30, + NONE: 0x4e, + FALSE: 0x46, + TRUE: 0x54, + STOPITER: 0x53, + ELLIPSIS: 0x2e, + INT: 0x69, + INT64: 0x49, + FLOAT: 0x66, + BINARY_FLOAT: 0x67, + COMPLEX: 0x78, + BINARY_COMPLEX: 0x79, + LONG: 0x6c, + STRING: 0x73, + INTERNED: 0x74, + STRINGREF: 0x52, + TUPLE: 0x28, + SMALL_TUPLE: 0x29, + LIST: 0x5b, + DICT: 0x7b, + CODE: 0x63, + UNICODE: 0x75, + SET: 0x3c, + FROZENSET: 0x3e, + ASCII: 0x61, + ASCII_INTERNED: 0x41, + SHORT_ASCII: 0x7a, + SHORT_ASCII_INTERNED: 0x5a, + REF: 0x72, +}; + +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const textEncoder = new TextEncoder(); + +/** + * Python marshal format codec for values expressible in JSON. + */ +class PythonMarshal { + + /** + * @param {ArrayBuffer} input + * @returns {Object|Array|string|number|boolean|null} + */ + static decode(input) { + const reader = new Reader(new Uint8Array(input)); + if (!reader.bytes.length) throw new OperationError("Python marshal input is empty"); + + const value = reader.readValue(0); + if (value === NULL) throw new OperationError("Unexpected null marker in Python marshal input"); + try { + JSON.stringify(value); + } catch (err) { + throw new OperationError("Python marshal data contains a cyclic value, which is not representable as JSON"); + } + return value; + } + + /** + * @param {Object|Array|string|number|boolean|null} value + * @returns {ArrayBuffer} + */ + static encode(value) { + const writer = new Writer(); + writer.writeValue(value, 0); + return writer.finish(); + } + +} + +class Reader { + + /** + * @param {Uint8Array} bytes + */ + constructor(bytes) { + this.bytes = bytes; + this.position = 0; + this.refs = []; + } + + /** + * @param {number} length + * @returns {Uint8Array} + */ + readBytes(length) { + if (!Number.isSafeInteger(length) || length < 0 || length > this.bytes.length - this.position) { + throw new OperationError("Unexpected end of Python marshal input"); + } + const start = this.position; + this.position += length; + return this.bytes.subarray(start, this.position); + } + + /** + * @returns {number} + */ + readUint8() { + return this.readBytes(1)[0]; + } + + /** + * @returns {number} + */ + readInt32() { + const bytes = this.readBytes(4); + return new DataView(bytes.buffer, bytes.byteOffset, 4).getInt32(0, true); + } + + /** + * @returns {number} + */ + readUint32() { + const bytes = this.readBytes(4); + return new DataView(bytes.buffer, bytes.byteOffset, 4).getUint32(0, true); + } + + /** + * @returns {number} + */ + readFloat64() { + const bytes = this.readBytes(8); + return new DataView(bytes.buffer, bytes.byteOffset, 8).getFloat64(0, true); + } + + /** + * @param {Uint8Array} bytes + * @returns {string} + */ + decodeText(bytes) { + try { + return textDecoder.decode(bytes); + } catch (err) { + throw new OperationError("Python marshal string is not valid UTF-8"); + } + } + + /** + * @param {number} depth + * @returns {Object|Array|string|number|boolean|null|symbol} + */ + readValue(depth) { + if (depth > MAX_DEPTH) throw new OperationError("Python marshal data is nested too deeply"); + + const typeWithFlags = this.readUint8(); + const type = typeWithFlags & ~FLAG_REF; + const shouldStoreReference = (typeWithFlags & FLAG_REF) !== 0; + const referenceIndex = shouldStoreReference ? this.refs.length : -1; + let value; + + if (shouldStoreReference && (type === TYPE.LIST || type === TYPE.TUPLE || type === TYPE.SMALL_TUPLE || type === TYPE.SET || type === TYPE.FROZENSET)) { + value = []; + this.refs.push(value); + this.readSequence(value, type === TYPE.SMALL_TUPLE ? this.readUint8() : this.readUint32(), depth); + return value; + } + if (shouldStoreReference && type === TYPE.DICT) { + value = Object.create(null); + this.refs.push(value); + this.readDict(value, depth); + return value; + } + + switch (type) { + case TYPE.NULL: + value = NULL; + break; + case TYPE.NONE: + value = null; + break; + case TYPE.FALSE: + value = false; + break; + case TYPE.TRUE: + value = true; + break; + case TYPE.INT: + value = this.readInt32(); + break; + case TYPE.INT64: + value = this.readInteger64(); + break; + case TYPE.LONG: + value = this.readLong(); + break; + case TYPE.FLOAT: + value = Number(this.decodeText(this.readBytes(this.readUint8()))); + break; + case TYPE.BINARY_FLOAT: + value = this.readFloat64(); + break; + case TYPE.STRING: + value = this.readByteString(this.readUint32()); + break; + case TYPE.INTERNED: + case TYPE.UNICODE: + case TYPE.ASCII: + case TYPE.ASCII_INTERNED: + value = this.decodeText(this.readBytes(this.readUint32())); + break; + case TYPE.SHORT_ASCII: + case TYPE.SHORT_ASCII_INTERNED: + value = this.decodeText(this.readBytes(this.readUint8())); + break; + case TYPE.STRINGREF: + case TYPE.REF: + value = this.readReference(); + break; + case TYPE.LIST: + case TYPE.TUPLE: + case TYPE.SET: + case TYPE.FROZENSET: + value = []; + if (shouldStoreReference) this.refs[referenceIndex] = value; + this.readSequence(value, this.readUint32(), depth); + break; + case TYPE.SMALL_TUPLE: + value = []; + if (shouldStoreReference) this.refs[referenceIndex] = value; + this.readSequence(value, this.readUint8(), depth); + break; + case TYPE.DICT: + value = Object.create(null); + if (shouldStoreReference) this.refs[referenceIndex] = value; + this.readDict(value, depth); + break; + case TYPE.STOPITER: + case TYPE.ELLIPSIS: + case TYPE.COMPLEX: + case TYPE.BINARY_COMPLEX: + case TYPE.CODE: + throw new OperationError(`Python marshal type '${String.fromCharCode(type)}' is not representable as JSON`); + default: + throw new OperationError(`Unsupported Python marshal type 0x${type.toString(16).padStart(2, "0")}`); + } + + if (shouldStoreReference) this.refs[referenceIndex] = value; + if (typeof value === "number" && !Number.isFinite(value)) { + throw new OperationError("Non-finite Python marshal numbers are not representable as JSON"); + } + return value; + } + + /** + * @returns {number|Object} + */ + readInteger64() { + const bytes = this.readBytes(8); + let value = 0n; + for (let i = 7; i >= 0; i--) value = (value << 8n) | BigInt(bytes[i]); + if (value >= (1n << 63n)) value -= 1n << 64n; + return this.toJsonInteger(value); + } + + /** + * @returns {number|Object} + */ + readLong() { + const size = this.readInt32(); + const digitCount = Math.abs(size); + if (digitCount > MAX_ITEMS) throw new OperationError("Python marshal integer has too many digits"); + + let value = 0n; + for (let i = 0; i < digitCount; i++) { + const bytes = this.readBytes(2); + const digit = BigInt(bytes[0] | (bytes[1] << 8)); + value |= digit << BigInt(i * 15); + } + return this.toJsonInteger(size < 0 ? -value : value); + } + + /** + * @param {bigint} value + * @returns {number|Object} + */ + toJsonInteger(value) { + if (value >= BigInt(Number.MIN_SAFE_INTEGER) && value <= BigInt(Number.MAX_SAFE_INTEGER)) return Number(value); + return { _pythonMarshalType: "int", value: value.toString() }; + } + + /** + * @param {number} length + * @returns {Object} + */ + readByteString(length) { + return { + _pythonMarshalType: "bytes", + hex: Array.from(this.readBytes(length), byte => byte.toString(16).padStart(2, "0")).join(""), + }; + } + + /** + * @returns {Object|Array|string|number|boolean|null} + */ + readReference() { + const reference = this.readUint32(); + if (reference >= this.refs.length) throw new OperationError("Python marshal data contains an invalid reference"); + return this.refs[reference]; + } + + /** + * @param {Array} value + * @param {number} length + * @param {number} depth + */ + readSequence(value, length, depth) { + if (length > MAX_ITEMS) throw new OperationError("Python marshal collection has too many items"); + for (let i = 0; i < length; i++) value.push(this.readValue(depth + 1)); + } + + /** + * @param {Object} value + * @param {number} depth + */ + readDict(value, depth) { + for (let i = 0; i < MAX_ITEMS; i++) { + const key = this.readValue(depth + 1); + if (key === NULL) return; + if (typeof key !== "string") throw new OperationError("Python marshal dictionaries with non-string keys are not representable as JSON"); + value[key] = this.readValue(depth + 1); + } + throw new OperationError("Python marshal dictionary has too many items"); + } + +} + +class Writer { + + constructor() { + this.parts = []; + } + + /** + * @returns {ArrayBuffer} + */ + finish() { + const length = this.parts.reduce((total, part) => total + part.length, 0); + const result = new Uint8Array(length); + let offset = 0; + for (const part of this.parts) { + result.set(part, offset); + offset += part.length; + } + return result.buffer; + } + + /** + * @param {number} type + */ + writeType(type) { + this.parts.push(Uint8Array.of(type)); + } + + /** + * @param {number} value + */ + writeInt32(value) { + const bytes = new Uint8Array(4); + new DataView(bytes.buffer).setInt32(0, value, true); + this.parts.push(bytes); + } + + /** + * @param {number} value + */ + writeUint32(value) { + const bytes = new Uint8Array(4); + new DataView(bytes.buffer).setUint32(0, value, true); + this.parts.push(bytes); + } + + /** + * @param {number} value + */ + writeFloat64(value) { + const bytes = new Uint8Array(8); + new DataView(bytes.buffer).setFloat64(0, value, true); + this.parts.push(bytes); + } + + /** + * @param {Object|Array|string|number|boolean|null} value + * @param {number} depth + */ + writeValue(value, depth) { + if (depth > MAX_DEPTH) throw new OperationError("JSON input is nested too deeply"); + + if (value === null) { + this.writeType(TYPE.NONE); + } else if (typeof value === "boolean") { + this.writeType(value ? TYPE.TRUE : TYPE.FALSE); + } else if (typeof value === "string") { + this.writeText(value); + } else if (typeof value === "number") { + this.writeNumber(value); + } else if (Array.isArray(value)) { + if (value.length > MAX_ITEMS) throw new OperationError("JSON array has too many items"); + this.writeType(TYPE.LIST); + this.writeUint32(value.length); + for (const item of value) this.writeValue(item, depth + 1); + } else if (typeof value === "object") { + if (value._pythonMarshalType === "bytes") { + this.writeBytes(value); + } else if (value._pythonMarshalType === "int") { + this.writeLong(value); + } else { + const keys = Object.keys(value); + if (keys.length > MAX_ITEMS) throw new OperationError("JSON object has too many properties"); + this.writeType(TYPE.DICT); + for (const key of keys) { + this.writeText(key); + this.writeValue(value[key], depth + 1); + } + this.writeType(TYPE.NULL); + } + } else { + throw new OperationError(`Cannot encode JSON value of type '${typeof value}' as Python marshal`); + } + } + + /** + * @param {string} value + */ + writeText(value) { + const bytes = textEncoder.encode(value); + this.writeType(TYPE.UNICODE); + this.writeUint32(bytes.length); + this.parts.push(bytes); + } + + /** + * @param {number} value + */ + writeNumber(value) { + if (!Number.isFinite(value)) throw new OperationError("JSON numbers must be finite"); + if (Number.isSafeInteger(value)) { + if (value >= -0x80000000 && value <= 0x7fffffff) { + this.writeType(TYPE.INT); + this.writeInt32(value); + } else { + this.writeBigInt(BigInt(value)); + } + } else { + this.writeType(TYPE.BINARY_FLOAT); + this.writeFloat64(value); + } + } + + /** + * @param {Object} value + */ + writeBytes(value) { + if (typeof value.hex !== "string" || !/^(?:[0-9a-f]{2})*$/i.test(value.hex)) { + throw new OperationError("Python marshal bytes values require an even-length hexadecimal 'hex' string"); + } + const bytes = new Uint8Array(value.hex.length / 2); + for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(value.hex.slice(i * 2, i * 2 + 2), 16); + this.writeType(TYPE.STRING); + this.writeUint32(bytes.length); + this.parts.push(bytes); + } + + /** + * @param {Object} value + */ + writeLong(value) { + if (typeof value.value !== "string" || !/^-?\d+$/.test(value.value)) { + throw new OperationError("Python marshal integer values require a decimal string 'value'"); + } + this.writeBigInt(BigInt(value.value)); + } + + /** + * @param {bigint} value + */ + writeBigInt(value) { + let integer = value; + const negative = integer < 0n; + if (negative) integer = -integer; + const digits = []; + while (integer) { + digits.push(Number(integer & 0x7fffn)); + integer >>= 15n; + } + this.writeType(TYPE.LONG); + this.writeInt32(negative ? -digits.length : digits.length); + for (const digit of digits) this.parts.push(Uint8Array.of(digit & 0xff, digit >> 8)); + } + +} + +export default PythonMarshal; diff --git a/src/core/operations/FromPythonMarshal.mjs b/src/core/operations/FromPythonMarshal.mjs new file mode 100644 index 0000000000..7f0c6b2744 --- /dev/null +++ b/src/core/operations/FromPythonMarshal.mjs @@ -0,0 +1,41 @@ +/** + * @author GCHQ + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import PythonMarshal from "../lib/PythonMarshal.mjs"; + +/** + * From Python Marshal operation + */ +class FromPythonMarshal extends Operation { + + /** + * FromPythonMarshal constructor + */ + constructor() { + super(); + + this.name = "From Python Marshal"; + this.module = "Code"; + this.description = "Decodes CPython marshal data to JSON-compatible values. Supports null, booleans, numbers, strings, byte strings, lists, tuples, sets and dictionaries with string keys. Byte strings are represented as {"_pythonMarshalType":"bytes","hex":"..."}; integers outside JavaScript's safe range use {"_pythonMarshalType":"int","value":"..."}. Code objects and other Python-only values are not supported."; + this.infoURL = "https://docs.python.org/3/library/marshal.html"; + this.inputType = "ArrayBuffer"; + this.outputType = "JSON"; + this.args = []; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + return PythonMarshal.decode(input); + } + +} + +export default FromPythonMarshal; diff --git a/src/core/operations/ToPythonMarshal.mjs b/src/core/operations/ToPythonMarshal.mjs new file mode 100644 index 0000000000..bcdf267e36 --- /dev/null +++ b/src/core/operations/ToPythonMarshal.mjs @@ -0,0 +1,41 @@ +/** + * @author GCHQ + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import PythonMarshal from "../lib/PythonMarshal.mjs"; + +/** + * To Python Marshal operation + */ +class ToPythonMarshal extends Operation { + + /** + * ToPythonMarshal constructor + */ + constructor() { + super(); + + this.name = "To Python Marshal"; + this.module = "Code"; + this.description = "Encodes JSON-compatible values using CPython marshal format 4. Use {"_pythonMarshalType":"bytes","hex":"..."} for Python byte strings and {"_pythonMarshalType":"int","value":"..."} for arbitrary-size integers. Arrays encode as Python lists; objects encode as dictionaries with string keys."; + this.infoURL = "https://docs.python.org/3/library/marshal.html"; + this.inputType = "JSON"; + this.outputType = "ArrayBuffer"; + this.args = []; + } + + /** + * @param {JSON} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + run(input, args) { + return PythonMarshal.encode(input); + } + +} + +export default ToPythonMarshal; diff --git a/tests/operations/tests/PythonMarshal.mjs b/tests/operations/tests/PythonMarshal.mjs new file mode 100644 index 0000000000..b3aabda558 --- /dev/null +++ b/tests/operations/tests/PythonMarshal.mjs @@ -0,0 +1,103 @@ +/** + * Python marshal operation tests. + * + * @author GCHQ + * @copyright Crown Copyright 2026 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +/* + * The decode fixtures below were generated with CPython 3.14.4. Reproduce them + * with the following script (the explicit version keeps the fixtures stable as + * CPython's default marshal format evolves): + * + * import marshal + * + * for value in [ + * {"hello": "world", "data": b"\x00\xff"}, + * [1, "two"], + * ]: + * print(repr(marshal.dumps(value, 4))) + * + * all_types = { + * "none": None, "false": False, "true": True, "int32": -42, + * "integer": 9007199254740992, "float": 3.5, "bytes": b"\x00\xff", + * "text": "Γειά", "list": [1, "two"], "tuple": (1, "two"), + * "set": {1, 2}, "dictionary": {"nested": "value"}, + * } + * print(marshal.dumps(all_types, 4).hex()) + * + * The encoder fixture is validated by CPython with: + * + * assert marshal.loads(bytes.fromhex( + * "7b750500000068656c6c6f7505000000776f726c64750400000064617461730200000000ff30" + * )) == {"hello": "world", "data": b"\x00\xff"} + * + * The test suite deliberately embeds these CPython-generated fixtures instead + * of invoking Python, so `npm test` remains a Node.js-only command. + */ +const hexToString = (hex) => hex.match(/../g).map((byte) => + String.fromCharCode(parseInt(byte, 16))).join(""); + +TestRegister.addTests([ + { + name: "From Python Marshal: CPython format 4 dictionary", + input: "\xfb\xda\x05hello\xda\x05world\xda\x04data\xf3\x02\x00\x00\x00\x00\xff\x30", + expectedOutput: "{\n \"hello\": \"world\",\n \"data\": {\n \"_pythonMarshalType\": \"bytes\",\n \"hex\": \"00ff\"\n }\n}", + recipeConfig: [ + { + op: "From Python Marshal", + args: [], + }, + ], + }, + { + name: "From Python Marshal: CPython format 4 references", + input: "\xdb\x02\x00\x00\x00\xe9\x01\x00\x00\x00\xda\x03two", + expectedOutput: "[\n 1,\n \"two\"\n]", + recipeConfig: [ + { + op: "From Python Marshal", + args: [], + }, + ], + }, + { + name: "From Python Marshal: documented value types", + input: hexToString("fbda046e6f6e654eda0566616c736546da047472756554da05696e743332e9d6ffffffda07696e7465676572ec040000000000000000000001da05666c6f6174e70000000000000c40da056279746573f30200000000ffda0474657874f508000000ce93ceb5ceb9ceacda046c6973745b02000000e901000000da0374776fda057475706c65a902720f0000007210000000da037365743c02000000720f000000e902000000da0a64696374696f6e6172797bda066e6573746564da0576616c75653030"), + expectedOutput: "{\n \"none\": null,\n \"false\": false,\n \"true\": true,\n \"int32\": -42,\n \"integer\": {\n \"_pythonMarshalType\": \"int\",\n \"value\": \"9007199254740992\"\n },\n \"float\": 3.5,\n \"bytes\": {\n \"_pythonMarshalType\": \"bytes\",\n \"hex\": \"00ff\"\n },\n \"text\": \"Γειά\",\n \"list\": [\n 1,\n \"two\"\n ],\n \"tuple\": [\n 1,\n \"two\"\n ],\n \"set\": [\n 1,\n 2\n ],\n \"dictionary\": {\n \"nested\": \"value\"\n }\n}", + recipeConfig: [ + { + op: "From Python Marshal", + args: [], + }, + ], + }, + { + name: "To Python Marshal: CPython format 4 dictionary", + input: "{\"hello\":\"world\",\"data\":{\"_pythonMarshalType\":\"bytes\",\"hex\":\"00ff\"}}", + expectedOutput: "{u\x05\x00\x00\x00hellou\x05\x00\x00\x00worldu\x04\x00\x00\x00datas\x02\x00\x00\x00\x00\xff0", + recipeConfig: [ + { + op: "To Python Marshal", + args: [], + }, + ], + }, + { + name: "Python Marshal: arbitrary-size integer round trip", + input: "{\"_pythonMarshalType\":\"int\",\"value\":\"9007199254740992\"}", + expectedOutput: "{\n \"_pythonMarshalType\": \"int\",\n \"value\": \"9007199254740992\"\n}", + recipeConfig: [ + { + op: "To Python Marshal", + args: [], + }, + { + op: "From Python Marshal", + args: [], + }, + ], + }, +]);