diff --git a/Dockerfile b/Dockerfile index cef74738100cb..483f1b557048e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -293,7 +293,7 @@ RUN --mount=type=tmpfs,target=/webkitbuild \ -G Ninja \ /webkit && \ cd /webkitbuild && \ - cmake --build /webkitbuild --config $WEBKIT_RELEASE_TYPE --target "jsc" && \ + cmake --build /webkitbuild --config $WEBKIT_RELEASE_TYPE --target "jsc" --target "testFFI" && \ python3 /webkit/Tools/Scripts/check-classinfo-uniqueness.py $WEBKIT_OUT_DIR/bin/jsc && \ cp -r $WEBKIT_OUT_DIR/lib/*.a /output/lib && \ cp $WEBKIT_OUT_DIR/*.h /output/include && \ diff --git a/Dockerfile.android b/Dockerfile.android index ab3dc3fbe0275..a8e4969f21c68 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -196,10 +196,11 @@ RUN --mount=type=tmpfs,target=/webkitbuild \ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH \ -G Ninja \ /webkit && \ - cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc && \ + cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc --target testFFI && \ python3 /webkit/Tools/Scripts/check-classinfo-uniqueness.py $WEBKIT_OUT_DIR/bin/jsc && \ cp -r $WEBKIT_OUT_DIR/lib/*.a /output/lib && \ cp $WEBKIT_OUT_DIR/*.h /output/include && \ + cp -r $WEBKIT_OUT_DIR/bin /output/bin && \ cp $WEBKIT_OUT_DIR/*.json /output && \ find $WEBKIT_OUT_DIR/JavaScriptCore/DerivedSources/ -name "*.h" -exec sh -c 'cp "$1" "/output/include/JavaScriptCore/$(basename "$1")"' sh {} \; && \ find $WEBKIT_OUT_DIR/JavaScriptCore/DerivedSources/ -name "*.json" -exec sh -c 'cp "$1" "/output/$(basename "$1")"' sh {} \; && \ diff --git a/Dockerfile.freebsd b/Dockerfile.freebsd index 4dd208a318f5e..283e4ab9892b7 100644 --- a/Dockerfile.freebsd +++ b/Dockerfile.freebsd @@ -182,10 +182,11 @@ RUN --mount=type=tmpfs,target=/webkitbuild \ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH \ -G Ninja \ /webkit && \ - cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc && \ + cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc --target testFFI && \ python3 /webkit/Tools/Scripts/check-classinfo-uniqueness.py $WEBKIT_OUT_DIR/bin/jsc && \ cp -r $WEBKIT_OUT_DIR/lib/*.a /output/lib && \ cp $WEBKIT_OUT_DIR/*.h /output/include && \ + cp -r $WEBKIT_OUT_DIR/bin /output/bin && \ cp $WEBKIT_OUT_DIR/*.json /output && \ find $WEBKIT_OUT_DIR/JavaScriptCore/DerivedSources/ -name "*.h" -exec sh -c 'cp "$1" "/output/include/JavaScriptCore/$(basename "$1")"' sh {} \; && \ find $WEBKIT_OUT_DIR/JavaScriptCore/DerivedSources/ -name "*.json" -exec sh -c 'cp "$1" "/output/$(basename "$1")"' sh {} \; && \ diff --git a/Dockerfile.macos b/Dockerfile.macos index dc2499ab9c666..3a97421351472 100644 --- a/Dockerfile.macos +++ b/Dockerfile.macos @@ -220,7 +220,7 @@ RUN --mount=type=tmpfs,target=/webkitbuild \ -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ -G Ninja \ /webkit && \ - cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc && \ + cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc --target testFFI && \ python3 /webkit/Tools/Scripts/check-classinfo-uniqueness.py $WEBKIT_OUT_DIR/bin/jsc && \ # -L everywhere: WebKit's cmake stages the WTF/bmalloc framework headers # as *symlinks* into the source tree when the host filesystem supports diff --git a/Dockerfile.musl b/Dockerfile.musl index 5b064cb5bfdae..3a1b52acc5d54 100644 --- a/Dockerfile.musl +++ b/Dockerfile.musl @@ -139,7 +139,7 @@ RUN --mount=type=tmpfs,target=/webkitbuild \ -G Ninja \ /webkit && \ cd /webkitbuild && \ - cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target "jsc" && \ + cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target "jsc" --target "testFFI" && \ python3 /webkit/Tools/Scripts/check-classinfo-uniqueness.py $WEBKIT_OUT_DIR/bin/jsc && \ cp -r $WEBKIT_OUT_DIR/lib/*.a /output/lib && \ cp $WEBKIT_OUT_DIR/*.h /output/include && \ diff --git a/Dockerfile.windows b/Dockerfile.windows index 8a69622f38f20..a86dec861827e 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -393,7 +393,7 @@ RUN --mount=type=tmpfs,target=/webkitbuild \ "-DCMAKE_CXX_FLAGS_DEBUG=/FS /O0 /Ob0" \ -G Ninja \ /webkit && \ - cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc && \ + cmake --build /webkitbuild --config ${WEBKIT_RELEASE_TYPE} --target jsc --target testFFI && \ \ mkdir -p /output/lib /output/include/JavaScriptCore /output/include/wtf /output/include/bmalloc && \ cp -rL $WEBKIT_OUT_DIR/lib/* /output/lib/ && \ diff --git a/JSTests/stress/ffi-align.js b/JSTests/stress/ffi-align.js new file mode 100644 index 0000000000000..643c105c4d801 --- /dev/null +++ b/JSTests/stress/ffi-align.js @@ -0,0 +1,69 @@ +//@ requireOptions("--useDollarVM=1") + +// Stack-alignment probes (SPEC section 11.1): each fixture performs an +// aligned 16-byte vector access on a 16-byte-aligned local, which faults if +// the FFI caller (host path, IC stub, DFG/FTL CallFFI, or the callback thunk +// on the way back into native code) mis-aligned the stack. Both probes must +// return exactly 1.0 in every tier. + +function main() { + const fixture = name => $vm.ffiFixture(name); + const probe0 = $vm.ffiFunction({ args: [], returns: "f64" }, fixture("ffi_align_probe_0"), "ffi_align_probe_0"); + const probe9 = $vm.ffiFunction({ args: new Array(9).fill("i32"), returns: "f64" }, fixture("ffi_align_probe_9"), "ffi_align_probe_9"); + const callCbVoid = $vm.ffiFunction({ args: ["function"], returns: "void" }, fixture("ffi_call_cb_void"), "ffi_call_cb_void"); + const callCbI32 = $vm.ffiFunction({ args: ["function", "i32"], returns: "i32" }, fixture("ffi_call_cb_i32"), "ffi_call_cb_i32"); + + if (probe0() !== 1) + throw new Error("ffi_align_probe_0 cold: " + probe0()); + if (probe9(1, 2, 3, 4, 5, 6, 7, 8, 9) !== 1) + throw new Error("ffi_align_probe_9 cold"); + // Missing / extra JS arguments must not change the call frame layout. + if (probe9(1, 2, 3) !== 1) + throw new Error("ffi_align_probe_9 with missing arguments"); + if (probe9(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) !== 1) + throw new Error("ffi_align_probe_9 with extra arguments"); + if (probe0(1) !== 1) + throw new Error("ffi_align_probe_0 with an extra argument"); + + // Hot: every tier's call path must keep 16-byte alignment. + for (let i = 0; i < 3e4; ++i) { + if (probe0() !== 1) + throw new Error("ffi_align_probe_0 hot iteration " + i); + } + for (let i = 0; i < 3e4; ++i) { + if (probe9(i, -i, i, -i, i, -i, i, -i, i) !== 1) + throw new Error("ffi_align_probe_9 hot iteration " + i); + } + // Mixed argument shapes (int32 and double) at the same call site. + for (let i = 0; i < 1e4; ++i) { + if (probe9(i + 0.5, 1, 2, 3, 4, 5, 6, 7, 8) !== 1) + throw new Error("ffi_align_probe_9 double first argument iteration " + i); + } + + // Alignment on the way back out: a callback that runs the probes from + // inside the native -> JS -> native sandwich. + const cbProbe = $vm.ffiCallback({ args: [], returns: "void" }, () => { + if (probe0() !== 1) + throw new Error("probe0 inside callback"); + if (probe9(9, 8, 7, 6, 5, 4, 3, 2, 1) !== 1) + throw new Error("probe9 inside callback"); + }); + for (let i = 0; i < 3000; ++i) + callCbVoid(cbProbe); + + // Nested: FFI -> callback -> FFI -> callback -> probe, to depth 20. + const nestCb = $vm.ffiCallback({ args: ["i32"], returns: "i32" }, depth => { + if (probe0() !== 1) + throw new Error("probe0 at depth " + depth); + if (depth <= 0) + return 0; + return callCbI32(nestCb, depth - 1) + 1; + }); + for (let i = 0; i < 200; ++i) { + if (callCbI32(nestCb, 20) !== 20) + throw new Error("nested alignment ladder iteration " + i); + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-arena-depth.js b/JSTests/stress/ffi-arena-depth.js new file mode 100644 index 0000000000000..07e695f9cdf4a --- /dev/null +++ b/JSTests/stress/ffi-arena-depth.js @@ -0,0 +1,41 @@ +//@ requireOptions("--useDollarVM=1") + +function main() { + const fixture = name => $vm.ffiFixture(name); + const depth = () => $vm.ffiArenaDepth(); + + if (depth() !== 0) + throw new Error("arena depth should start at 0, got " + depth()); + + const strlen = $vm.ffiFunction({ args: ["cstring"], returns: "u64" }, fixture("ffi_strlen"), "ffi_strlen"); + for (let i = 0; i < 2e4; ++i) { + strlen("call " + (i & 7)); + if (depth() !== 0) + throw new Error("arena depth leaked after a normal call at iteration " + i + ": " + depth()); + } + + let calls = 0; + const callback = $vm.ffiCallback({ args: [], returns: "cstring" }, () => { + ++calls; + throw new Error("thrown from callback"); + }); + const callThrough = $vm.ffiFunction({ args: ["ptr"], returns: "cstring" }, fixture("ffi_call_cb_ret_cstring"), "ffi_call_cb_ret_cstring"); + for (let i = 0; i < 2e4; ++i) { + let threw = false; + try { + callThrough(callback.ptr); + } catch (e) { + threw = e instanceof Error && e.message === "thrown from callback"; + } + if (!threw) + throw new Error("expected the callback exception to propagate at iteration " + i); + if (depth() !== 0) + throw new Error("arena depth leaked after a throwing cstring call at iteration " + i + ": " + depth()); + } + callback.close(); + if (calls !== 2e4) + throw new Error("callback ran " + calls + " times, expected 20000"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-arity-ladders.js b/JSTests/stress/ffi-arity-ladders.js new file mode 100644 index 0000000000000..7c00f349aa60f --- /dev/null +++ b/JSTests/stress/ffi-arity-ladders.js @@ -0,0 +1,208 @@ +//@ requireOptions("--useDollarVM=1") + +// Arity ladders that straddle every register->stack boundary of the +// supported ABIs, plus the interleaved ffi_mix_* fixtures. Each mix returns +// the position-weighted checksum sum((k + 1) * arg_k), so any argument that +// lands in the wrong register or stack slot changes the result. + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + String(expected) + " but got " + String(actual)); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (name, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(name), name); + + // Deterministic PRNG (mulberry32). + let seed = 0x1abe11ed; + function random() { + seed = (seed + 0x6D2B79F5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + } + const randomInt32 = () => (Math.floor(random() * 4294967296) | 0); + const randomSmall = () => Math.floor(random() * 2001) - 1000; + + // ---- ffi_sum_i32_: int64 sums returned as BigInt. + const sumI32Arities = [0, 1, 2, 4, 6, 7, 8, 9, 12, 16]; + const sumI32 = new Map(); + for (const n of sumI32Arities) + sumI32.set(n, bind("ffi_sum_i32_" + n, new Array(n).fill("i32"), "i64")); + + function checkSumI32(n, values) { + let expected = 0n; + for (const v of values) + expected += BigInt(v | 0); + const actual = sumI32.get(n)(...values); + check(actual, expected, "ffi_sum_i32_" + n + "(" + values.join(",") + ")"); + } + for (const n of sumI32Arities) { + checkSumI32(n, new Array(n).fill(0)); + checkSumI32(n, new Array(n).fill(-1)); + checkSumI32(n, new Array(n).fill(2147483647)); + checkSumI32(n, new Array(n).fill(-2147483648)); + // Distinct powers of two: catches any duplicated or swapped argument. + checkSumI32(n, new Array(n).fill(0).map((_, i) => (i % 2 ? -1 : 1) * (1 << (i + 5)))); + for (let iteration = 0; iteration < 30; ++iteration) + checkSumI32(n, new Array(n).fill(0).map(() => randomInt32())); + } + + // ---- ffi_sum_f64_: double sums. + const sumF64Arities = [1, 2, 7, 8, 9, 12]; + const sumF64 = new Map(); + for (const n of sumF64Arities) + sumF64.set(n, bind("ffi_sum_f64_" + n, new Array(n).fill("f64"), "f64")); + function checkSumF64(n, values) { + let expected = 0; + for (const v of values) + expected += v; // same left-to-right order as the fixture + const actual = sumF64.get(n)(...values); + check(actual, expected, "ffi_sum_f64_" + n + "(" + values.join(",") + ")"); + } + for (const n of sumF64Arities) { + checkSumF64(n, new Array(n).fill(0)); + checkSumF64(n, new Array(n).fill(-0.5)); + checkSumF64(n, new Array(n).fill(0).map((_, i) => 1 / (1 << i))); // exact binary fractions + checkSumF64(n, new Array(n).fill(0).map((_, i) => (i % 2 ? -1 : 1) * 2 ** (i * 4))); + for (let iteration = 0; iteration < 30; ++iteration) + checkSumF64(n, new Array(n).fill(0).map(() => randomSmall() * 2 ** (Math.floor(random() * 60) - 30))); + } + + // ---- Sub-8-byte stack ladders (Apple arm64 packing). + const sumU8_10 = bind("ffi_sum_u8_10", new Array(10).fill("u8"), "i64"); + const sumU8_12 = bind("ffi_sum_u8_12", new Array(12).fill("u8"), "i64"); + const sumI16_10 = bind("ffi_sum_i16_10", new Array(10).fill("i16"), "i64"); + const sumI16_12 = bind("ffi_sum_i16_12", new Array(12).fill("i16"), "i64"); + function checkSubword(fn, name, values, widthMask, signed) { + let expected = 0n; + for (const v of values) { + let w = (v | 0) & widthMask; + if (signed && (w & ((widthMask + 1) >>> 1))) + w -= widthMask + 1; + expected += BigInt(w); + } + check(fn(...values), expected, name + "(" + values.join(",") + ")"); + } + for (const [fn, name, n, mask, signed] of [ + [sumU8_10, "ffi_sum_u8_10", 10, 0xff, false], + [sumU8_12, "ffi_sum_u8_12", 12, 0xff, false], + [sumI16_10, "ffi_sum_i16_10", 10, 0xffff, true], + [sumI16_12, "ffi_sum_i16_12", 12, 0xffff, true], + ]) { + checkSubword(fn, name, new Array(n).fill(0), mask, signed); + checkSubword(fn, name, new Array(n).fill(-1), mask, signed); // 255 / -1 + checkSubword(fn, name, new Array(n).fill(0).map((_, i) => 1 << i), mask, signed); // distinct powers of two + checkSubword(fn, name, new Array(n).fill(0).map((_, i) => i + 1), mask, signed); + checkSubword(fn, name, new Array(n).fill(mask), mask, signed); + checkSubword(fn, name, new Array(n).fill((mask + 1) >>> 1), mask, signed); // sign bit + for (let iteration = 0; iteration < 30; ++iteration) + checkSubword(fn, name, new Array(n).fill(0).map(() => randomInt32()), mask, signed); + } + + // ---- Mixes. checksum = sum (k + 1) * cast(arg_k) + const mix1 = bind("ffi_mix_1", ["i32", "f64", "i64", "f32", "ptr", "u8", "f64", "i16", "f64", "i32"], "f64"); + const mix2 = bind("ffi_mix_2", ["f32", "i32", "f32", "i32", "f32", "i32", "f32", "i32", "f32", "i32"], "f64"); + const mix3 = bind("ffi_mix_3", ["f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "i32"], "f64"); + const mix4 = bind("ffi_mix_4", ["i64", "i64", "i64", "i64", "i64", "i64", "f64", "i64", "f64"], "f64"); + const mix5 = bind("ffi_mix_5", ["u8", "i8", "u16", "i16", "u32", "i32", "u64", "i64"], "f64"); + const mix6 = bind("ffi_mix_6", ["bool", "bool", "i32", "bool", "f64", "bool", "f32", "bool", "bool", "bool", "bool", "bool", "bool"], "f64"); + const mix7 = bind("ffi_mix_7", ["ptr", "char", "ptr", "char", "ptr", "char", "ptr", "char", "ptr", "char"], "f64"); + const mix8 = bind("ffi_mix_8", ["f32", "f64", "f32", "f64", "f32", "f64", "f32", "f64", "f32", "f64", "f32", "f64"], "f64"); + + // JS reference of the C casts used by the fixtures. + const castByType = { + "i32": v => v | 0, + "f64": v => +v, + "i64": v => Number(BigInt.asIntN(64, BigInt(Math.trunc(v)))), + "f32": v => Math.fround(v), + "ptr": v => Math.trunc(v), // small non-negative pointers only + "u8": v => (v | 0) & 0xff, + "i16": v => ((v | 0) << 16) >> 16, + "i8": v => ((v | 0) << 24) >> 24, + "char": v => ((v | 0) << 24) >> 24, + "u16": v => (v | 0) & 0xffff, + "u32": v => (v | 0) >>> 0, + "u64": v => Number(BigInt.asUintN(64, BigInt(Math.trunc(v)))), + "bool": v => (v ? 1 : 0), + }; + function checksum(types, values) { + let sum = 0; + for (let k = 0; k < types.length; ++k) + sum += (k + 1) * castByType[types[k]](values[k]); + return sum; + } + const mixes = [ + [mix1, "ffi_mix_1", ["i32", "f64", "i64", "f32", "ptr", "u8", "f64", "i16", "f64", "i32"]], + [mix2, "ffi_mix_2", ["f32", "i32", "f32", "i32", "f32", "i32", "f32", "i32", "f32", "i32"]], + [mix3, "ffi_mix_3", ["f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "i32"]], + [mix4, "ffi_mix_4", ["i64", "i64", "i64", "i64", "i64", "i64", "f64", "i64", "f64"]], + [mix5, "ffi_mix_5", ["u8", "i8", "u16", "i16", "u32", "i32", "u64", "i64"]], + [mix6, "ffi_mix_6", ["bool", "bool", "i32", "bool", "f64", "bool", "f32", "bool", "bool", "bool", "bool", "bool", "bool"]], + [mix7, "ffi_mix_7", ["ptr", "char", "ptr", "char", "ptr", "char", "ptr", "char", "ptr", "char"]], + [mix8, "ffi_mix_8", ["f32", "f64", "f32", "f64", "f32", "f64", "f32", "f64", "f32", "f64", "f32", "f64"]], + ]; + // Value generators per type. All chosen so that the checksum arithmetic + // is exact in double (weights <= 13, magnitudes <= 2^40). + const generatorByType = { + "i32": () => randomSmall() * 65536 + Math.floor(random() * 65536), + "f64": () => randomSmall() / 8, + "i64": () => randomSmall() * 1048576, + "f32": () => Math.fround(randomSmall() / 16), + "ptr": () => Math.floor(random() * 65536) * 8, + "u8": () => Math.floor(random() * 512) - 128, + "i16": () => Math.floor(random() * 200000) - 100000, + "i8": () => Math.floor(random() * 512) - 256, + "char": () => Math.floor(random() * 512) - 256, + "u16": () => Math.floor(random() * 200000) - 100000, + "u32": () => Math.floor(random() * 4294967296) - 2147483648, + "u64": () => Math.floor(random() * 65536), + "bool": () => [0, 1, 2, -1, 0.5, 0, 1][Math.floor(random() * 7)], + }; + for (const [fn, name, types] of mixes) { + // Distinct-position probe: 1 at each position in turn. + for (let k = 0; k < types.length; ++k) { + const values = types.map((_, i) => (i === k ? 1 : 0)); + check(fn(...values), checksum(types, values), name + " unit vector at " + k); + } + // All-ones and per-type extremes. + check(fn(...types.map(() => 1)), checksum(types, types.map(() => 1)), name + " all ones"); + for (let iteration = 0; iteration < 200; ++iteration) { + const values = types.map(t => generatorByType[t]()); + check(fn(...values), checksum(types, values), name + " random iteration " + iteration + " (" + values.join(",") + ")"); + } + } + + // ---- Hot loops so the ladders are also driven through the JIT tiers. + // Every hot call site below is exact-arity, monomorphic and non-spread so + // it can become a typed CallFFI node (SPEC section 10.2); a spread call + // (CallVarargs) is never converted. + const nine = sumI32.get(9); + for (let i = 0; i < 4e4; ++i) { + const r = nine(1, -2, 3, -4, 5, -6, 7, -8, 100000); + if (r !== 99996n) + throw new Error("ffi_sum_i32_9 hot iteration " + i + " got " + r); + } + // 2^52 keeps every weighted product exactly representable, so FMA + // contraction inside the C fixture cannot change the result. + const mixValues = [7, 1.5, 4503599627370496, 2.5, 4096, 250, -3.25, -1234, 8.75, -99]; + const mixTypes = mixes[0][2]; + const mixExpected = checksum(mixTypes, mixValues); + for (let i = 0; i < 4e4; ++i) { + const r = mix1(7, 1.5, 4503599627370496, 2.5, 4096, 250, -3.25, -1234, 8.75, -99); + if (r !== mixExpected) + throw new Error("ffi_mix_1 hot iteration " + i + " got " + r + " expected " + mixExpected); + } + const mix8Values = mixes[7][2].map((t, i) => (t === "f32" ? Math.fround(i + 0.5) : -(i + 0.25))); + const mix8Expected = checksum(mixes[7][2], mix8Values); + const [m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11] = mix8Values; + for (let i = 0; i < 4e4; ++i) { + const r = mix8(m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11); + if (r !== mix8Expected) + throw new Error("ffi_mix_8 hot iteration " + i + " got " + r + " expected " + mix8Expected); + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-arity.js b/JSTests/stress/ffi-arity.js new file mode 100644 index 0000000000000..18849196c2318 --- /dev/null +++ b/JSTests/stress/ffi-arity.js @@ -0,0 +1,144 @@ +//@ requireOptions("--useDollarVM=1") + +// Arity handling (SPEC sections 3, 8.1, 8.2, 10.2): missing JS arguments are +// undefined (per-type undefined rules), extra arguments are ignored, and +// non-exact-arity call sites are simply not converted to CallFFI. Also the +// JSFunction surface: length, name, callability protocols, non-constructor. + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (Object.is(value, -0)) + return "-0"; + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const addI32 = $vm.ffiFunction({ args: ["i32", "i32"], returns: "i32" }, fixture("ffi_add_i32"), "ffi_add_i32"); + const addF64 = $vm.ffiFunction({ args: ["f64", "f64"], returns: "f64" }, fixture("ffi_add_f64"), "ffi_add_f64"); + const addF32 = $vm.ffiFunction({ args: ["f32", "f32"], returns: "f32" }, fixture("ffi_add_f32"), "ffi_add_f32"); + const echoBool = $vm.ffiFunction({ args: ["bool"], returns: "bool" }, fixture("ffi_echo_bool"), "ffi_echo_bool"); + const echoPtr = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, fixture("ffi_echo_ptr"), "ffi_echo_ptr"); + const echoI64 = $vm.ffiFunction({ args: ["i64"], returns: "i64" }, fixture("ffi_echo_i64"), "ffi_echo_i64"); + const sum4 = $vm.ffiFunction({ args: ["i32", "i32", "i32", "i32"], returns: "i64" }, fixture("ffi_sum_i32_4"), "ffi_sum_i32_4"); + const sum0 = $vm.ffiFunction({ args: [], returns: "i64" }, fixture("ffi_sum_i32_0"), "ffi_sum_i32_0"); + + // ---- JSFunction surface. + check(addI32.length, 2, "length"); + check(sum4.length, 4, "length of ffi_sum_i32_4"); + check(sum0.length, 0, "length of ffi_sum_i32_0"); + check(addI32.name, "ffi_add_i32", "name"); + check(typeof addI32, "function", "typeof"); + check(addI32 instanceof Function, true, "instanceof Function"); + check(Object.getPrototypeOf(addI32), Function.prototype, "prototype is Function.prototype"); + let constructThrew = false; + try { + new addI32(1, 2); + } catch (e) { + constructThrew = e instanceof TypeError; + } + check(constructThrew, true, "new on an FFI function throws TypeError"); + let reflectConstructThrew = false; + try { + Reflect.construct(addI32, [1, 2]); + } catch (e) { + reflectConstructThrew = e instanceof TypeError; + } + check(reflectConstructThrew, true, "Reflect.construct on an FFI function throws TypeError"); + + // ---- Missing arguments: undefined semantics per type. + check(addI32(), 0, "add_i32()"); + check(addI32(5), 5, "add_i32(5)"); + check(addI32(undefined, undefined), 0, "add_i32(undefined, undefined)"); + check(addF64(), NaN, "add_f64() -> NaN + NaN (missing f64 args are undefined -> NaN)"); + check(addF64(1), NaN, "add_f64(1) -> 1 + NaN"); + check(echoBool(), false, "echo_bool()"); + check(echoPtr(), null, "echo_ptr() -> null pointer"); + check(sum4(1, 2), 3n, "sum_i32_4 with two arguments"); + check(sum4(), 0n, "sum_i32_4 with no arguments"); + // i64 does NOT accept undefined (SPEC section 5): missing i64 arguments throw. + let i64Threw = false; + try { + echoI64(); + } catch (e) { + i64Threw = e instanceof TypeError; + } + check(i64Threw, true, "echo_i64() with a missing i64 argument throws TypeError"); + // f32 follows the same loose rule as f64: a missing argument is undefined -> NaN. + check(addF32(1.5), NaN, "add_f32(1.5) with a missing f32 argument is 1.5 + NaN"); + + // ---- Extra arguments are ignored. + check(addI32(1, 2, 3), 3, "add_i32(1,2,3)"); + check(addI32(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), 3, "add_i32 with 12 arguments"); + check(sum0(1, 2, 3), 0n, "sum_i32_0 with extra arguments"); + check(echoBool(true, Symbol("ignored"), {}), true, "extra arguments of unconvertible types are ignored"); + check(addI32(1, 2, Symbol("ignored")), 3, "extra symbol argument is ignored"); + + // ---- Call protocols. + check(addI32.call(undefined, 40, 2), 42, "call"); + check(addI32.call(null, 40, 2, 99), 42, "call with extra"); + check(addI32.call({}, 40), 40, "call with this and a missing argument"); + check(addI32.apply(undefined, [40, 2]), 42, "apply"); + check(addI32.apply(undefined, [40]), 40, "apply short"); + check(addI32.apply(undefined, [40, 2, 3, 4]), 42, "apply long"); + check(addI32.apply(undefined), 0, "apply without a list"); + check(addI32(...[40, 2]), 42, "spread"); + check(addI32(...[40]), 40, "spread short"); + check(addI32(...new Array(30).fill(1)), 2, "spread of 30 ones"); + const bound = addI32.bind(null, 40); + check(bound(2), 42, "bound one argument"); + check(bound(), 40, "bound with a missing argument"); + check(bound(2, 3), 42, "bound with an extra argument"); + check(Reflect.apply(addI32, undefined, [40, 2]), 42, "Reflect.apply"); + check([[1, 2], [3, 4], [5, 6]].map(pair => addI32(...pair)).join(","), "3,7,11", "used in map"); + check(Array.from([[1, 2], [3, 4]], ([a, b]) => addI32(a, b)).join(","), "3,7", "used in Array.from"); + // Passing the FFI function itself as a callback to a builtin. + check([1, 2, 3].reduce(addI32), 6, "reduce with the FFI function directly (extra index/array arguments ignored)"); + + // ---- Hot exact-arity vs hot non-exact-arity call sites. + function exact(a, b) { return addI32(a, b); } + function missingOne(a) { return addI32(a); } + function extraOne(a, b, c) { return addI32(a, b, c); } + function viaCall(a, b) { return addI32.call(undefined, a, b); } + function viaApply(a, b) { return addI32.apply(undefined, [a, b]); } + function viaSpread(pair) { return addI32(...pair); } + noInline(exact); noInline(missingOne); noInline(extraOne); noInline(viaCall); noInline(viaApply); noInline(viaSpread); + for (let i = 0; i < 3e4; ++i) { + check(exact(i, 1), (i + 1) | 0, "hot exact"); + check(missingOne(i), i | 0, "hot missing one"); + check(extraOne(i, 2, 999), (i + 2) | 0, "hot extra one"); + check(viaCall(i, 3), (i + 3) | 0, "hot via call"); + check(viaApply(i, 4), (i + 4) | 0, "hot via apply"); + check(viaSpread([i, 5]), (i + 5) | 0, "hot via spread"); + } + // After tier-up, the same sites with the "wrong" number of arguments. + check(exact(1), 1, "exact site called with one argument after tier-up"); + check(exact(1, 2, 3), 3, "exact site called with three arguments after tier-up"); + // missingOne(a) forwards only `a`; its extra argument (2) never reaches the + // inner one-argument FFI call site, so this is still addI32(1, undefined) = 1 + 0. + check(missingOne(1, 2), 1, "missingOne site called with two arguments"); + check(extraOne(1), 1, "extraOne site called with one argument"); + + // A varargs wrapper (arity unknown at the site). + function varargs(...args) { return addI32(...args); } + noInline(varargs); + for (let i = 0; i < 2e4; ++i) + check(varargs(i, i), (i + i) | 0, "hot varargs"); + check(varargs(), 0, "varargs()"); + check(varargs(7), 7, "varargs(7)"); + check(varargs(1, 2, 3), 3, "varargs(1,2,3)"); + + // arguments object interplay. + function withArguments() { return addI32.apply(null, arguments); } + check(withArguments(9, 10), 19, "arguments object apply"); + check(withArguments(9), 9, "arguments object apply short"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-buffer-length.js b/JSTests/stress/ffi-buffer-length.js new file mode 100644 index 0000000000000..b2587629aadda --- /dev/null +++ b/JSTests/stress/ffi-buffer-length.js @@ -0,0 +1,138 @@ +//@ requireOptions("--useDollarVM=1") +// FFI Type::BufferLength ("buffer_length"): the length twin of "buffer". Given a +// TypedArray / DataView argument it marshals the view's byteLength() as an unsigned 64-bit +// integer. Bound as args: ["ptr", "buffer_length"] with the SAME view passed for both, the +// engine reads pointer and length off one cell at call time. Argument-only; accepts exactly +// what "buffer" accepts (a view) and throws a TypeError for anything else. Every tier converts +// buffer_length through the C++ path, so a hot function and its noDFG-pinned reference twin +// must agree on every iteration. +if (!$vm.useJIT()) quit(); + +const fixture = name => $vm.ffiFixture(name); +const byteLength = $vm.ffiFunction({ args: ["ptr", "buffer_length"], returns: "u64" }, fixture("ffi_view_byte_length"), "ffi_view_byte_length"); +const byteLengthAlias = $vm.ffiFunction({ args: ["ptr", "buffer_bytelength"], returns: "u64" }, fixture("ffi_view_byte_length"), "ffi_view_byte_length(alias)"); +const lastByte = $vm.ffiFunction({ args: ["ptr", "buffer_length"], returns: "i32" }, fixture("ffi_view_last_byte"), "ffi_view_last_byte"); + +// Oracle twins pinned below the DFG: their (out-of-line C++ conversion) answer is what the +// tiered-up caller must reproduce exactly. +function refByteLength(v) { return byteLength(v, v); } +function refLastByte(v) { return lastByte(v, v); } +noDFG(refByteLength); noDFG(refLastByte); +noInline(refByteLength); noInline(refLastByte); + +function hotByteLength(v) { return byteLength(v, v); } +function hotLastByte(v) { return lastByte(v, v); } +noInline(hotByteLength); noInline(hotLastByte); + +let failures = 0; +function check(actual, expected, label) { + if (actual !== expected) { + print(`FAIL [${label}]: got ${String(actual)} (${typeof actual}), expected ${String(expected)} (${typeof expected})`); + if (++failures > 8) throw new Error("too many failures"); + } +} +function agree(label, hot, ref) { + if (hot !== ref) { + print(`TIER MISMATCH [${label}]: hot=${String(hot)} ref=${String(ref)}`); + if (++failures > 8) throw new Error("too many tier mismatches"); + } +} + +// --------------------------------------------------------------------------------------------- +// 1. The marshalled length is the view's byteLength: several sizes, a DataView, a subarray +// with a byteOffset, and the buffer_bytelength alias spelling. +// --------------------------------------------------------------------------------------------- +for (const size of [0, 1, 4, 4096]) { + const view = new Uint8Array(size); + check(refByteLength(view), BigInt(view.byteLength), `Uint8Array(${size}) byteLength`); + check(byteLengthAlias(view, view), BigInt(view.byteLength), `Uint8Array(${size}) via buffer_bytelength alias`); +} +{ + const backing = new ArrayBuffer(256); + const dataView = new DataView(backing, 32, 96); + check(refByteLength(dataView), 96n, "DataView(32, 96) byteLength"); + const wide = new Float64Array(backing, 64, 10); // byteLength is in BYTES, not elements + check(refByteLength(wide), 80n, "Float64Array(64, 10) byteLength"); + const sub = new Uint8Array(backing).subarray(100, 150); + check(refByteLength(sub), 50n, "subarray(100, 150) byteLength"); + // Pointer + length come off the same cell: the last byte through (ptr, byteLength) is the + // subarray's own last byte, not the backing store's. + sub[sub.length - 1] = 0x5a; + check(refLastByte(sub), 0x5a, "subarray pointer+length agree"); + check(refLastByte(new Uint8Array(0)), -1, "empty view last byte"); +} + +// --------------------------------------------------------------------------------------------- +// 2. Anything that is not a view throws a TypeError -- numbers included (unlike ptr, which +// accepts them). Identical message from a cold and a warmed caller. +// --------------------------------------------------------------------------------------------- +function expectTypeError(thunk, label) { + try { + thunk(); + } catch (error) { + if (!(error instanceof TypeError)) { + print(`FAIL [${label}]: threw ${describeError(error)}, expected a TypeError`); + ++failures; + } + return; + } + print(`FAIL [${label}]: did not throw`); + ++failures; +} +function describeError(error) { + try { return String(error); } catch { return Object.prototype.toString.call(error); } +} +const badValues = [ + [42, "number"], + [4096n, "bigint"], + ["not a view", "string"], + [{}, "plain object"], + [undefined, "undefined"], + [null, "null"], + [new ArrayBuffer(8), "ArrayBuffer (not a view)"], +]; +const validView = new Uint8Array(16); +for (const [bad, label] of badValues) + expectTypeError(() => byteLength(validView, bad), `cold buffer_length=${label}`); + +// --------------------------------------------------------------------------------------------- +// 3. Tier differential: hammer the hot twins alongside the noDFG oracles, then re-check that +// the bad-value TypeErrors still fire from the (now tiered-up) callers. +// --------------------------------------------------------------------------------------------- +const backing = new ArrayBuffer(4096); +const views = [ + new Uint8Array(0), + new Uint8Array(1), + new Uint8Array(4), + new Uint8Array(4096), + new DataView(backing, 8, 24), + new Uint8Array(backing).subarray(17, 900), + new Uint32Array(backing, 64, 7), + new Float64Array(3), +]; +views[1][0] = 0x7f; +views[3][4095] = 0x11; +const iterations = 50000; +for (let i = 0; i < iterations; ++i) { + const view = views[i % views.length]; + agree(`byteLength#${i}`, hotByteLength(view), refByteLength(view)); + agree(`lastByte#${i}`, hotLastByte(view), refLastByte(view)); + if (hotByteLength(view) !== BigInt(view.byteLength)) { + print(`FAIL [hot byteLength#${i}]: ${hotByteLength(view)} != ${view.byteLength}`); + if (++failures > 8) throw new Error("too many failures"); + } +} + +// The bad-value paths must still throw the same TypeError once the callers are hot. +function hotThrows(view, bad) { return byteLength(view, bad); } +noInline(hotThrows); +for (let i = 0; i < 20000; ++i) + hotThrows(validView, validView); +for (const [bad, label] of badValues) + expectTypeError(() => hotThrows(validView, bad), `hot buffer_length=${label}`); + +// buffer_length is argument-only: a "length" return type is rejected at signature creation. +expectTypeError(() => $vm.ffiFunction({ args: ["ptr"], returns: "buffer_length" }, fixture("ffi_view_byte_length"), "bad"), "buffer_length as return type"); + +if (failures) + throw new Error(`ffi-buffer-length: ${failures} failure(s)`); diff --git a/JSTests/stress/ffi-callback-throw-unwind.js b/JSTests/stress/ffi-callback-throw-unwind.js new file mode 100644 index 0000000000000..1502eb1e1f0fd --- /dev/null +++ b/JSTests/stress/ffi-callback-throw-unwind.js @@ -0,0 +1,34 @@ +//@ requireOptions("--useDollarVM=1") +if (!$vm.useJIT()) quit(); +// #2: FTL CallFFI stores topCallFrame but no CallSiteIndex. A callback invoked from inside the +// native call that THROWS then unwinds using the frame's STALE callSiteIndex (left by the last +// operation call in this FTL function). If an earlier try{} region installed a handler at that +// stale index, the exception is delivered to the WRONG catch -- one that does not enclose the call. +const callCbVoid = $vm.ffiFunction({ args: ["ptr"], returns: "void" }, $vm.ffiFixture("ffi_call_cb_void"), "call_cb_void"); +const boom = $vm.ffiCallback({ args: [], returns: "void" }, () => { throw new RangeError("from-callback"); }); +noInline(f); +function f(mode) { + // An earlier try/catch that becomes an FTL exception-handler region + call site. + try { + if (mode === "early") throw new TypeError("early"); // exercises this handler + JSON.parse('{"ok":true}'); // an operation call inside the try (sets a callSiteIndex) + } catch (e) { + return "EARLY_HANDLER:" + e.constructor.name; // must NEVER see the callback's RangeError + } + // The FFI call is OUTSIDE the try. Its callback throws. Correct behavior: it propagates OUT of f. + callCbVoid(boom.ptr); + return "no-exception"; +} +let out; +for (let i = 0; i < 100000; ++i) { + try { + out = f("normal"); + } catch (e) { + out = "PROPAGATED:" + e.constructor.name; // <-- the ONLY correct outcome + } + if (out !== "PROPAGATED:RangeError") { + throw new Error("WRONG at iteration " + i + ": " + out + + " (EARLY_HANDLER means the exception was routed to the try's stale handler)"); + } +} +if (out === "PROPAGATED:RangeError") print("OK: callback exception propagated correctly in all tiers"); diff --git a/JSTests/stress/ffi-callbacks.js b/JSTests/stress/ffi-callbacks.js new file mode 100644 index 0000000000000..dd8cd3add3a48 --- /dev/null +++ b/JSTests/stress/ffi-callbacks.js @@ -0,0 +1,441 @@ +//@ requireOptions("--useDollarVM=1") + +// JSFFICallback: native -> JS calls through every ffi_call_cb_* fixture. +// Covers argument marshaling into JS (register and stack ladders, sub-8-byte +// arguments, mixed classes), return-value coercion, exceptions surfacing at +// the FFI call site, GC inside a callback while a pointer argument is +// outstanding, re-entrancy (loop and nested to depth 100), and the +// JS -> native -> JS round trip of a callback wrapped back into an FFI +// function. + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (Object.is(value, -0)) + return "-0"; + if (typeof value === "symbol") + return value.toString(); + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +// Returns a NEW caller function every time (a distinct FunctionExecutable / +// CodeBlock via `new Function`), so the FFI call site inside it is +// monomorphic, exact-arity and non-spread -- the only call-site shape the DFG +// Call -> CallFFI conversion accepts (SPEC section 10.2). The hot round-trip +// loop uses it so the JS -> native -> JS sandwich runs under a compiled +// CallFFI rather than a shared spread call site. +function makeMonomorphicCaller(arity) { + const argumentList = Array.from({ length: arity }, (_, i) => "args[" + i + "]").join(", "); + return new Function("callable", "args", "return callable(" + argumentList + ");"); +} + +function checkThrows(fn, validate, message) { + let thrown = false; + try { + fn(); + } catch (e) { + thrown = true; + if (validate) + validate(e); + } + if (!thrown) + throw new Error(message + ": expected an exception"); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (name, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(name), name); + const callback = (args, ret, fn) => $vm.ffiCallback({ args, returns: ret }, fn); + + const callCbI32 = bind("ffi_call_cb_i32", ["function", "i32"], "i32"); + const callCbF64x8 = bind("ffi_call_cb_f64_x8", ["function", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64"], "f64"); + const callCbF64x9 = bind("ffi_call_cb_f64_x9", ["function", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64"], "f64"); + const callCbI32x9 = bind("ffi_call_cb_i32_x9", ["function", "i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32"], "i64"); + const callCbU8x10 = bind("ffi_call_cb_u8_x10", ["function", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8"], "i64"); + const callCbMix = bind("ffi_call_cb_mix", ["function", "i32", "f64", "i64", "f32", "ptr"], "f64"); + const callCbVoid = bind("ffi_call_cb_void", ["function"], "void"); + const callCbReentrant = bind("ffi_call_cb_reentrant", ["function", "i32"], "i32"); + const callCbRetI8 = bind("ffi_call_cb_ret_i8", ["function"], "i64_fast"); + const callCbRetU8 = bind("ffi_call_cb_ret_u8", ["function"], "i64_fast"); + const callCbRetI64 = bind("ffi_call_cb_ret_i64", ["function"], "i64"); + const callCbRetU64 = bind("ffi_call_cb_ret_u64", ["function"], "u64"); + const callCbRetBool = bind("ffi_call_cb_ret_bool", ["function"], "i32"); + const callCbRetF32 = bind("ffi_call_cb_ret_f32", ["function"], "f32"); + const callCbRetF64 = bind("ffi_call_cb_ret_f64", ["function"], "f64"); + const callCbRetPtr = bind("ffi_call_cb_ret_ptr", ["function"], "ptr"); + const callCbThenReadU32 = bind("ffi_call_cb_then_read_u32", ["function", "ptr"], "u32"); + const addI32 = bind("ffi_add_i32", ["i32", "i32"], "i32"); + + // ---- Basics: arguments in, results out, .ptr and object forms. + { + const cb = callback(["i32"], "i32", x => x * 2 + 1); + check(typeof cb.ptr, "number", "callback .ptr typeof"); + if (!(cb.ptr > 0)) + throw new Error("callback .ptr should be a positive address"); + check(cb.threadsafe, false, "callback .threadsafe"); + for (const x of [0, 1, -1, 21, 1073741823, -1073741824, 2147483647, -2147483648]) { + const expected = (x * 2 + 1) | 0; + check(callCbI32(cb, x), expected, "ffi_call_cb_i32(cb, " + x + ")"); + check(callCbI32(cb.ptr, x), expected, "ffi_call_cb_i32(cb.ptr, " + x + ")"); + } + for (let i = 0; i < 3e4; ++i) { + const r = callCbI32(cb, i & 1023); + if (r !== ((i & 1023) * 2 + 1)) + throw new Error("hot ffi_call_cb_i32 iteration " + i + " got " + r); + } + } + + // ---- Argument ladders into JS: 8 and 9 doubles, 9 int32s (stack args), 10 uint8s (packed stack args). + { + const received = []; + const recorder = (...args) => { + received.length = 0; + for (const a of args) + received.push(a); + let sum = 0; + for (let i = 0; i < args.length; ++i) + sum += (i + 1) * args[i]; + return sum; + }; + const cbF64x8 = callback(["f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64"], "f64", recorder); + const values8 = [0.5, -1.25, 3.75, -4.5, 5.0625, -6.5, 7.75, -8.875]; + check(callCbF64x8(cbF64x8, ...values8), values8.reduce((s, v, i) => s + (i + 1) * v, 0), "ffi_call_cb_f64_x8 result"); + for (let i = 0; i < 8; ++i) + check(received[i], values8[i], "ffi_call_cb_f64_x8 argument " + i); + + const cbF64x9 = callback(["f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64"], "f64", recorder); + const values9 = [1e10, -0.03125, 2 ** 40, -(2 ** 39), 0.5, 1.5, 2.5, 3.5, -0]; + check(callCbF64x9(cbF64x9, ...values9), values9.reduce((s, v, i) => s + (i + 1) * v, 0), "ffi_call_cb_f64_x9 result"); + for (let i = 0; i < 9; ++i) + check(received[i], values9[i], "ffi_call_cb_f64_x9 argument " + i); + + const recorderInt = (...args) => { + received.length = 0; + for (const a of args) + received.push(a); + let sum = 0; + for (let i = 0; i < args.length; ++i) + sum += (i + 1) * args[i]; + return sum; + }; + const cbI32x9 = callback(["i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32"], "i64", recorderInt); + const ints = [1, -2, 3, -4, 5, -6, 7, -8, 2147483647]; + check(callCbI32x9(cbI32x9, ...ints), 1n - 4n + 9n - 16n + 25n - 36n + 49n - 64n + 9n * 2147483647n, "ffi_call_cb_i32_x9 result"); + for (let i = 0; i < 9; ++i) + check(received[i], ints[i], "ffi_call_cb_i32_x9 argument " + i); + const negatives = [-2147483648, -1, -2147483648, -1, -2147483648, -1, -2147483648, -1, -2147483648]; + callCbI32x9(cbI32x9, ...negatives); + for (let i = 0; i < 9; ++i) + check(received[i], negatives[i], "ffi_call_cb_i32_x9 negative argument " + i); + + const cbU8x10 = callback(["u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8"], "i64", recorderInt); + const bytes = [255, 0, 128, 1, 200, 17, 254, 3, 99, 250]; + check(callCbU8x10(cbU8x10, ...bytes), BigInt(bytes.reduce((s, v, i) => s + (i + 1) * v, 0)), "ffi_call_cb_u8_x10 result"); + for (let i = 0; i < 10; ++i) + check(received[i], bytes[i], "ffi_call_cb_u8_x10 argument " + i); + // Distinct powers of two catch any swapped or dropped stack byte. + const powers = [1, 2, 4, 8, 16, 32, 64, 128, 3, 5]; + check(callCbU8x10(cbU8x10, ...powers), BigInt(powers.reduce((s, v, i) => s + (i + 1) * v, 0)), "ffi_call_cb_u8_x10 powers"); + for (let i = 0; i < 10; ++i) + check(received[i], powers[i], "ffi_call_cb_u8_x10 powers argument " + i); + } + + // ---- Mixed argument classes; pointer arguments arrive as numbers, null pointers as null. + { + let last = null; + const cbMixFast = callback(["i32", "f64", "i64_fast", "f32", "ptr"], "f64", (a, b, c, d, e) => { + last = [a, b, c, d, e]; + return a + 2 * b + 3 * Number(c) + 4 * d + 5 * (e === null ? -1 : e); + }); + check(callCbMix(cbMixFast, -5, 2.5, 4503599627370496, 1.25, 8192), -5 + 5 + 3 * 4503599627370496 + 5 + 40960, "ffi_call_cb_mix result"); + check(last[0], -5, "mix arg i32"); + check(last[1], 2.5, "mix arg f64"); + check(last[2], 4503599627370496, "mix arg i64_fast (Number range)"); + check(last[3], 1.25, "mix arg f32"); + check(last[4], 8192, "mix arg ptr"); + callCbMix(cbMixFast, 0, -0, 0, Math.fround(1.1), 0); + check(last[1], -0, "mix arg f64 keeps the sign of zero"); + check(last[3], Math.fround(1.1), "mix arg f32 is the exact float value"); + check(last[4], null, "mix arg ptr null becomes JS null"); + const cbMixBig = callback(["i32", "f64", "i64", "f32", "ptr"], "f64", (a, b, c, d, e) => { + last = [a, b, c, d, e]; + return 0; + }); + callCbMix(cbMixBig, 1, 2, 2n ** 62n, 3, 4); + check(last[2], 2n ** 62n, "mix arg i64 as BigInt"); + check(typeof last[2], "bigint", "mix arg i64 typeof"); + } + + // ---- Void callback and side effects. + { + let count = 0; + const cbVoid = callback([], "void", () => { count++; }); + check(callCbVoid(cbVoid), undefined, "ffi_call_cb_void returns undefined"); + check(count, 1, "void callback invoked once"); + for (let i = 0; i < 2e4; ++i) + callCbVoid(cbVoid); + check(count, 2e4 + 1, "void callback hot count"); + } + + // ---- Return-value coercion (what native code sees after conversion). + { + check(callCbRetU8(callback([], "u8", () => 511)), 255, "u8 callback return wraps mod 256"); + check(callCbRetU8(callback([], "u8", () => -1)), 255, "u8 callback return of -1"); + check(callCbRetI8(callback([], "i8", () => 128)), -128, "i8 callback return wraps"); + check(callCbRetI8(callback([], "i8", () => undefined)), 0, "i8 callback returning undefined -> 0"); + check(callCbRetI8(callback([], "i8", () => null)), 0, "i8 callback returning null -> 0"); + check(callCbRetI8(callback([], "i8", () => true)), 1, "i8 callback returning true -> 1"); + check(callCbRetI64(callback([], "i64", () => 2n ** 63n - 1n)), 9223372036854775807n, "i64 callback returning INT64_MAX BigInt"); + check(callCbRetI64(callback([], "i64", () => -1)), -1n, "i64 callback returning -1 number"); + check(callCbRetI64(callback([], "i64", () => 2 ** 53)), 9007199254740992n, "i64 callback returning 2^53 number"); + check(callCbRetI64(callback([], "i64", () => -1.75)), -1n, "i64 callback returning -1.75 truncates"); + check(callCbRetU64(callback([], "u64", () => -1)), 18446744073709551615n, "u64 callback returning -1"); + check(callCbRetU64(callback([], "u64", () => 2n ** 64n + 5n)), 5n, "u64 callback BigInt mod 2^64"); + check(callCbRetBool(callback([], "bool", () => 2)), 10, "bool callback returning 2 -> true"); + check(callCbRetBool(callback([], "bool", () => 0)), 20, "bool callback returning 0 -> false"); + check(callCbRetBool(callback([], "bool", () => null)), 20, "bool callback returning null -> false"); + check(callCbRetBool(callback([], "bool", () => -0.5)), 10, "bool callback returning -0.5 -> true"); + check(callCbRetBool(callback([], "bool", () => NaN)), 20, "bool callback returning NaN -> false"); + check(Number.isNaN(callCbRetF32(callback([], "f32", () => NaN))), true, "f32 callback NaN return"); + check(callCbRetF32(callback([], "f32", () => 1.1)), Math.fround(1.1), "f32 callback return is rounded to float"); + check(callCbRetF64(callback([], "f64", () => -0)), -0, "f64 callback -0 return"); + check(callCbRetF64(callback([], "f64", () => undefined)), NaN, "f64 callback returning undefined -> NaN"); + check(callCbRetPtr(callback([], "ptr", () => 0)), null, "ptr callback returning 0 -> null"); + check(callCbRetPtr(callback([], "ptr", () => 65536)), 65536, "ptr callback returning 65536"); + check(callCbRetPtr(callback([], "ptr", () => null)), null, "ptr callback returning null"); + const array = new Uint8Array(8); + const address = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, fixture("ffi_ptr_identity"), "identity")(array); + check(callCbRetPtr(callback([], "ptr", () => array)), address, "ptr callback returning a TypedArray"); + } + + // ---- Callback returning a value that cannot convert: TypeError at the call site. + { + const badReturns = [ + [callback([], "i8", () => "not a number"), callCbRetI8, "string for i8"], + [callback([], "i8", () => Symbol("s")), callCbRetI8, "symbol for i8"], + [callback([], "ptr", () => "string"), callCbRetPtr, "string for ptr"], + [callback([], "ptr", () => ({})), callCbRetPtr, "plain object for ptr"], + [callback([], "u64", () => "abc"), callCbRetU64, "string for u64"], + ]; + for (const [cb, caller, label] of badReturns) { + checkThrows(() => caller(cb), e => { + if (!(e instanceof TypeError)) + throw new Error(label + ": expected a TypeError, got " + e); + }, label); + } + } + + // ---- A throwing callback: the exception (with its JS stack) surfaces at the FFI call site. + { + function throwingCallback(x) { + if (x === 13) + throw new RangeError("thirteen from callback"); + return x + 1; + } + const cbThrow = callback(["i32"], "i32", throwingCallback); + check(callCbI32(cbThrow, 12), 13, "throwing callback fine path"); + let caught = null; + try { + callCbI32(cbThrow, 13); + } catch (e) { + caught = e; + } + if (!(caught instanceof RangeError)) + throw new Error("expected the callback's RangeError to propagate, got " + caught); + check(caught.message, "thirteen from callback", "callback exception message"); + if (typeof caught.stack !== "string" || caught.stack.indexOf("throwingCallback") === -1) + throw new Error("expected the JS stack of the callback frame in the exception, got: " + caught.stack); + // The VM must be fully usable afterwards. + check(callCbI32(cbThrow, 41), 42, "callback usable after an exception"); + // Exceptions from within the last iteration of a native loop over the callback. + const cbThrowOnLast = callback(["i32"], "i32", i => { + if (i === 2) + throw new EvalError("last iteration"); + return i; + }); + checkThrows(() => callCbReentrant(cbThrowOnLast, 3), e => check(e instanceof EvalError, true, "EvalError from loop callback"), "loop callback throw"); + // In a hot loop with try/catch: every throw is caught, none escapes. + let count = 0; + for (let i = 0; i < 5000; ++i) { + try { + callCbI32(cbThrow, 13); + throw new Error("should not reach"); + } catch (e) { + if (e instanceof RangeError) + count++; + else + throw e; + } + } + check(count, 5000, "throwing callback in a hot try/catch loop"); + } + + // ---- gc() / fullGC() inside a callback while a TypedArray pointer argument is outstanding. + { + let churn = null; + const cbGC = callback([], "u32", () => { + for (let i = 0; i < 100; ++i) + churn = { i, payload: new Array(16).fill(i) }; + gc(); + fullGC(); + return 42; + }); + for (let i = 0; i < 20; ++i) { + // The Uint32Array is a temporary: only the outstanding native call + // references its storage while the callback collects. + const result = callCbThenReadU32(cbGC, new Uint32Array([123456789 + i])); + check(result, 123456789 + i, "read after GC-ing callback iteration " + i); + } + // Same, but the view is also written before the call and read after. + const persistent = new Uint32Array(4); + persistent[0] = 0xfeedface; + check(callCbThenReadU32(cbGC, persistent), 0xfeedface >>> 0, "persistent view read after GC-ing callback"); + check(persistent[0], 0xfeedface >>> 0, "persistent view intact after GC-ing callback"); + } + + // ---- Re-entrancy. + { + // (a) Loop of 100 callback invocations, each of which re-enters the engine + // through another FFI call. + let seen = 0; + const cbLoop = callback(["i32"], "i32", i => { + seen++; + return addI32(i, 1); + }); + check(callCbReentrant(cbLoop, 100), 5050, "ffi_call_cb_reentrant(cb, 100)"); + check(seen, 100, "loop callback invocation count"); + check(callCbReentrant(cbLoop, 0), 0, "ffi_call_cb_reentrant depth 0"); + + // (b) True nesting to depth 100: JS -> native -> JS -> native -> ... . + const nestCb = callback(["i32"], "i32", d => 1 + nest(d - 1)); + function nest(depth) { + if (depth <= 0) + return 0; + return callCbI32(nestCb, depth); + } + check(nest(100), 100, "nested FFI/callback depth 100"); + check(nest(1), 1, "nested depth 1"); + + // (c) A callback that calls the very FFI function that invoked it (with a base case). + const selfCb = callback(["i32"], "i32", x => x <= 0 ? 0 : callCbI32(selfCb, x - 1) + 2); + check(callCbI32(selfCb, 40), 80, "self-recursive callback"); + + // (d) An exception thrown at depth 50 unwinds through 50 native frames. + const deepThrowCb = callback(["i32"], "i32", d => { + if (d === 50) + throw new URIError("depth 50"); + return 1 + nestThrow(d - 1); + }); + function nestThrow(depth) { + if (depth <= 0) + return 0; + return callCbI32(deepThrowCb, depth); + } + checkThrows(() => nestThrow(80), e => check(e instanceof URIError, true, "deep unwind error type"), "exception from depth 50"); + // Everything still works afterwards. + check(nest(10), 10, "nested depth 10 after deep unwind"); + } + + // ---- A callback wrapped back into an FFI function: JS -> invoke thunk -> callback thunk -> JS. + { + const roundTrips = [ + [{ args: ["i32", "i32"], returns: "i32" }, (a, b) => (a - b) | 0, [[5, 3, 2], [0x7fffffff, -1, -2147483648], [-2147483648, 1, 2147483647]]], + [{ args: ["f64", "f64"], returns: "f64" }, (a, b) => a / b, [[1, 4, 0.25], [1, 0, Infinity], [-1, 0, -Infinity], [0, 0, NaN]]], + [{ args: ["f32"], returns: "f32" }, x => x * 2, [[1.5, 3], [Math.fround(1.1), Math.fround(1.1) * 2], [NaN, NaN], [1e39, Infinity]]], + [{ args: ["u8", "i16"], returns: "i64" }, (a, b) => BigInt(a * 1000 + b), [[255, -1, 254999n], [0, -32768, -32768n], [-1, 32767, 287767n]]], + [{ args: ["bool", "bool"], returns: "bool" }, (a, b) => a && !b, [[true, false, true], [2, 0, true], [0, 1, false]]], + [{ args: ["i64", "u64"], returns: "i64" }, (a, b) => a - b, [[10n, 3n, 7n], [-1n, 1n, -2n], [0, 0, 0n]]], + [{ args: ["char"], returns: "char" }, c => c, [[-1, -1], [255, -1], [0x80, -128], [127, 127]]], + ]; + for (const [signature, fn, cases] of roundTrips) { + const cb = $vm.ffiCallback(signature, fn); + const wrapped = $vm.ffiFunction(signature, cb, "roundtrip " + $vm.ffiSignatureString(signature)); + for (const c of cases) { + const inputs = c.slice(0, c.length - 1); + const expected = c[c.length - 1]; + check(wrapped(...inputs), expected, "round trip " + $vm.ffiSignatureString(signature) + "(" + inputs.map(describe).join(",") + ")"); + } + // Hot: the JS->native->JS sandwich under the JIT tiers, through a + // dedicated exact-arity monomorphic caller (a spread call site + // could never become a CallFFI). + const c = cases[0]; + const inputs = c.slice(0, c.length - 1); + const expected = c[c.length - 1]; + const caller = makeMonomorphicCaller(inputs.length); + for (let i = 0; i < 1e4; ++i) { + const r = caller(wrapped, inputs); + if (!Object.is(r, expected)) + throw new Error("hot round trip " + $vm.ffiSignatureString(signature) + " iteration " + i + " got " + describe(r)); + } + } + } + + // ---- Property surface; un-close()d callbacks are engine-rooted, so they SURVIVE gc()/fullGC() + // (the destructor runs only after close()); the collection here checks a rooted callback + // stays fully functional across a full GC. + { + const cb = callback(["i32"], "i32", x => x); + check(typeof cb.ptr, "number", "callback .ptr is a number"); + const descriptor = Object.getOwnPropertyDescriptor(cb, "ptr"); + check(descriptor !== undefined, true, "ptr is an own property"); + check(descriptor.writable === true, false, "ptr is read-only"); + check(descriptor.enumerable, false, "ptr is don't-enum"); + check(descriptor.configurable, false, "ptr is don't-delete"); + const threadsafeDescriptor = Object.getOwnPropertyDescriptor(cb, "threadsafe"); + check(threadsafeDescriptor !== undefined, true, "threadsafe is an own property"); + check(cb.threadsafe, false, "threadsafe is false"); + check(Object.keys(cb).length, 0, "own properties are non-enumerable"); + for (let i = 0; i < 200; ++i) + callback(["i32"], "i32", x => x + i); + gc(); + fullGC(); + // The surviving callback still works after a full collection. + check(callCbI32(cb, 41), 41, "callback survives GC"); + } + + // ---- The single close() rule (SPEC section 9.1) as seen from JS: `ptr` + // becomes null, close() is idempotent, the entry code stays alive with + // the cell (a pointer captured before close() keeps working), and the + // $vm glue rejects the closed object wherever it takes a pointer. + { + const closed = callback(["i32"], "i32", x => x + 100); + const entryBefore = closed.ptr; + check(typeof entryBefore, "number", "ptr before close"); + check(callCbI32(entryBefore, 5), 105, "call through the raw entry pointer before close"); + check(typeof closed.close, "function", "close is callable from JS"); + check(closed.close(), undefined, "close() returns undefined"); + check(closed.ptr, null, "ptr is null after close"); + check(closed.close(), undefined, "close() is idempotent"); + check(closed.ptr, null, "ptr stays null after a second close"); + const descriptor = Object.getOwnPropertyDescriptor(closed, "ptr"); + check(descriptor.value, null, "closed ptr descriptor value"); + check(descriptor.enumerable, false, "closed ptr stays don't-enum"); + // Nothing native-side was dropped: the code lives with the cell, so + // the entry pointer obtained earlier is still a valid callback. + check(callCbI32(entryBefore, 6), 106, "entry code alive after close"); + gc(); + check(callCbI32(entryBefore, 7), 107, "entry code alive after close and GC"); + // The $vm glue rejects a closed callback wherever it converts it to + // a pointer (target of ffiFunction, ffiCString, ffiRead). + for (const [label, use] of [ + ["ffiFunction target", () => $vm.ffiFunction({ args: ["i32"], returns: "i32" }, closed, "closed target")], + ["ffiCString", () => $vm.ffiCString(closed)], + ["ffiRead", () => $vm.ffiRead(closed, "u8")], + ]) { + checkThrows(use, e => { + if (!(e instanceof TypeError)) + throw new Error(label + " with a closed callback: expected a TypeError, got " + e); + if (String(e.message).indexOf("closed") === -1) + throw new Error(label + " with a closed callback: unexpected message: " + e.message); + }, label + " must reject a closed callback"); + } + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-callffi-was-compiled.js b/JSTests/stress/ffi-callffi-was-compiled.js new file mode 100644 index 0000000000000..8e421a81cbd39 --- /dev/null +++ b/JSTests/stress/ffi-callffi-was-compiled.js @@ -0,0 +1,76 @@ +//@ requireOptions("--useDollarVM=1", "--useConcurrentJIT=0", "--jitPolicyScale=0") + +// Proves the ByteCodeParser feed + strength-reduction conversion are not +// dead code: after a hot exact-arity monomorphic call site, at least one +// CallFFI node must have been compiled by the DFG or FTL, and creating an FFI +// function must have compiled an IC entry stub (SPEC sections 10.2, 11.2). +// The compile counts are process-global atomics read via +// $vm.ffiCompileCounts(). + +function main() { + const before = $vm.ffiCompileCounts(); + if (typeof before !== "object" || typeof before.icStub !== "number" || typeof before.dfgCallFFI !== "number" || typeof before.ftlCallFFI !== "number") + throw new Error("bad $vm.ffiCompileCounts() shape: " + JSON.stringify(before)); + + const addI32 = $vm.ffiFunction({ args: ["i32", "i32"], returns: "i32" }, $vm.ffiFixture("ffi_add_i32"), "ffi_add_i32"); + const echoF64 = $vm.ffiFunction({ args: ["f64"], returns: "f64" }, $vm.ffiFixture("ffi_echo_f64"), "ffi_echo_f64"); + const echoBool = $vm.ffiFunction({ args: ["bool"], returns: "bool" }, $vm.ffiFixture("ffi_echo_bool"), "ffi_echo_bool"); + const echoPtr = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, $vm.ffiFixture("ffi_echo_ptr"), "ffi_echo_ptr"); + const echoI64 = $vm.ffiFunction({ args: ["i64"], returns: "i64" }, $vm.ffiFixture("ffi_echo_i64"), "ffi_echo_i64"); + + const afterCreation = $vm.ffiCompileCounts(); + if (afterCreation.icStub <= before.icStub) { + // The IC stub is generated eagerly in JSFFIFunction::create() when + // Options::useFFIICStub() (default true). + throw new Error("no IC stub was compiled by JSFFIFunction creation: before " + before.icStub + ", after " + afterCreation.icStub); + } + if (afterCreation.icStub < before.icStub + 5) + throw new Error("expected one IC stub per JSFFIFunction: before " + before.icStub + ", after " + afterCreation.icStub); + + // Exact-arity, monomorphic, hot: everything the conversion requires. + function hot(a, b) { + return addI32(a, b); + } + noInline(hot); + function hotTyped(d, flag, view, big) { + // Several typed CallFFI conversions in one code block. + const x = echoF64(d) + (echoBool(flag) ? 1 : 0); + const p = echoPtr(view); + const b = echoI64(big); + return x + (p === null ? 0 : 1) + Number(b & 0xffn); + } + noInline(hotTyped); + + const view = new Uint8Array(4); + let sink = 0; + for (let i = 0; i < 1e5; ++i) + sink += hot(i, 1); + if (sink !== 5000050000) + throw new Error("hot arithmetic wrong: " + sink); + for (let i = 0; i < 1e5; ++i) + sink += hotTyped(i + 0.5, i & 1, view, BigInt(i) & 0x7fn); + if (!Number.isFinite(sink)) + throw new Error("hotTyped produced a non-finite sum"); + + const counts = $vm.ffiCompileCounts(); + // Only demand CallFFI compilation when the DFG actually compiled the hot + // callers in this configuration (some harness configs disable the DFG). + const dfgRan = numberOfDFGCompiles(hot) > 0 || numberOfDFGCompiles(hotTyped) > 0; + if (dfgRan && counts.dfgCallFFI + counts.ftlCallFFI === 0) + throw new Error("DFG compiled the hot callers but no CallFFI node was compiled: " + JSON.stringify(counts)); + if (counts.dfgCallFFI + counts.ftlCallFFI < before.dfgCallFFI + before.ftlCallFFI) + throw new Error("compile counters went backwards"); + + // Results are still exactly right after tier-up. + if (hot(2147483647, 1) !== -2147483648) + throw new Error("hot(overflow) wrong after compilation"); + if (hot(-5, 10) !== 5) + throw new Error("hot(-5, 10) wrong after compilation"); + if (hotTyped(1.5, true, view, 255n) !== 1.5 + 1 + 1 + 255) + throw new Error("hotTyped exact value wrong after compilation"); + if (hotTyped(-0.25, false, null, -1n) !== -0.25 + 0 + 0 + Number(BigInt.asIntN(64, -1n) & 0xffn)) + throw new Error("hotTyped with null pointer wrong after compilation"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-canary.js b/JSTests/stress/ffi-canary.js new file mode 100644 index 0000000000000..15728b8856d88 --- /dev/null +++ b/JSTests/stress/ffi-canary.js @@ -0,0 +1,100 @@ +//@ requireOptions("--useDollarVM=1") + +// Callee-saved register canary (SPEC section 11.1): ffi_canary_call loads +// sentinels into every ABI-callee-saved GPR/FPR, calls the callback, and +// returns a bitmask of the registers that were clobbered. The callback thunk +// (native entry -> callbackDispatch -> JS) plus everything the JS side does +// must preserve all of them, in every JIT tier and with callbacks of every +// arity and behavior. + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + String(expected) + " but got " + String(actual)); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const canary = $vm.ffiFunction({ args: ["function"], returns: "i32" }, fixture("ffi_canary_call"), "ffi_canary_call"); + const addI32 = $vm.ffiFunction({ args: ["i32", "i32"], returns: "i32" }, fixture("ffi_add_i32"), "ffi_add_i32"); + const sumF64_12 = $vm.ffiFunction({ args: new Array(12).fill("f64"), returns: "f64" }, fixture("ffi_sum_f64_12"), "ffi_sum_f64_12"); + const callCbI32 = $vm.ffiFunction({ args: ["function", "i32"], returns: "i32" }, fixture("ffi_call_cb_i32"), "ffi_call_cb_i32"); + const alignProbe0 = $vm.ffiFunction({ args: [], returns: "f64" }, fixture("ffi_align_probe_0"), "ffi_align_probe_0"); + const makeCanaryCallback = fn => $vm.ffiCallback({ args: [], returns: "void" }, fn); + + let sink = 0; + // Callbacks of every "arity"/shape wrapped as void(void) native callbacks. + const behaviours = [ + () => { }, + () => { sink++; }, + (a) => { sink += a === undefined ? 1 : 0; }, + (a, b, c, d, e, f, g, h, i, j, k, l) => { sink += (a === undefined) + (l === undefined); }, + (...rest) => { sink += rest.length; }, + function usesArguments() { sink += arguments.length; }, + () => { let x = 0; for (let i = 0; i < 200; ++i) x = Math.imul(x + i, 31) ^ (x >>> 7); sink += x & 1; }, + () => { const o = []; for (let i = 0; i < 500; ++i) o.push({ i, s: "s" + i }); sink += o.length; }, + () => { sink += addI32(20, 22); }, // re-enter an FFI function from inside the callback + () => { sink += sumF64_12(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) | 0; }, // fp-heavy re-entry + () => { const inner = makeCanaryCallback(() => { sink++; }); sink += canary(inner); }, // nested canary + () => { const cb = $vm.ffiCallback({ args: ["i32"], returns: "i32" }, x => x * 3); sink += callCbI32(cb, 14); }, + () => { let d = 0.5; for (let i = 0; i < 100; ++i) d = Math.sqrt(d + i) * 1.0001; sink += d | 0; }, // touches many FP registers + () => { gc(); }, + () => { fullGC(); }, + () => { try { throw new Error("caught inside"); } catch (e) { sink += e.message.length; } }, + () => { sink += alignProbe0() === 1 ? 1 : 100; }, // stack alignment inside the callback frame + () => { const big = 2n ** 200n + 1n; sink += Number(big % 3n); }, + () => { sink += "abc".repeat(64).length; }, + async () => { sink++; }, // returns a promise (ignored by the void return conversion) + () => 12345, // returns a value for a void callback: ignored + ]; + + for (let i = 0; i < behaviours.length; ++i) { + const cb = makeCanaryCallback(behaviours[i]); + const mask = canary(cb); + check(mask, 0, "canary with behaviour #" + i); + } + + // The same callbacks, but hot: the FFI call to ffi_canary_call itself goes + // through the IC stub / DFG / FTL paths, whose register state differs. The + // full-heap gc()/fullGC() behaviours (#13/#14) stay in the cold pass only: + // hundreds of synchronous full collections would blow the per-test time + // budget in debug builds without adding register coverage. + const hotBehaviours = behaviours.filter((_, index) => index !== 13 && index !== 14); + const hotCallbacks = hotBehaviours.map(makeCanaryCallback); + for (let iteration = 0; iteration < 4000; ++iteration) { + const cb = hotCallbacks[iteration % hotCallbacks.length]; + const mask = canary(cb); + if (mask !== 0) + throw new Error("canary clobber mask 0x" + mask.toString(16) + " at hot iteration " + iteration + " (behaviour #" + (iteration % hotCallbacks.length) + ")"); + } + + // A monomorphic hot loop so the caller reliably tiers up with one callback. + const trivial = makeCanaryCallback(() => { sink++; }); + for (let iteration = 0; iteration < 2e4; ++iteration) { + if (canary(trivial) !== 0) + throw new Error("canary trivial hot iteration " + iteration); + } + + // A canary callback that throws: the exception propagates from the + // canary's FFI call site (the canary's own return value is then not + // observable, but the throw path must not corrupt the frame either). + const throwing = makeCanaryCallback(() => { throw new TypeError("boom"); }); + let caught = 0; + for (let i = 0; i < 200; ++i) { + try { + canary(throwing); + } catch (e) { + if (!(e instanceof TypeError)) + throw new Error("wrong exception type from throwing canary callback: " + e); + caught++; + } + } + check(caught, 200, "throwing canary callbacks"); + // ... and the canary is unharmed afterwards. + check(canary(trivial), 0, "canary after exceptions"); + + if (sink < 0) + throw new Error("unreachable"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-conversion-errors-host.js b/JSTests/stress/ffi-conversion-errors-host.js new file mode 100644 index 0000000000000..384d9a5803685 --- /dev/null +++ b/JSTests/stress/ffi-conversion-errors-host.js @@ -0,0 +1,261 @@ +//@ requireOptions("--useDollarVM=1", "--useFFIICStub=0", "--useFFICallInDFG=0") + +// Same battery as ffi-conversion-errors.js, but with the IC stub and the +// DFG/FTL CallFFI conversion disabled so every call takes the C++ host path +// (SPEC section 8.2). Argument-conversion failures must be TypeErrors with a +// message identical between the cold call and the hot (tiered-up caller) +// call. + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (typeof value === "symbol") + return value.toString(); + if (typeof value === "function") + return "function"; + if (Object.is(value, -0)) + return "-0"; + try { + return String(value); + } catch { + return Object.prototype.toString.call(value); + } +} + +// Returns a NEW caller function every time (a distinct FunctionExecutable / +// CodeBlock via `new Function`), so the FFI call site inside it is +// monomorphic, exact-arity and non-spread -- kept identical to +// ffi-conversion-errors.js so the two files differ only in the option pair +// (here every call still takes the C++ host path). +function makeMonomorphicCaller(arity) { + const argumentList = Array.from({ length: arity }, (_, i) => "args[" + i + "]").join(", "); + return new Function("callable", "args", "return callable(" + argumentList + ");"); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (fixtureName, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(fixtureName), fixtureName + "(" + args.join(",") + ")"); + + const detachedView = (() => { + const buffer = new ArrayBuffer(8); + const view = new Uint8Array(buffer); + if (typeof transferArrayBuffer === "function") + transferArrayBuffer(buffer); + else + buffer.transfer(); + return view; + })(); + const symbol = Symbol("bad"); + const plainObject = { valueOf() { return 42; }, toString() { return "42"; } }; + const array = [1, 2, 3]; + const jsFunction = function () { return 7; }; + const proxy = new Proxy({}, {}); + + // [callable, valid argument for warm-up, bad argument, label] + const echoI32 = bind("ffi_echo_i32", ["i32"], "i32"); + const echoU8 = bind("ffi_echo_u8", ["u8"], "u8"); + const echoI16 = bind("ffi_echo_i16", ["i16"], "i16"); + const echoBool = bind("ffi_echo_bool", ["bool"], "bool"); + const echoF64 = bind("ffi_echo_f64", ["f64"], "f64"); + const echoF32 = bind("ffi_echo_f32", ["f32"], "f32"); + const echoI64 = bind("ffi_echo_i64", ["i64"], "i64"); + const echoU64 = bind("ffi_echo_u64", ["u64"], "u64"); + const echoI64Fast = bind("ffi_echo_i64", ["i64_fast"], "i64_fast"); + const echoPtr = bind("ffi_echo_ptr", ["ptr"], "ptr"); + const echoCString = bind("ffi_echo_cstring", ["cstring"], "cstring"); + const bufferArg = bind("ffi_ptr_identity", ["buffer"], "ptr"); + const functionArg = bind("ffi_ptr_identity", ["function"], "ptr"); + const validCallback = $vm.ffiCallback({ args: [], returns: "void" }, () => { }); + const validView = new Uint8Array(16); + + const cases = [ + [echoI32, 1, symbol, "Symbol -> i32"], + [echoI32, 1, "42", "string -> i32 (strings never coerce into numeric params)"], + [echoU8, 1, "255", "string -> u8"], + [echoF64, 1.5, "1.5", "string -> f64"], + [echoF32, 1.5, "1.5", "string -> f32"], + [echoU8, 1, symbol, "Symbol -> u8"], + [echoF64, 1.5, symbol, "Symbol -> f64"], + [echoF32, 1.5, symbol, "Symbol -> f32"], + [echoI64, 1, symbol, "Symbol -> i64"], + [echoI64, 1, plainObject, "object -> i64"], + [echoI64, 1, "5", "string -> i64"], + [echoI64, 1, undefined, "undefined -> i64"], + [echoI64, 1, null, "null -> i64"], + [echoI64, 1, true, "boolean -> i64"], + [echoU64, 1, "5", "string -> u64"], + [echoU64, 1, undefined, "undefined -> u64"], + [echoU64, 1, plainObject, "object -> u64"], + [echoI64Fast, 1, symbol, "Symbol -> i64_fast"], + [echoI64Fast, 1, "5", "string -> i64_fast"], + [echoPtr, validView, symbol, "Symbol -> ptr"], + [echoPtr, validView, plainObject, "object -> ptr"], + [echoPtr, validView, "hello", "JS string -> ptr (only cstring transcodes)"], + [echoPtr, validView, array, "array -> ptr"], + [echoPtr, validView, jsFunction, "JS function -> ptr"], + [echoPtr, validView, proxy, "proxy -> ptr"], + // (BigInt -> ptr / cstring is ACCEPTED as an exact 64-bit address -- + // oven-sh/bun#22751, #28068 -- and is covered by ffi-pointers-and-buffers.js.) + [echoPtr, validView, true, "boolean -> ptr"], + [echoCString, validView, symbol, "Symbol -> cstring"], + [echoCString, validView, plainObject, "object -> cstring"], + [echoCString, validView, true, "boolean -> cstring"], + [bufferArg, validView, 5, "number -> buffer (buffer requires a view)"], + [bufferArg, validView, null, "null -> buffer"], + [bufferArg, validView, undefined, "undefined -> buffer"], + [bufferArg, validView, plainObject, "object -> buffer"], + [bufferArg, validView, "abc", "string -> buffer"], + [bufferArg, validView, new ArrayBuffer(8), "ArrayBuffer -> buffer (not a view)"], + [bufferArg, validView, symbol, "Symbol -> buffer"], + [functionArg, validCallback, "cb", "JS string -> function"], + [functionArg, validCallback, plainObject, "object -> function"], + [functionArg, validCallback, jsFunction, "raw JS function -> function (must be a JSFFICallback)"], + [functionArg, validCallback, symbol, "Symbol -> function"], + [functionArg, validCallback, true, "boolean -> function"], + ]; + + // ---- The loose-coercion contract (bun parity): [callable, input, expected, label]. + // These MUST NOT throw; they pin the exact coerced value the callee receives. + function checkCoercion(actual, expected, label) { + if (Number.isNaN(expected) ? !Number.isNaN(actual) : !Object.is(actual, expected)) + throw new Error("coercion " + label + ": expected " + describe(expected) + " but got " + describe(actual)); + } + const coercions = [ + [echoI32, plainObject, 42, "object.valueOf -> i32"], + [echoI32, array, 0, "array -> i32 (Number([1,2,3]) = NaN -> 0)"], + [echoI32, 10n, 10, "BigInt -> i32"], + [echoI32, true, 1, "true -> i32"], + [echoI32, null, 0, "null -> i32"], + [echoI32, undefined, 0, "undefined -> i32"], + [echoI32, 4294902015, -65281, "u32 pattern into i32 wraps (bun#7007 class)"], + [echoU8, 300, 44, "u8 wraps mod 256 (300 -> 44), never clamps"], + [echoU8, -1, 255, "u8 wraps negative (-1 -> 255)"], + [echoI16, jsFunction, 0, "function -> i16 (Number(fn) = NaN -> 0)"], + [echoBool, plainObject, true, "object -> bool"], + [echoBool, 1n, true, "BigInt 1n -> bool"], + [echoF64, plainObject, 42, "object.valueOf -> f64"], + [echoF64, true, 1, "true -> f64"], + [echoF64, null, 0, "null -> f64"], + [echoF64, undefined, NaN, "undefined -> f64 (Number(undefined) = NaN)"], + [echoF64, 2n, 2, "BigInt -> f64 (Number(5n)-style)"], + [echoF32, undefined, NaN, "undefined -> f32"], + [echoF32, null, 0, "null -> f32"], + [echoI64, 5, 5n, "number -> i64"], + [echoI64, 5n, 5n, "BigInt -> i64"], + ]; + for (const [callable, input, expected, label] of coercions) { + let actual; + try { + actual = callable(input); + } catch (e) { + throw new Error("coercion " + label + ": threw " + e); + } + checkCoercion(actual, expected, "cold " + label); + } + // ...and after tier-up the SAME coercions produce the SAME values. + for (const [callable, input, expected, label] of coercions) { + const caller = makeMonomorphicCaller(1); + for (let i = 0; i < 5000; ++i) + caller(callable, [input]); + checkCoercion(caller(callable, [input]), expected, "hot " + label); + } + + const coldMessages = new Map(); + for (const [callable, good, bad, label] of cases) { + // Sanity: the valid argument works. + callable(good); + let error = null; + try { + callable(bad); + } catch (e) { + error = e; + } + if (error === null) + throw new Error("cold: " + label + ": " + describe(bad) + " did not throw"); + if (!(error instanceof TypeError)) + throw new Error("cold: " + label + ": expected a TypeError, got " + error); + if (typeof error.message !== "string" || !error.message.length) + throw new Error("cold: " + label + ": TypeError has no message"); + coldMessages.set(label, error.message); + // The function must remain usable. + callable(good); + } + + // Warm every callable with valid arguments through its own monomorphic + // caller so that caller tiers up, then re-trigger the same error through + // the SAME (tiered-up) call site and demand the identical message. + for (const [callable, good, bad, label] of cases) { + const caller = makeMonomorphicCaller(1); + const goodArgs = [good]; + const badArgs = [bad]; + for (let i = 0; i < 3000; ++i) + caller(callable, goodArgs); + for (let i = 0; i < 150; ++i) { + let error = null; + try { + caller(callable, badArgs); + } catch (e) { + error = e; + } + if (error === null) + throw new Error("hot: " + label + " iteration " + i + " did not throw"); + if (!(error instanceof TypeError)) + throw new Error("hot: " + label + " iteration " + i + ": expected a TypeError, got " + error); + if (error.message !== coldMessages.get(label)) + throw new Error("hot: " + label + " iteration " + i + ": message \"" + error.message + "\" != cold \"" + coldMessages.get(label) + "\""); + // Interleave valid calls so the site stays optimized. + caller(callable, goodArgs); + } + } + + // A single hot function that alternates good and bad values (the same + // compiled CallFFI site takes both the fast and the throwing slow path). + function guarded(value) { + try { + return { ok: true, value: echoI32(value) }; + } catch (e) { + return { ok: false, error: e }; + } + } + noInline(guarded); + for (let i = 0; i < 8000; ++i) { + const result = guarded(i); + if (!result.ok || result.value !== (i | 0)) + throw new Error("guarded warm iteration " + i); + } + for (let i = 0; i < 3000; ++i) { + const bad = (i % 5) === 4; + const result = guarded(bad ? symbol : i); + if (bad) { + if (result.ok) + throw new Error("guarded(Symbol) did not throw at iteration " + i); + if (!(result.error instanceof TypeError)) + throw new Error("guarded(Symbol) wrong error at iteration " + i + ": " + result.error); + if (result.error.message !== coldMessages.get("Symbol -> i32")) + throw new Error("guarded(Symbol) message differs from the cold message at iteration " + i); + } else if (!result.ok || result.value !== (i | 0)) + throw new Error("guarded good iteration " + i); + } + + // FFI-SPEC-GAP: SPEC section 11.4 lists "detached buffer as ptr" among + // the TypeError cases, but the normative conversion table (section 5) + // says "vector() (0 if detached)". The normative rule wins here: detached + // views convert to a null pointer in every tier, without throwing. + for (let i = 0; i < 3; ++i) { + if (echoPtr(detachedView) !== null) + throw new Error("detached view as ptr should yield null (iteration " + i + ")"); + if (bufferArg(detachedView) !== null) + throw new Error("detached view as buffer should yield null (iteration " + i + ")"); + if (echoCString(detachedView) !== null) + throw new Error("detached view as cstring should yield null (iteration " + i + ")"); + } + for (let i = 0; i < 5000; ++i) { + if (echoPtr(i & 1 ? detachedView : validView) === undefined) + throw new Error("unreachable"); + } + if (echoPtr(detachedView) !== null) + throw new Error("detached view as ptr should yield null when hot"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-conversion-errors.js b/JSTests/stress/ffi-conversion-errors.js new file mode 100644 index 0000000000000..f5f60991931a9 --- /dev/null +++ b/JSTests/stress/ffi-conversion-errors.js @@ -0,0 +1,266 @@ +//@ requireOptions("--useDollarVM=1") + +// Argument-conversion failures must be TypeErrors, identical whether they +// come from the C++ host path, the IC stub's slow path or the DFG/FTL +// operationFFIWriteSlot slow path (SPEC section 11.4). This file captures +// each error message cold, then re-triggers the same failure from a hot +// (tiered-up) call site and requires the identical constructor and message. +// ffi-conversion-errors-host.js runs the same battery with the IC stub and +// CallFFI disabled. + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (typeof value === "symbol") + return value.toString(); + if (typeof value === "function") + return "function"; + if (Object.is(value, -0)) + return "-0"; + try { + return String(value); + } catch { + return Object.prototype.toString.call(value); + } +} + +// Returns a NEW caller function every time (a distinct FunctionExecutable / +// CodeBlock via `new Function`), so the FFI call site inside it is +// monomorphic, exact-arity and non-spread -- the only call-site shape the DFG +// Call -> CallFFI conversion accepts (SPEC section 10.2). Each case below +// warms through its own caller so the bad value reaches the SAME optimized +// call site (typed check or operationFFIWriteSlot slow path) instead of a +// polymorphic shared site the DFG never converts. +function makeMonomorphicCaller(arity) { + const argumentList = Array.from({ length: arity }, (_, i) => "args[" + i + "]").join(", "); + return new Function("callable", "args", "return callable(" + argumentList + ");"); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (fixtureName, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(fixtureName), fixtureName + "(" + args.join(",") + ")"); + + const detachedView = (() => { + const buffer = new ArrayBuffer(8); + const view = new Uint8Array(buffer); + if (typeof transferArrayBuffer === "function") + transferArrayBuffer(buffer); + else + buffer.transfer(); + return view; + })(); + const symbol = Symbol("bad"); + const plainObject = { valueOf() { return 42; }, toString() { return "42"; } }; + const array = [1, 2, 3]; + const jsFunction = function () { return 7; }; + const proxy = new Proxy({}, {}); + + // [callable, valid argument for warm-up, bad argument, label] + const echoI32 = bind("ffi_echo_i32", ["i32"], "i32"); + const echoU8 = bind("ffi_echo_u8", ["u8"], "u8"); + const echoI16 = bind("ffi_echo_i16", ["i16"], "i16"); + const echoBool = bind("ffi_echo_bool", ["bool"], "bool"); + const echoF64 = bind("ffi_echo_f64", ["f64"], "f64"); + const echoF32 = bind("ffi_echo_f32", ["f32"], "f32"); + const echoI64 = bind("ffi_echo_i64", ["i64"], "i64"); + const echoU64 = bind("ffi_echo_u64", ["u64"], "u64"); + const echoI64Fast = bind("ffi_echo_i64", ["i64_fast"], "i64_fast"); + const echoPtr = bind("ffi_echo_ptr", ["ptr"], "ptr"); + const echoCString = bind("ffi_echo_cstring", ["cstring"], "cstring"); + const bufferArg = bind("ffi_ptr_identity", ["buffer"], "ptr"); + const functionArg = bind("ffi_ptr_identity", ["function"], "ptr"); + const validCallback = $vm.ffiCallback({ args: [], returns: "void" }, () => { }); + const validView = new Uint8Array(16); + + const cases = [ + [echoI32, 1, symbol, "Symbol -> i32"], + [echoI32, 1, "42", "string -> i32 (strings never coerce into numeric params)"], + [echoU8, 1, "255", "string -> u8"], + [echoF64, 1.5, "1.5", "string -> f64"], + [echoF32, 1.5, "1.5", "string -> f32"], + [echoU8, 1, symbol, "Symbol -> u8"], + [echoF64, 1.5, symbol, "Symbol -> f64"], + [echoF32, 1.5, symbol, "Symbol -> f32"], + [echoI64, 1, symbol, "Symbol -> i64"], + [echoI64, 1, plainObject, "object -> i64"], + [echoI64, 1, "5", "string -> i64"], + [echoI64, 1, undefined, "undefined -> i64"], + [echoI64, 1, null, "null -> i64"], + [echoI64, 1, true, "boolean -> i64"], + [echoU64, 1, "5", "string -> u64"], + [echoU64, 1, undefined, "undefined -> u64"], + [echoU64, 1, plainObject, "object -> u64"], + [echoI64Fast, 1, symbol, "Symbol -> i64_fast"], + [echoI64Fast, 1, "5", "string -> i64_fast"], + [echoPtr, validView, symbol, "Symbol -> ptr"], + [echoPtr, validView, plainObject, "object -> ptr"], + [echoPtr, validView, "hello", "JS string -> ptr (only cstring transcodes)"], + [echoPtr, validView, array, "array -> ptr"], + [echoPtr, validView, jsFunction, "JS function -> ptr"], + [echoPtr, validView, proxy, "proxy -> ptr"], + // (BigInt -> ptr / cstring is ACCEPTED as an exact 64-bit address -- + // oven-sh/bun#22751, #28068 -- and is covered by ffi-pointers-and-buffers.js.) + [echoPtr, validView, true, "boolean -> ptr"], + [echoCString, validView, symbol, "Symbol -> cstring"], + [echoCString, validView, plainObject, "object -> cstring"], + [echoCString, validView, true, "boolean -> cstring"], + [bufferArg, validView, 5, "number -> buffer (buffer requires a view)"], + [bufferArg, validView, null, "null -> buffer"], + [bufferArg, validView, undefined, "undefined -> buffer"], + [bufferArg, validView, plainObject, "object -> buffer"], + [bufferArg, validView, "abc", "string -> buffer"], + [bufferArg, validView, new ArrayBuffer(8), "ArrayBuffer -> buffer (not a view)"], + [bufferArg, validView, symbol, "Symbol -> buffer"], + [functionArg, validCallback, "cb", "JS string -> function"], + [functionArg, validCallback, plainObject, "object -> function"], + [functionArg, validCallback, jsFunction, "raw JS function -> function (must be a JSFFICallback)"], + [functionArg, validCallback, symbol, "Symbol -> function"], + [functionArg, validCallback, true, "boolean -> function"], + ]; + + // ---- The loose-coercion contract (bun parity): [callable, input, expected, label]. + // These MUST NOT throw; they pin the exact coerced value the callee receives. + function checkCoercion(actual, expected, label) { + if (Number.isNaN(expected) ? !Number.isNaN(actual) : !Object.is(actual, expected)) + throw new Error("coercion " + label + ": expected " + describe(expected) + " but got " + describe(actual)); + } + const coercions = [ + [echoI32, plainObject, 42, "object.valueOf -> i32"], + [echoI32, array, 0, "array -> i32 (Number([1,2,3]) = NaN -> 0)"], + [echoI32, 10n, 10, "BigInt -> i32"], + [echoI32, true, 1, "true -> i32"], + [echoI32, null, 0, "null -> i32"], + [echoI32, undefined, 0, "undefined -> i32"], + [echoI32, 4294902015, -65281, "u32 pattern into i32 wraps (bun#7007 class)"], + [echoU8, 300, 44, "u8 wraps mod 256 (300 -> 44), never clamps"], + [echoU8, -1, 255, "u8 wraps negative (-1 -> 255)"], + [echoI16, jsFunction, 0, "function -> i16 (Number(fn) = NaN -> 0)"], + [echoBool, plainObject, true, "object -> bool"], + [echoBool, 1n, true, "BigInt 1n -> bool"], + [echoF64, plainObject, 42, "object.valueOf -> f64"], + [echoF64, true, 1, "true -> f64"], + [echoF64, null, 0, "null -> f64"], + [echoF64, undefined, NaN, "undefined -> f64 (Number(undefined) = NaN)"], + [echoF64, 2n, 2, "BigInt -> f64 (Number(5n)-style)"], + [echoF32, undefined, NaN, "undefined -> f32"], + [echoF32, null, 0, "null -> f32"], + [echoI64, 5, 5n, "number -> i64"], + [echoI64, 5n, 5n, "BigInt -> i64"], + ]; + for (const [callable, input, expected, label] of coercions) { + let actual; + try { + actual = callable(input); + } catch (e) { + throw new Error("coercion " + label + ": threw " + e); + } + checkCoercion(actual, expected, "cold " + label); + } + // ...and after tier-up the SAME coercions produce the SAME values. + for (const [callable, input, expected, label] of coercions) { + const caller = makeMonomorphicCaller(1); + for (let i = 0; i < 5000; ++i) + caller(callable, [input]); + checkCoercion(caller(callable, [input]), expected, "hot " + label); + } + + const coldMessages = new Map(); + for (const [callable, good, bad, label] of cases) { + // Sanity: the valid argument works. + callable(good); + let error = null; + try { + callable(bad); + } catch (e) { + error = e; + } + if (error === null) + throw new Error("cold: " + label + ": " + describe(bad) + " did not throw"); + if (!(error instanceof TypeError)) + throw new Error("cold: " + label + ": expected a TypeError, got " + error); + if (typeof error.message !== "string" || !error.message.length) + throw new Error("cold: " + label + ": TypeError has no message"); + coldMessages.set(label, error.message); + // The function must remain usable. + callable(good); + } + + // Warm every callable with valid arguments through its own monomorphic + // caller so that caller tiers up with a converted CallFFI site, then + // re-trigger the same error through the SAME (optimized) call site and + // demand the identical message. + for (const [callable, good, bad, label] of cases) { + const caller = makeMonomorphicCaller(1); + const goodArgs = [good]; + const badArgs = [bad]; + for (let i = 0; i < 3000; ++i) + caller(callable, goodArgs); + for (let i = 0; i < 150; ++i) { + let error = null; + try { + caller(callable, badArgs); + } catch (e) { + error = e; + } + if (error === null) + throw new Error("hot: " + label + " iteration " + i + " did not throw"); + if (!(error instanceof TypeError)) + throw new Error("hot: " + label + " iteration " + i + ": expected a TypeError, got " + error); + if (error.message !== coldMessages.get(label)) + throw new Error("hot: " + label + " iteration " + i + ": message \"" + error.message + "\" != cold \"" + coldMessages.get(label) + "\""); + // Interleave valid calls so the site stays optimized. + caller(callable, goodArgs); + } + } + + // A single hot function that alternates good and bad values (the same + // compiled CallFFI site takes both the fast and the throwing slow path). + function guarded(value) { + try { + return { ok: true, value: echoI32(value) }; + } catch (e) { + return { ok: false, error: e }; + } + } + noInline(guarded); + for (let i = 0; i < 8000; ++i) { + const result = guarded(i); + if (!result.ok || result.value !== (i | 0)) + throw new Error("guarded warm iteration " + i); + } + for (let i = 0; i < 3000; ++i) { + const bad = (i % 5) === 4; + const result = guarded(bad ? symbol : i); + if (bad) { + if (result.ok) + throw new Error("guarded(Symbol) did not throw at iteration " + i); + if (!(result.error instanceof TypeError)) + throw new Error("guarded(Symbol) wrong error at iteration " + i + ": " + result.error); + if (result.error.message !== coldMessages.get("Symbol -> i32")) + throw new Error("guarded(Symbol) message differs from the cold message at iteration " + i); + } else if (!result.ok || result.value !== (i | 0)) + throw new Error("guarded good iteration " + i); + } + + // FFI-SPEC-GAP: SPEC section 11.4 lists "detached buffer as ptr" among + // the TypeError cases, but the normative conversion table (section 5) + // says "vector() (0 if detached)". The normative rule wins here: detached + // views convert to a null pointer in every tier, without throwing. + for (let i = 0; i < 3; ++i) { + if (echoPtr(detachedView) !== null) + throw new Error("detached view as ptr should yield null (iteration " + i + ")"); + if (bufferArg(detachedView) !== null) + throw new Error("detached view as buffer should yield null (iteration " + i + ")"); + if (echoCString(detachedView) !== null) + throw new Error("detached view as cstring should yield null (iteration " + i + ")"); + } + for (let i = 0; i < 5000; ++i) { + if (echoPtr(i & 1 ? detachedView : validView) === undefined) + throw new Error("unreachable"); + } + if (echoPtr(detachedView) !== null) + throw new Error("detached view as ptr should yield null when hot"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-fuzz-signatures.js b/JSTests/stress/ffi-fuzz-signatures.js new file mode 100644 index 0000000000000..b4d8e87e31b7f --- /dev/null +++ b/JSTests/stress/ffi-fuzz-signatures.js @@ -0,0 +1,320 @@ +//@ requireOptions("--useDollarVM=1") + +// Seeded fuzz (500+ cases) over the echo/add/sum fixtures with random edge +// values, verified against a JS reference implementation of the SPEC +// section 5 conversion rules in both directions. + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (typeof value === "symbol") + return value.toString(); + if (Object.is(value, -0)) + return "-0"; + return String(value); +} + +// Returns a NEW caller function every time (a distinct FunctionExecutable / +// CodeBlock via `new Function`), so the FFI call site inside it is +// monomorphic, exact-arity and non-spread -- the only call-site shape the DFG +// Call -> CallFFI conversion accepts (SPEC section 10.2). The hot phase below +// gives each callee its own caller so the optimized typed path is what the +// random edges keep flowing through. +function makeMonomorphicCaller(arity) { + const argumentList = Array.from({ length: arity }, (_, i) => "args[" + i + "]").join(", "); + return new Function("callable", "args", "return callable(" + argumentList + ");"); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (fixtureName, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(fixtureName), fixtureName + "->" + ret); + + // ---- Deterministic PRNG (xorshift128+ over two 64-bit BigInt states, simplified via mulberry32). + let state = 0x0badc0de | 0; + function random() { + state = (state + 0x6D2B79F5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + } + const randomIntBits = () => Math.floor(random() * 4294967296) - 2147483648; // uniform int32 + const pick = list => list[Math.floor(random() * list.length)]; + + // ---- JS reference of SPEC section 5. + const twoTo32 = 4294967296; + const twoTo64 = 2n ** 64n; + const twoTo63 = 2n ** 63n; + const MAX_INT52 = 9007199254740991; + // ECMAScript ToInt32 for the value classes we generate (number, boolean, undefined, null). + function toInt32(v) { + if (typeof v === "boolean") + return v ? 1 : 0; + if (v === undefined || v === null) + return 0; + const n = Number(v); + if (!Number.isFinite(n)) + return 0; + const t = Math.trunc(n) % twoTo32; + const u = t < 0 ? t + twoTo32 : t; + const r = u >= 2147483648 ? u - twoTo32 : u; + // Math.trunc(-0.999999) is -0 and survives the arithmetic above; ECMAScript + // ToInt32(-0.999999) is +0, and Object.is distinguishes 0 from -0, so + // normalize (r is always an integer here, so `+ 0` only flips -0 to +0). + return r + 0; + } + // Both hardware truncations agree exactly when |d| < 2^63; the fuzzer only + // generates such doubles (the arch-specific saturation edges live in + // testFFI's doubleToInt64 corpus). + function doubleToInt64(d) { + return BigInt(Math.trunc(d)); + } + const reference = { + "char": v => (toInt32(v) << 24) >> 24, + "i8": v => (toInt32(v) << 24) >> 24, + "u8": v => toInt32(v) & 0xff, + "i16": v => (toInt32(v) << 16) >> 16, + "u16": v => toInt32(v) & 0xffff, + "i32": v => toInt32(v), + "u32": v => toInt32(v) >>> 0, + "bool": v => Boolean(v), + "i64": v => { + if (typeof v === "bigint") + return BigInt.asIntN(64, v); + if (Number.isInteger(v) && Math.abs(v) <= 2147483647) + return BigInt(v); // int32 -> sign-extend + return BigInt.asIntN(64, doubleToInt64(v)); + }, + "u64": v => { + if (typeof v === "bigint") + return BigInt.asUintN(64, v); + if (Number.isInteger(v) && Math.abs(v) <= 2147483647) + return BigInt.asUintN(64, BigInt(v)); // int32 -> sign-extend then reinterpret + return BigInt.asUintN(64, doubleToInt64(v)); + }, + "i64_fast": v => { + const r = reference["i64"](v); + return (r >= BigInt(-MAX_INT52) && r <= BigInt(MAX_INT52)) ? Number(r) : r; + }, + "u64_fast": v => { + const r = reference["u64"](v); + return r < BigInt(MAX_INT52) ? Number(r) : r; + }, + // Bun parity: plain Number(); f32 is Math.fround of the same. + "f64": v => Number(v), + "f32": v => Math.fround(Number(v)), + "ptr": v => { + let bits; + if (v === null || v === undefined) + bits = 0n; + else if (Number.isInteger(v) && Math.abs(v) <= 2147483647) + bits = BigInt.asUintN(64, BigInt(v)); + else + bits = BigInt.asUintN(64, doubleToInt64(v)); + if (bits === 0n) + return null; + // Addresses above 2^53 are surfaced as an exact BigInt (bun#28068). + return bits <= 9007199254740991n ? Number(bits) : bits; + }, + }; + reference["cstring"] = value => (value === null || value === undefined ? null : String(value)); + + // ---- Value generators (per FFI type). + const int32Edges = [0, 1, -1, 2147483647, -2147483648, 2147483646, -2147483647, 65535, 65536, -65536, 255, 256, 127, 128, -128, -129, 32767, 32768, -32768]; + const doubleEdges = [0, -0, 0.5, -0.5, 1.5, -1.5, 2.5, 0.999999, -0.999999, 2 ** 31, -(2 ** 31), 2 ** 32 + 5, -(2 ** 32) - 5, 2 ** 52, 2 ** 53, 2 ** 53 - 1, -(2 ** 53), 2 ** 62, -(2 ** 62), 1e15 + 0.75, -1e15 - 0.75, NaN, Infinity, -Infinity, Number.MAX_VALUE, Number.MIN_VALUE, Number.EPSILON]; + const bigIntEdges = [0n, 1n, -1n, twoTo63 - 1n, -twoTo63, twoTo63, twoTo64 - 1n, twoTo64, twoTo64 + 12345n, -twoTo64, 2n ** 100n + 7n, -(2n ** 90n), 9007199254740993n, 4611686018427387904n]; + const oddballs = [true, false, undefined, null]; + function genFor(type) { + switch (type) { + case "char": case "i8": case "u8": case "i16": case "u16": case "i32": case "u32": case "bool": + switch (Math.floor(random() * 4)) { + case 0: return pick(int32Edges); + case 1: return pick(doubleEdges); + case 2: return pick(oddballs); + default: return randomIntBits() * (random() < 0.5 ? 1 : 2.3); + } + case "i64": case "u64": case "i64_fast": case "u64_fast": { + switch (Math.floor(random() * 4)) { + case 0: return pick(int32Edges); + case 1: return pick(bigIntEdges); + case 2: return BigInt.asIntN(64, BigInt(randomIntBits()) * BigInt(randomIntBits()) * 4294967311n); + default: { + // doubles strictly inside (-2^63, 2^63) so both hardware truncations agree + const d = pick(doubleEdges.filter(x => Number.isFinite(x) && Math.abs(x) < 9007199254740992 * 512)); + return d; + } + } + } + case "f64": + return random() < 0.8 ? pick(doubleEdges) : randomIntBits() / (1 + Math.floor(random() * 7)); + case "f32": + return random() < 0.7 ? pick(doubleEdges) : randomIntBits() / 8; + case "cstring": + switch (Math.floor(random() * 5)) { + case 0: return pick([null, undefined]); + case 1: return ""; + case 2: return pick(["a", "hello", "with space", "0123456789".repeat(20)]); + case 3: return pick(["h\u00e9!", "\u2603 snowman", "\u{1F600} astral", "mix\u00e9d\u2603up"]); + default: return String(randomIntBits()); + } + case "ptr": + switch (Math.floor(random() * 4)) { + case 0: return pick([0, null, undefined, 4096, 65535, 0x7fffffff, -1, -4096]); + case 1: return pick([2 ** 40, 2 ** 47 - 1, 140737488355327, 0x00007fffdeadbee0]); + case 2: return Math.floor(random() * 2 ** 46); + default: return -Math.floor(random() * 2 ** 30); + } + } + throw new Error("no generator for " + type); + } + + // ---- Fixture bindings by declared FFI type (echo family). + const echoBindings = { + "char": bind("ffi_echo_char", ["char"], "char"), + "i8": bind("ffi_echo_i8", ["i8"], "i8"), + "u8": bind("ffi_echo_u8", ["u8"], "u8"), + "i16": bind("ffi_echo_i16", ["i16"], "i16"), + "u16": bind("ffi_echo_u16", ["u16"], "u16"), + "i32": bind("ffi_echo_i32", ["i32"], "i32"), + "u32": bind("ffi_echo_u32", ["u32"], "u32"), + "bool": bind("ffi_echo_bool", ["bool"], "bool"), + "i64": bind("ffi_echo_i64", ["i64"], "i64"), + "u64": bind("ffi_echo_u64", ["u64"], "u64"), + "i64_fast": bind("ffi_echo_i64", ["i64_fast"], "i64_fast"), + "u64_fast": bind("ffi_echo_u64", ["u64_fast"], "u64_fast"), + "f64": bind("ffi_echo_f64", ["f64"], "f64"), + "f32": bind("ffi_echo_f32", ["f32"], "f32"), + "ptr": bind("ffi_echo_ptr", ["ptr"], "ptr"), + "cstring": bind("ffi_echo_cstring", ["cstring"], "cstring"), + }; + // Echo semantics: the native fixture returns its argument unchanged, so the + // result is the JS->native argument conversion followed by the native->JS + // return boxing of the same type. + function echoReference(type, value) { + const asArgument = reference[type](value); + switch (type) { + case "bool": + return asArgument; // already a boolean + case "ptr": + return asArgument; // null or number + case "cstring": + return asArgument === null || asArgument === undefined ? null : String(asArgument); + default: + return asArgument; + } + } + const echoTypes = Object.keys(echoBindings); + + // ---- Two-argument adders. + const addI32 = bind("ffi_add_i32", ["i32", "i32"], "i32"); + const addF64 = bind("ffi_add_f64", ["f64", "f64"], "f64"); + const addI64 = bind("ffi_add_i64", ["i64", "i64"], "i64"); + const addU64 = bind("ffi_add_u64", ["u64", "u64"], "u64"); + const addF32 = bind("ffi_add_f32", ["f32", "f32"], "f32"); + const adders = [ + ["i32", addI32, (a, b) => (reference["i32"](a) + reference["i32"](b)) | 0], + ["f64", addF64, (a, b) => reference["f64"](a) + reference["f64"](b)], + ["i64", addI64, (a, b) => BigInt.asIntN(64, reference["i64"](a) + reference["i64"](b))], + ["u64", addU64, (a, b) => BigInt.asUintN(64, reference["u64"](a) + reference["u64"](b))], + ["f32", addF32, (a, b) => Math.fround(Math.fround(a) + Math.fround(b))], + ]; + + // ---- Sum ladders. + const sumI32_16 = bind("ffi_sum_i32_16", new Array(16).fill("i32"), "i64"); + const sumF64_12 = bind("ffi_sum_f64_12", new Array(12).fill("f64"), "f64"); + const sumU8_12 = bind("ffi_sum_u8_12", new Array(12).fill("u8"), "i64"); + const sumI16_12 = bind("ffi_sum_i16_12", new Array(12).fill("i16"), "i64"); + + let executed = 0; + let mismatches = 0; + function fail(message) { + mismatches++; + throw new Error(message); + } + function verify(label, actual, expected) { + executed++; + if (!Object.is(actual, expected)) + fail(label + ": expected " + describe(expected) + " but got " + describe(actual)); + } + + const totalCases = 600; + for (let caseIndex = 0; caseIndex < totalCases; ++caseIndex) { + const kind = random(); + if (kind < 0.55) { + // Echo case. + const type = pick(echoTypes); + const value = genFor(type); + const expected = echoReference(type, value); + const actual = echoBindings[type](value); + verify("echo " + type + "(" + describe(value) + ")", actual, expected); + } else if (kind < 0.8) { + // Adder case. + const [type, fn, ref] = pick(adders); + const a = genFor(type === "f32" ? "f32" : type); + const b = genFor(type === "f32" ? "f32" : type); + verify(type + " add(" + describe(a) + ", " + describe(b) + ")", fn(a, b), ref(a, b)); + } else if (kind < 0.9) { + // 16-way int32 ladder. + const values = new Array(16).fill(0).map(() => genFor("i32")); + let expected = 0n; + for (const v of values) + expected += BigInt(reference["i32"](v)); + verify("sum_i32_16(" + values.map(describe).join(",") + ")", sumI32_16(...values), expected); + } else if (kind < 0.95) { + // 12-way double ladder (finite dyadic values). Fold left-to-right + // starting from the first operand exactly like the fixture's + // `a0 + a1 + ... + a11`, so even the sign of a zero sum matches. + const values = new Array(12).fill(0).map(() => Math.round(random() * 1024 - 512) / 8); + let expected = values[0]; + for (let i = 1; i < values.length; ++i) + expected += values[i]; + verify("sum_f64_12(" + values.join(",") + ")", sumF64_12(...values), expected); + } else { + // Sub-8-byte stack ladders. + const useSigned = random() < 0.5; + const type = useSigned ? "i16" : "u8"; + const fn = useSigned ? sumI16_12 : sumU8_12; + const values = new Array(12).fill(0).map(() => genFor(type)); + let expected = 0n; + for (const v of values) + expected += BigInt(reference[type](v)); + verify("sum_" + type + "_12(" + values.map(describe).join(",") + ")", fn(...values), expected); + } + } + if (executed !== totalCases || mismatches !== 0) + throw new Error("fuzz bookkeeping: executed " + executed + ", mismatches " + mismatches); + + // A second, tighter phase: monomorphic random calls per callee, each + // through its own dedicated exact-arity caller, so the callers tier up + // (typed CallFFI sites) while the fuzzer keeps feeding random edges + // through the optimized code. + for (const [type, fn, ref] of adders) { + const caller = makeMonomorphicCaller(2); + const args = [0, 0]; + for (let i = 0; i < 3000; ++i) { + const a = genFor(type === "f32" ? "f32" : type); + const b = genFor(type === "f32" ? "f32" : type); + args[0] = a; + args[1] = b; + const actual = caller(fn, args); + const expected = ref(a, b); + if (!Object.is(actual, expected)) + throw new Error("hot fuzz " + type + " add(" + describe(a) + ", " + describe(b) + "): expected " + describe(expected) + " but got " + describe(actual) + " at iteration " + i); + } + } + for (const type of echoTypes) { + const fn = echoBindings[type]; + const caller = makeMonomorphicCaller(1); + const args = [0]; + for (let i = 0; i < 2000; ++i) { + const value = genFor(type); + args[0] = value; + const actual = caller(fn, args); + const expected = echoReference(type, value); + if (!Object.is(actual, expected)) + throw new Error("hot fuzz echo " + type + "(" + describe(value) + "): expected " + describe(expected) + " but got " + describe(actual) + " at iteration " + i); + } + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-hooks-and-owner.js b/JSTests/stress/ffi-hooks-and-owner.js new file mode 100644 index 0000000000000..b01c6484d44d3 --- /dev/null +++ b/JSTests/stress/ffi-hooks-and-owner.js @@ -0,0 +1,53 @@ +//@ requireOptions("--useDollarVM=1") +// JSFFIFunction owner cell + CallHooks: +// 1) hooks bracket EVERY call as before:N / after:N with the token round-tripping; +// 2) after fires even when the call throws (a callback raised an exception mid-call); +// 3) a hooked function is host-path-only: it must NOT be lifted into a CallFFI node -- observable +// as the hooks still firing on every call after the caller is FTL-hot (a CallFFI node would +// bypass ffiHostCall, the only place hooks run, so the log would go quiet); +// 4) the owner is kept alive by the function (WeakRef stays populated while the fn is reachable). +if (!$vm.useJIT()) quit(); + +const owner = { hookLog: [] }; +const addI32 = $vm.ffiFunction({ args: ["i32", "i32"], returns: "i32" }, $vm.ffiFixture("ffi_add_i32"), "add_i32", { owner, hooks: "test" }); + +// (1) bracketing + token round trip +addI32(1, 2); +if (owner.hookLog.length !== 2) throw new Error("expected before+after, got " + JSON.stringify(owner.hookLog)); +const t = owner.hookLog[0].split(":")[1]; +if (owner.hookLog[0] !== "before:" + t || owner.hookLog[1] !== "after:" + t) + throw new Error("bad bracket order/token: " + JSON.stringify(owner.hookLog)); + +// (2) after runs even when the native call throws (callback throws inside the call) +const callCbVoid = $vm.ffiFunction({ args: ["ptr"], returns: "void" }, $vm.ffiFixture("ffi_call_cb_void"), "call_cb_void", { owner, hooks: "test" }); +const boom = $vm.ffiCallback({ args: [], returns: "void" }, () => { throw new RangeError("cb"); }); +owner.hookLog.length = 0; +let threw = false; +try { callCbVoid(boom.ptr); } catch (e) { threw = e instanceof RangeError; } +if (!threw) throw new Error("callback exception did not propagate"); +if (owner.hookLog.length !== 2 || !owner.hookLog[1].startsWith("after:")) + throw new Error("after hook did not run on the throwing call: " + JSON.stringify(owner.hookLog)); + +// (3) host-path-only: hooks keep firing on every call even when the caller is FTL-hot. +function hot(a, b) { return addI32(a, b); } +noInline(hot); +owner.hookLog.length = 0; +let sum = 0; +for (let i = 0; i < 20000; ++i) sum += hot(i, 1); +const expected = (19999 * 20000) / 2 + 20000; // sum over i in [0,20000) of (i+1) = 200010000 +if (sum !== expected) throw new Error("wrong sum " + sum + " != " + expected); +if (owner.hookLog.length !== 40000) + throw new Error("hooks stopped firing when hot (CallFFI took over?): " + owner.hookLog.length + " entries for 20000 calls"); + +// (4) owner liveness: the function alone must keep its owner reachable. +let ref; +(function () { + const localOwner = { hookLog: [] }; + ref = new WeakRef(localOwner); + globalThis.keptFn = $vm.ffiFunction({ args: ["i32"], returns: "i32" }, $vm.ffiFixture("ffi_echo_i32"), "echo42", { owner: localOwner, hooks: "test" }); +})(); +for (let i = 0; i < 5; ++i) { fullGC(); edenGC(); } +if (ref.deref() === undefined) throw new Error("owner was collected while its function is still reachable"); +if (keptFn(42) !== 42) throw new Error("kept function no longer callable"); +globalThis.keptFn = null; +print("ffi hooks + owner: all checks passed"); diff --git a/JSTests/stress/ffi-host-path.js b/JSTests/stress/ffi-host-path.js new file mode 100644 index 0000000000000..3f1ed87aba672 --- /dev/null +++ b/JSTests/stress/ffi-host-path.js @@ -0,0 +1,222 @@ +//@ requireOptions("--useDollarVM=1", "--useFFIICStub=0", "--useFFICallInDFG=0") + +// FFI-SPEC-GAP: SPEC section 11.4 describes the host-path-vs-tiers +// differential as "a single file"; this companion file (row T also owns +// JSTests/stress/ffi-*.js) forces the host path with the option pair the +// spec names, so that path is exercised by the harness on every run. +// The C++ host-call path (SPEC section 8.2) only: no IC stub, no CallFFI. +// This is the same battery and the same HARDCODED expected table as +// ffi-tier-differential.js, so the host path is pinned to exactly the same +// answers the JIT tiers must produce (SPEC section 11.4). + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (Object.is(value, -0)) + return "-0"; + if (typeof value === "symbol") + return value.toString(); + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +// Returns a NEW caller function every time (a distinct FunctionExecutable / +// CodeBlock via `new Function`), so the FFI call site inside it is +// monomorphic, exact-arity and non-spread. Under this file's options the +// callee is always the C++ host path, but the caller shape is kept identical +// to ffi-tier-differential.js so the two differ only in the option pair. +function makeMonomorphicCaller(arity) { + const argumentList = Array.from({ length: arity }, (_, i) => "args[" + i + "]").join(", "); + return new Function("callable", "args", "return callable(" + argumentList + ");"); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (name, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(name), name); + + const echoChar = bind("ffi_echo_char", ["char"], "char"); + const echoI8 = bind("ffi_echo_i8", ["i8"], "i8"); + const echoU8 = bind("ffi_echo_u8", ["u8"], "u8"); + const echoI16 = bind("ffi_echo_i16", ["i16"], "i16"); + const echoU16 = bind("ffi_echo_u16", ["u16"], "u16"); + const echoI32 = bind("ffi_echo_i32", ["i32"], "i32"); + const echoU32 = bind("ffi_echo_u32", ["u32"], "u32"); + const echoI64 = bind("ffi_echo_i64", ["i64"], "i64"); + const echoU64 = bind("ffi_echo_u64", ["u64"], "u64"); + const echoI64Fast = $vm.ffiFunction({ args: ["i64_fast"], returns: "i64_fast" }, fixture("ffi_echo_i64"), "ffi_echo_i64:fast"); + const echoU64Fast = $vm.ffiFunction({ args: ["u64_fast"], returns: "u64_fast" }, fixture("ffi_echo_u64"), "ffi_echo_u64:fast"); + const echoF32 = bind("ffi_echo_f32", ["f32"], "f32"); + const echoF64 = bind("ffi_echo_f64", ["f64"], "f64"); + const echoBool = bind("ffi_echo_bool", ["bool"], "bool"); + const echoPtr = bind("ffi_echo_ptr", ["ptr"], "ptr"); + const echoNapiValue = bind("ffi_echo_jsvalue", ["napi_value"], "napi_value"); + const addI32 = bind("ffi_add_i32", ["i32", "i32"], "i32"); + const addF64 = bind("ffi_add_f64", ["f64", "f64"], "f64"); + const addI64 = bind("ffi_add_i64", ["i64", "i64"], "i64"); + const addU64 = bind("ffi_add_u64", ["u64", "u64"], "u64"); + const addF32 = bind("ffi_add_f32", ["f32", "f32"], "f32"); + const sumI32_9 = bind("ffi_sum_i32_9", ["i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32"], "i64"); + const sumF64_9 = bind("ffi_sum_f64_9", ["f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64"], "f64"); + const sumU8_12 = bind("ffi_sum_u8_12", ["u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8"], "i64"); + const sumI16_10 = bind("ffi_sum_i16_10", ["i16", "i16", "i16", "i16", "i16", "i16", "i16", "i16", "i16", "i16"], "i64"); + const mix1 = bind("ffi_mix_1", ["i32", "f64", "i64", "f32", "ptr", "u8", "f64", "i16", "f64", "i32"], "f64"); + const mix6 = bind("ffi_mix_6", ["bool", "bool", "i32", "bool", "f64", "bool", "f32", "bool", "bool", "bool", "bool", "bool", "bool"], "f64"); + const widenChar = bind("ffi_widen_char", ["char"], "i64_fast"); + const widenU16 = bind("ffi_widen_u16", ["u16"], "i64_fast"); + const twoAsBool = bind("ffi_ret_two_as_bool", [], "bool"); + const retNullPtr = bind("ffi_ret_null_ptr", [], "ptr"); + const highPtr = bind("ffi_high_ptr", [], "ptr"); + const retNegOneI8 = bind("ffi_ret_neg_one_i8", [], "i8"); + const retNegOneU32 = bind("ffi_ret_neg_one_u32", [], "u32"); + const retNegOneU64 = bind("ffi_ret_neg_one_u64", [], "u64"); + const retDenormalF32 = bind("ffi_ret_denormal_f32", [], "f32"); + const retNegZeroF64 = bind("ffi_ret_neg_zero_f64", [], "f64"); + const retInfF64 = bind("ffi_ret_inf_f64", [], "f64"); + + const sharedObject = { shared: true }; + + // Each row: [callable, [arguments...], expectedLiteral, label] + // Every expected value below is a literal, not a computed reference. + const battery = [ + [echoChar, [-1], -1, "char(-1)"], + [echoChar, [255], -1, "char(255)"], + [echoChar, [0x80], -128, "char(0x80)"], + [echoI8, [127], 127, "i8(127)"], + [echoI8, [128], -128, "i8(128)"], + [echoI8, [0x1ff], -1, "i8(0x1ff)"], + [echoU8, [-1], 255, "u8(-1)"], + [echoU8, [511], 255, "u8(511)"], + [echoU8, [256], 0, "u8(256)"], + [echoI16, [32768], -32768, "i16(32768)"], + [echoI16, [-32769], 32767, "i16(-32769)"], + [echoU16, [-1], 65535, "u16(-1)"], + [echoU16, [70000], 4464, "u16(70000)"], + [echoI32, [2147483648], -2147483648, "i32(2^31)"], + [echoI32, [-2147483649], 2147483647, "i32(-2^31-1)"], + [echoI32, [4294967301], 5, "i32(2^32+5)"], + [echoI32, [-1.9], -1, "i32(-1.9)"], + [echoI32, [NaN], 0, "i32(NaN)"], + [echoI32, [Infinity], 0, "i32(Infinity)"], + [echoI32, [undefined], 0, "i32(undefined)"], + [echoI32, [true], 1, "i32(true)"], + [echoU32, [-1], 4294967295, "u32(-1)"], + [echoU32, [2147483648], 2147483648, "u32(2^31)"], + [echoU32, [4294967296], 0, "u32(2^32)"], + [echoI64, [0], 0n, "i64(0)"], + [echoI64, [-1], -1n, "i64(-1)"], + [echoI64, [4294967296], 4294967296n, "i64(2^32)"], + [echoI64, [2n ** 63n - 1n], 9223372036854775807n, "i64(2^63-1)"], + [echoI64, [2n ** 63n], -9223372036854775808n, "i64(2^63)"], + [echoI64, [-1.5], -1n, "i64(-1.5)"], + [echoI64, [9007199254740992], 9007199254740992n, "i64(2^53 as number)"], + [echoU64, [-1], 18446744073709551615n, "u64(-1)"], + [echoU64, [-2147483648], 18446744071562067968n, "u64(-2^31)"], + [echoU64, [2n ** 64n + 3n], 3n, "u64(2^64+3)"], + [echoI64Fast, [9007199254740991], 9007199254740991, "i64_fast(2^53-1)"], + [echoI64Fast, [-9007199254740991], -9007199254740991, "i64_fast(-(2^53-1))"], + [echoI64Fast, [2n ** 53n], 9007199254740992n, "i64_fast(2^53)"], + [echoI64Fast, [-(2n ** 53n)], -9007199254740992n, "i64_fast(-2^53)"], + [echoI64Fast, [-1], -1, "i64_fast(-1)"], + [echoU64Fast, [9007199254740990], 9007199254740990, "u64_fast(2^53-2)"], + [echoU64Fast, [2n ** 53n - 1n], 9007199254740991n, "u64_fast(2^53-1)"], + [echoU64Fast, [-1], 18446744073709551615n, "u64_fast(-1)"], + [echoF32, [1.1], 1.100000023841858, "f32(1.1)"], + [echoF32, [-0], -0, "f32(-0)"], + [echoF32, [NaN], NaN, "f32(NaN)"], + [echoF32, [1e39], Infinity, "f32(1e39)"], + [echoF32, [16777217], 16777216, "f32(2^24+1)"], + [echoF64, [-0], -0, "f64(-0)"], + [echoF64, [NaN], NaN, "f64(NaN)"], + [echoF64, [Number.MIN_VALUE], 5e-324, "f64(min denormal)"], + [echoF64, [undefined], NaN, "f64(undefined) -> NaN"], + [echoBool, [2], true, "bool(2)"], + [echoBool, [-1], true, "bool(-1)"], + [echoBool, [0], false, "bool(0)"], + [echoBool, [0.5], true, "bool(0.5)"], + [echoBool, [-0], false, "bool(-0)"], + [echoBool, [NaN], false, "bool(NaN)"], + [echoBool, [256], true, "bool(256)"], + [echoBool, [null], false, "bool(null)"], + [echoPtr, [0], null, "ptr(0)"], + [echoPtr, [null], null, "ptr(null)"], + [echoPtr, [undefined], null, "ptr(undefined)"], + [echoPtr, [-1], 18446744073709551615n, "ptr(-1) (exact BigInt, > 2^53)"], + [echoPtr, [1099511627776], 1099511627776, "ptr(2^40)"], + [echoNapiValue, [sharedObject], sharedObject, "napi_value(object)"], + [echoNapiValue, ["x"], "x", "napi_value(string)"], + [echoNapiValue, [-0], -0, "napi_value(-0)"], + [addI32, [2147483647, 1], -2147483648, "add_i32 overflow"], + [addI32, [-2147483648, -1], 2147483647, "add_i32 underflow"], + [addI32, [7], 7, "add_i32 missing argument"], + [addI32, [7, 8, 9], 15, "add_i32 extra argument"], + [addF64, [0.1, 0.2], 0.30000000000000004, "add_f64(0.1, 0.2)"], + [addF64, [-0, -0], -0, "add_f64(-0, -0)"], + [addF64, [Infinity, -Infinity], NaN, "add_f64(inf, -inf)"], + [addI64, [2n ** 63n - 1n, 1n], -9223372036854775808n, "add_i64 wrap"], + [addI64, [-1, -1], -2n, "add_i64(-1,-1)"], + [addU64, [-1, 2], 1n, "add_u64 wrap"], + [addU64, [2n ** 32n, 2n ** 32n], 8589934592n, "add_u64(2^32,2^32)"], + [addF32, [16777216, 1], 16777216, "add_f32 precision loss"], + [addF32, [0.5, 0.25], 0.75, "add_f32 dyadics"], + [addF32, [3.4e38, 3.4e38], Infinity, "add_f32 overflow to +inf"], + [sumI32_9, [1, -2, 3, -4, 5, -6, 7, -8, 100000], 99996n, "sum_i32_9"], + [sumI32_9, [2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647], 19327352823n, "sum_i32_9 max"], + [sumF64_9, [1, 0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, 0.0078125, 0.00390625], 1.99609375, "sum_f64_9 dyadics"], + [sumU8_12, [255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255], 3060n, "sum_u8_12 max"], + [sumU8_12, [1, 2, 4, 8, 16, 32, 64, 128, -1, 256, 257, 511], 766n, "sum_u8_12 wrapped powers"], + [sumI16_10, [-32768, -32768, 32767, 32767, -1, 1, 40000, -40000, 65535, 65536], -3n, "sum_i16_10 edges"], + [mix1, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 385, "mix_1 identity ramp"], + [mix1, [-2147483648, -0.5, -1000000, -1.5, 4096, 255, 0, -32768, 2, 2147483647], 19324112699, "mix_1 edges"], + [mix6, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 91, "mix_6 all ones"], + [mix6, [2, 0, -3, -1, -0.25, NaN, 0.5, 256, true, false, null, undefined, true], 28.25, "mix_6 truthiness edges"], + [widenChar, [-1], -1, "widen_char(-1)"], + [widenChar, [200], -56, "widen_char(200)"], + [widenU16, [-1], 65535, "widen_u16(-1)"], + [twoAsBool, [], true, "ret_two_as_bool"], + [retNullPtr, [], null, "ret_null_ptr"], + [highPtr, [], 0x00007fffdeadbee0, "high_ptr"], + [retNegOneI8, [], -1, "ret_neg_one_i8"], + [retNegOneU32, [], 4294967295, "ret_neg_one_u32"], + [retNegOneU64, [], 18446744073709551615n, "ret_neg_one_u64"], + [retDenormalF32, [], 2 ** -149, "ret_denormal_f32"], + [retNegZeroF64, [], -0, "ret_neg_zero_f64"], + [retInfF64, [], Infinity, "ret_inf_f64"], + ]; + + function runBattery(phase) { + for (const [callable, args, expected, label] of battery) { + const actual = callable(...args); + if (!Object.is(actual, expected)) + throw new Error(phase + " " + label + ": expected " + describe(expected) + " but got " + describe(actual)); + } + } + + // Cold pass (whatever tier the harness starts in). + runBattery("cold"); + // Warm each row through its OWN exact-arity, non-spread, single-callee + // caller (same shape as ffi-tier-differential.js), then re-run the whole + // battery. + for (const [callable, args, expected, label] of battery) { + const caller = makeMonomorphicCaller(args.length); + for (let i = 0; i < 4000; ++i) { + const actual = caller(callable, args); + if (!Object.is(actual, expected)) + throw new Error("warm " + label + " iteration " + i + ": expected " + describe(expected) + " but got " + describe(actual)); + } + } + runBattery("hot"); + // A few thousand mixed iterations across every row (megamorphic-ish). + for (let i = 0; i < 6000; ++i) { + const [callable, args, expected, label] = battery[i % battery.length]; + const actual = callable(...args); + if (!Object.is(actual, expected)) + throw new Error("mixed " + label + " iteration " + i + ": expected " + describe(expected) + " but got " + describe(actual)); + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-jsvalue.js b/JSTests/stress/ffi-jsvalue.js new file mode 100644 index 0000000000000..a81abd6925641 --- /dev/null +++ b/JSTests/stress/ffi-jsvalue.js @@ -0,0 +1,87 @@ +//@ requireOptions("--useDollarVM=1") + +// The "jsvalue" type is a raw EncodedJSValue pass-through in both directions: every JS value +// kind (objects, functions, symbols, -0, BigInt) must round-trip with identity intact. + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (typeof value === "symbol") + return value.toString(); + if (Object.is(value, -0)) + return "-0"; + if (value !== null && (typeof value === "object" || typeof value === "function")) { + // Object.create(null), Proxy, etc. may have no toString/valueOf, so + // String(value) would throw "No default value" while merely + // formatting a message; describe by structure instead. + const tag = Object.prototype.toString.call(value); + return typeof value === "function" ? "[function " + (value.name || "anonymous") + "]" : tag; + } + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const echoNapiValue = $vm.ffiFunction({ args: ["jsvalue"], returns: "jsvalue" }, fixture("ffi_echo_jsvalue"), "ffi_echo_jsvalue"); + + // ---- napi_value: identity of arbitrary JSValues in both directions. + const object = { deep: { array: [1, 2, 3] } }; + const array = [1, "two", 3n]; + const fn = function named() { return 1; }; + const symbol = Symbol("napi"); + const registrySymbol = Symbol.for("napi.registry"); + const bigint = 123456789012345678901234567890n; + const values = [ + object, array, fn, symbol, registrySymbol, bigint, 0, -0, 1, -1, 0.5, NaN, Infinity, -Infinity, + 2147483647, -2147483648, 2147483648, 4294967295, Number.MAX_SAFE_INTEGER, Number.MIN_VALUE, + true, false, null, undefined, "", "string", "\u{1F600}", 0n, -1n, + new Uint8Array(3), new ArrayBuffer(2), Object.freeze({}), Object.create(null), + echoNapiValue, // a JSFFIFunction itself + $vm.ffiCallback({ args: [], returns: "void" }, () => { }), // a JSFFICallback + new Proxy({}, {}), new Error("as a value"), Promise.resolve(1), new Map(), new WeakRef(object), + ]; + for (const value of values) { + const result = echoNapiValue(value); + check(result, value, "napi_value identity for " + describe(value)); + if ((typeof value === "object" && value !== null) || typeof value === "function" || typeof value === "symbol") { + if (result !== value) + throw new Error("napi_value must preserve object identity (===), got a different object for " + describe(value)); + } + } + // Missing napi_value argument: undefined bits pass through. + check(echoNapiValue(), undefined, "missing napi_value argument is undefined"); + // Hot identity through the tiers with a few classes of values. + for (let i = 0; i < 3e4; ++i) { + const value = values[i % values.length]; + const result = echoNapiValue(value); + if (!Object.is(result, value)) + throw new Error("hot napi_value identity iteration " + i + " for " + describe(value) + " got " + describe(result)); + } + // Values created inside the loop (young objects): identity, and no GC crash. + for (let i = 0; i < 5000; ++i) { + const young = { i, payload: new Array(8).fill(i) }; + if (echoNapiValue(young) !== young) + throw new Error("young object identity iteration " + i); + if ((i & 1023) === 0) + gc(); + } + // napi_value inside a callback: JS -> native -> JS receives the very same values. + const seen = []; + const cb = $vm.ffiCallback({ args: ["jsvalue"], returns: "jsvalue" }, v => { seen.push(v); return v; }); + const throughCallback = $vm.ffiFunction({ args: ["jsvalue"], returns: "jsvalue" }, cb, "napi_value round trip"); + for (const value of values) { + seen.length = 0; + const result = throughCallback(value); + check(result, value, "callback napi_value round trip for " + describe(value)); + check(seen.length, 1, "callback invoked once"); + check(seen[0], value, "callback saw the identical value"); + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-no-jit.js b/JSTests/stress/ffi-no-jit.js new file mode 100644 index 0000000000000..ceb7d891070cd --- /dev/null +++ b/JSTests/stress/ffi-no-jit.js @@ -0,0 +1,49 @@ +//@ runNoJIT +//@ requireOptions("--useDollarVM=1") + +// bun:ffi requires the JIT (SPEC section 0.1): with --useJIT=false the +// creation of JSFFIFunction / JSFFICallback must throw a TypeError with the +// message "bun:ffi requires the JIT", and nothing must crash. (When the +// harness runs this file with the JIT enabled anyway, creation must succeed.) + +function expectRequiresJIT(fn, label) { + let error = null; + try { + fn(); + } catch (e) { + error = e; + } + if (error === null) + throw new Error(label + ": expected a TypeError, nothing thrown"); + if (!(error instanceof TypeError)) + throw new Error(label + ": expected a TypeError, got " + error); + if (String(error.message).indexOf("bun:ffi requires the JIT") === -1) + throw new Error(label + ": unexpected message: " + error.message); +} + +const signature = { args: ["i32", "i32"], returns: "i32" }; +const callbackSignature = { args: ["i32"], returns: "i32" }; +const target = $vm.ffiFixture("ffi_add_i32"); + +if ($vm.useJIT()) { + // The harness may also run this file in JIT configurations: then creation + // works and the function is callable. + const add = $vm.ffiFunction(signature, target, "ffi_add_i32"); + if (add(40, 2) !== 42) + throw new Error("JIT configuration: ffi_add_i32(40, 2) !== 42"); + const cb = $vm.ffiCallback(callbackSignature, x => x + 1); + if (typeof cb.ptr !== "number") + throw new Error("JIT configuration: callback .ptr should be a number"); +} else { + for (let i = 0; i < 3; ++i) { + expectRequiresJIT(() => $vm.ffiFunction(signature, target, "ffi_add_i32"), "$vm.ffiFunction without JIT"); + expectRequiresJIT(() => $vm.ffiCallback(callbackSignature, x => x + 1), "$vm.ffiCallback without JIT"); + } + // Signature-only APIs and fixtures still work without the JIT. + if ($vm.ffiSignatureString(signature) !== "i32(i32,i32)") + throw new Error("ffiSignatureString should work without the JIT: " + $vm.ffiSignatureString(signature)); + if (typeof target !== "number" || !(target > 0)) + throw new Error("ffiFixture should return a pointer without the JIT: " + target); + if (!Array.isArray($vm.ffiFixtures()) || $vm.ffiFixtures().length < 90) + throw new Error("ffiFixtures should work without the JIT"); +} diff --git a/JSTests/stress/ffi-non-int32-int-args.js b/JSTests/stress/ffi-non-int32-int-args.js new file mode 100644 index 0000000000000..98c29a531e8d9 --- /dev/null +++ b/JSTests/stress/ffi-non-int32-int-args.js @@ -0,0 +1,23 @@ +//@ requireOptions("--useDollarVM=1") +// An integer FFI parameter validly accepts doubles / booleans / null (the conversion table), +// so a call site that passes such values must NOT get an unconditional Int32Use check (which +// would OSR-exit every call -> a deopt storm). The site must stay compiled and agree with the +// interpreter. We assert (a) tier agreement and (b) that the function is not endlessly recompiled. +if (!$vm.useJIT()) quit(); +const echoI32 = $vm.ffiFunction({ args: ["i32"], returns: "i32" }, $vm.ffiFixture("ffi_echo_i32"), "echo_i32"); +function ref(v) { return echoI32(v); } +function hot(v) { return echoI32(v); } +noDFG(ref); noInline(ref); noInline(hot); +let failures = 0; +const args = [true, false, null, undefined, 0.5, -1.5, 3.9, 2147483647.0, 1, 0]; +for (let i = 0; i < 200000; ++i) { + const a = args[i % args.length]; + const h = hot(a), r = ref(a); + if (!Object.is(h, r)) { print(`MISMATCH ${String(a)}: hot=${h} ref=${r}`); if (++failures > 4) throw new Error("tier mismatch"); } +} +// After 200k calls hot() must be optimized and STAY optimized (not exit-storming). +const compiles = numberOfDFGCompiles(hot); +print("DFG compiles of hot():", compiles); +if (compiles > 6) throw new Error(`hot() recompiled ${compiles} times -- deopt storm on valid non-int32 args`); +if (failures) throw new Error("failures"); +print("ffi non-int32 int args: all checks passed"); diff --git a/JSTests/stress/ffi-osr-and-exceptions.js b/JSTests/stress/ffi-osr-and-exceptions.js new file mode 100644 index 0000000000000..2a2a6aea1d3b4 --- /dev/null +++ b/JSTests/stress/ffi-osr-and-exceptions.js @@ -0,0 +1,244 @@ +//@ requireOptions("--useDollarVM=1") + +// FFI calls in optimized code: OSR-exit-inducing argument type changes midway +// through a hot loop, and exceptions (from callbacks and from argument +// conversion) thrown inside DFG/FTL-compiled code with and without a +// surrounding try/catch. Results must be identical before/after any exit. + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + String(expected) + " but got " + String(actual)); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const addI32 = $vm.ffiFunction({ args: ["i32", "i32"], returns: "i32" }, fixture("ffi_add_i32"), "ffi_add_i32"); + const addF64 = $vm.ffiFunction({ args: ["f64", "f64"], returns: "f64" }, fixture("ffi_add_f64"), "ffi_add_f64"); + const echoBool = $vm.ffiFunction({ args: ["bool"], returns: "bool" }, fixture("ffi_echo_bool"), "ffi_echo_bool"); + const echoU8 = $vm.ffiFunction({ args: ["u8"], returns: "u8" }, fixture("ffi_echo_u8"), "ffi_echo_u8"); + const echoPtr = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, fixture("ffi_echo_ptr"), "ffi_echo_ptr"); + const callCbVoid = $vm.ffiFunction({ args: ["function"], returns: "void" }, fixture("ffi_call_cb_void"), "ffi_call_cb_void"); + const callCbI32 = $vm.ffiFunction({ args: ["function", "i32"], returns: "i32" }, fixture("ffi_call_cb_i32"), "ffi_call_cb_i32"); + + // ---- 1. Type change after the loop is hot: int32 -> double -> boolean -> undefined. + function hotAdd(a, b) { + return addI32(a, b); + } + noInline(hotAdd); + for (let i = 0; i < 3e4; ++i) { + const r = hotAdd(i, 1); + if (r !== ((i + 1) | 0)) + throw new Error("hotAdd int32 phase iteration " + i + " got " + r); + } + // Now feed non-int32 values through the same (compiled) call site. + check(hotAdd(0.5, 5), 5, "hotAdd(0.5, 5) after tier-up (toInt32(0.5) == 0)"); + check(hotAdd(2.9, -3.9), -1, "hotAdd(2.9, -3.9)"); + check(hotAdd(4294967296 + 7, 1), 8, "hotAdd(2^32 + 7, 1)"); + check(hotAdd(true, false), 1, "hotAdd(true, false)"); + check(hotAdd(undefined, 41), 41, "hotAdd(undefined, 41)"); + check(hotAdd(null, -2), -2, "hotAdd(null, -2)"); + check(hotAdd(NaN, 3), 3, "hotAdd(NaN, 3)"); + check(hotAdd(Infinity, 3), 3, "hotAdd(Infinity, 3)"); + check(hotAdd(-0, 3), 3, "hotAdd(-0, 3)"); + // Alternating types every iteration (the site cannot stay speculated). + for (let i = 0; i < 1e4; ++i) { + const a = (i & 1) ? i + 0.5 : i; + const r = hotAdd(a, 2); + if (r !== ((i + 2) | 0)) + throw new Error("hotAdd alternating phase iteration " + i + " got " + r); + } + // And back to int32 only: still correct after re-optimization. + for (let i = 0; i < 2e4; ++i) { + const r = hotAdd(i, -i); + if (r !== 0) + throw new Error("hotAdd re-warm iteration " + i + " got " + r); + } + + // ---- 2. Double edges: NaN / -0 / infinities through a hot double call site. + function hotAddF64(a, b) { + return addF64(a, b); + } + noInline(hotAddF64); + for (let i = 0; i < 3e4; ++i) { + const r = hotAddF64(i * 0.5, 0.25); + if (r !== i * 0.5 + 0.25) + throw new Error("hotAddF64 iteration " + i + " got " + r); + } + check(hotAddF64(NaN, 1), NaN, "hotAddF64(NaN, 1)"); + check(hotAddF64(-0, -0), -0, "hotAddF64(-0, -0)"); + check(hotAddF64(-0, 0), 0, "hotAddF64(-0, 0)"); + check(hotAddF64(Infinity, -Infinity), NaN, "hotAddF64(Inf, -Inf)"); + check(hotAddF64(1, undefined), NaN, "hotAddF64(1, undefined)"); // undefined -> NaN + check(hotAddF64(2, 3), 5, "hotAddF64 int32 arguments (Int32 -> Double)"); + check(hotAddF64(1e308, 1e308), Infinity, "hotAddF64 overflow"); + + // ---- 3. bool / u8 / ptr sites that see every input class after warm-up. + function hotBool(x) { + return echoBool(x); + } + noInline(hotBool); + for (let i = 0; i < 2e4; ++i) { + if (hotBool(true) !== true) + throw new Error("hotBool warm iteration " + i); + } + check(hotBool(0), false, "hotBool(0)"); + check(hotBool(2), true, "hotBool(2)"); + check(hotBool(-0), false, "hotBool(-0)"); + check(hotBool(NaN), false, "hotBool(NaN)"); + check(hotBool(0.5), true, "hotBool(0.5)"); + check(hotBool(null), false, "hotBool(null)"); + check(hotBool(undefined), false, "hotBool(undefined)"); + check(hotBool(256), true, "hotBool(256): any non-zero int32 is true (never and32(1))"); + for (let i = 0; i < 2e4; ++i) { + if (hotBool(i & 3) !== ((i & 3) !== 0)) + throw new Error("hotBool int32 phase iteration " + i); + } + + function hotU8(x) { + return echoU8(x); + } + noInline(hotU8); + for (let i = 0; i < 2e4; ++i) { + if (hotU8(i) !== (i & 0xff)) + throw new Error("hotU8 warm iteration " + i); + } + check(hotU8(-1), 255, "hotU8(-1)"); + check(hotU8(3.99), 3, "hotU8(3.99)"); + check(hotU8(300), 44, "hotU8(300) wraps mod 256"); + check(hotU8(true), 1, "hotU8(true)"); + check(hotU8(null), 0, "hotU8(null)"); + let symbolThrew = false; + try { + hotU8(Symbol("bad")); + } catch (e) { + symbolThrew = e instanceof TypeError; + } + check(symbolThrew, true, "hotU8(Symbol) throws a TypeError (Symbols do not coerce)"); + + function hotPtr(x) { + return echoPtr(x); + } + noInline(hotPtr); + const view = new Uint8Array(8); + const viewAddress = hotPtr(view); + for (let i = 0; i < 2e4; ++i) { + if (hotPtr(view) !== viewAddress) + throw new Error("hotPtr view warm iteration " + i); + } + check(hotPtr(0), null, "hotPtr(0)"); + check(hotPtr(null), null, "hotPtr(null)"); + check(hotPtr(4096), 4096, "hotPtr(number)"); + check(hotPtr(new ArrayBuffer(4)) > 0, true, "hotPtr(ArrayBuffer)"); + check(hotPtr(view), viewAddress, "hotPtr(view) after other classes"); + + // ---- 4. Exceptions thrown inside optimized code. + // (a) Conversion errors from an FFI argument, caught inside the hot loop. + function guarded(value) { + try { + return { ok: true, value: hotPtr(value) }; + } catch (e) { + if (!(e instanceof TypeError)) + throw new Error("expected TypeError from pointer conversion, got " + e); + return { ok: false, message: e.message }; + } + } + noInline(guarded); + for (let i = 0; i < 1e4; ++i) { + const good = guarded(view); + if (!good.ok || good.value !== viewAddress) + throw new Error("guarded warm iteration " + i); + } + const symbolResult = guarded(Symbol("nope")); + check(symbolResult.ok, false, "guarded(Symbol) throws TypeError in optimized code"); + const stringResult = guarded("not a pointer"); + check(stringResult.ok, false, "guarded(string) throws TypeError in optimized code"); + const objectResult = guarded({ length: 4 }); + check(objectResult.ok, false, "guarded(plain object) throws TypeError in optimized code"); + check(guarded(view).value, viewAddress, "guarded still fine after exceptions"); + for (let i = 0; i < 5000; ++i) { + const bad = (i % 100) === 99; + const result = guarded(bad ? Symbol.iterator : view); + if (bad !== !result.ok) + throw new Error("guarded mixed iteration " + i); + } + + // (b) A throwing callback inside a hot loop, try/catch inside the loop. + const throwingCb = $vm.ffiCallback({ args: [], returns: "void" }, () => { + throw new RangeError("callback says no"); + }); + function loopWithCatch(iterations) { + let caught = 0; + for (let i = 0; i < iterations; ++i) { + try { + callCbVoid(throwingCb); + } catch (e) { + if (e instanceof RangeError) + caught++; + else + throw e; + } + } + return caught; + } + noInline(loopWithCatch); + check(loopWithCatch(10), 10, "loopWithCatch cold"); + check(loopWithCatch(15000), 15000, "loopWithCatch hot"); + + // (c) A throwing callback with the try/catch OUTSIDE the hot function: + // the exception unwinds out of optimized code exactly once. + let armed = -1; + const armedCb = $vm.ffiCallback({ args: ["i32"], returns: "i32" }, x => { + if (x === armed) + throw new EvalError("armed at " + x); + return x * 2; + }); + function unguardedLoop(count) { + let sum = 0; + for (let i = 0; i < count; ++i) + sum += callCbI32(armedCb, i); + return sum; + } + noInline(unguardedLoop); + check(unguardedLoop(1000), 999000, "unguardedLoop warm 1"); + for (let i = 0; i < 30; ++i) + check(unguardedLoop(1000), 999000, "unguardedLoop warm loop " + i); + armed = 500; + let seen = null; + try { + unguardedLoop(1000); + } catch (e) { + seen = e; + } + if (!(seen instanceof EvalError) || seen.message !== "armed at 500") + throw new Error("expected the armed EvalError from optimized code, got " + seen); + armed = -1; + check(unguardedLoop(1000), 999000, "unguardedLoop after the exception"); + + // (d) An exception thrown by a JS callee INSIDE a callback that itself was + // invoked from an FFI call inside a try: nested unwinding. + const outerCb = $vm.ffiCallback({ args: ["i32"], returns: "i32" }, x => { + if (x < 3) + return unguardedLoopThrow(x); + return x; + }); + function unguardedLoopThrow(x) { + armed = 0; + try { + return unguardedLoop(10); + } finally { + armed = -1; + } + } + let nested = null; + try { + callCbI32(outerCb, 1); + } catch (e) { + nested = e; + } + if (!(nested instanceof EvalError)) + throw new Error("expected the nested EvalError, got " + nested); + check(callCbI32(outerCb, 7), 7, "outer callback usable after nested throw"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-pointers-and-buffers.js b/JSTests/stress/ffi-pointers-and-buffers.js new file mode 100644 index 0000000000000..d56249ade0b3d --- /dev/null +++ b/JSTests/stress/ffi-pointers-and-buffers.js @@ -0,0 +1,193 @@ +//@ requireOptions("--useDollarVM=1") + +// Pointer-family conversions: TypedArray / DataView / ArrayBuffer / number +// arguments, detached views, cstring transcoding of JS strings (a new +// capability, SPEC section 5), pointer round trips and raw memory pokes via +// $vm.ffiRead / $vm.ffiWrite. + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (Object.is(value, -0)) + return "-0"; + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (name, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(name), name); + + const ptrIdentity = bind("ffi_ptr_identity", ["ptr"], "ptr"); + const ptrWriteU32 = bind("ffi_ptr_write_u32", ["ptr", "u32"], "void"); + const ptrReadU32 = bind("ffi_ptr_read_u32", ["ptr"], "u32"); + const strlen = bind("ffi_strlen", ["cstring"], "u64"); + const highPtr = bind("ffi_high_ptr", [], "ptr"); + const retNullPtr = bind("ffi_ret_null_ptr", [], "ptr"); + const echoPtr = bind("ffi_echo_ptr", ["ptr"], "ptr"); + const echoCString = bind("ffi_echo_cstring", ["cstring"], "cstring"); + const bufferArg = $vm.ffiFunction({ args: ["buffer"], returns: "ptr" }, fixture("ffi_ptr_identity"), "ffi_ptr_identity(buffer)"); + + // ---- Null in, null out. + check(retNullPtr(), null, "ffi_ret_null_ptr()"); + check(ptrIdentity(0), null, "ffi_ptr_identity(0)"); + check(ptrIdentity(null), null, "ffi_ptr_identity(null)"); + check(ptrIdentity(undefined), null, "ffi_ptr_identity(undefined)"); + check(echoPtr(0), null, "ffi_echo_ptr(0)"); + check(echoPtr(null), null, "ffi_echo_ptr(null)"); + + // ---- TypedArray / DataView / ArrayBuffer addresses are consistent. + const buffer = new ArrayBuffer(64); + const u8 = new Uint8Array(buffer); + const u32 = new Uint32Array(buffer); + const u32Offset = new Uint32Array(buffer, 8, 4); + const dataView = new DataView(buffer, 12, 8); + const base = ptrIdentity(u8); + check(typeof base, "number", "typed array address typeof"); + if (!(base > 0)) + throw new Error("expected a positive address, got " + base); + check(ptrIdentity(u8), base, "address is stable"); + check(ptrIdentity(u32), base, "views of the same buffer share the base address"); + check(ptrIdentity(buffer), base, "ArrayBuffer -> data()"); + check(ptrIdentity(u32Offset), base + 8, "byteOffset is honored (Uint32Array)"); + check(ptrIdentity(dataView), base + 12, "byteOffset is honored (DataView)"); + check(ptrIdentity(u8.subarray(3)), base + 3, "byteOffset is honored (subarray)"); + check(bufferArg(u32Offset), base + 8, "Type::Buffer view"); + check(ptrIdentity(base), base, "numeric pointer round trip"); + check(echoPtr(base + 5), base + 5, "numeric pointer arithmetic round trip"); + + // ---- Writes/reads through native pointers. + ptrWriteU32(u32, 0xdeadbeef); + check(u32[0], 0xdeadbeef >>> 0, "ffi_ptr_write_u32 through a Uint32Array"); + ptrWriteU32(u32Offset, 7); + check(u32[2], 7, "ffi_ptr_write_u32 through an offset view"); + ptrWriteU32(base + 4, 0x11223344); + check(u32[1], 0x11223344, "ffi_ptr_write_u32 through a numeric pointer"); + u32[3] = 0xffffffff; + check(ptrReadU32(u32Offset.subarray(1)), 4294967295, "ffi_ptr_read_u32 returns unsigned above INT32_MAX"); + u32[3] = 0x80000000; + check(ptrReadU32(base + 12), 2147483648, "ffi_ptr_read_u32 of 0x80000000"); + ptrWriteU32(u32, -1); // u32 argument: toInt32 then reinterpret + check(u32[0], 4294967295, "u32 argument -1 wraps to 0xffffffff"); + ptrWriteU32(u32, 4294967296 + 9); + check(u32[0], 9, "u32 argument wraps mod 2^32"); + + // ---- $vm.ffiRead / $vm.ffiWrite over the same memory. + $vm.ffiWrite(base, "u8", 200); + check(u8[0], 200, "$vm.ffiWrite u8"); + check($vm.ffiRead(base, "u8"), 200, "$vm.ffiRead u8"); + check($vm.ffiRead(base, "i8"), -56, "$vm.ffiRead i8 sign"); + const f64 = new Float64Array(buffer, 32, 2); + $vm.ffiWrite(base + 32, "f64", -0.5); + check(f64[0], -0.5, "$vm.ffiWrite f64"); + f64[1] = Math.PI; + check($vm.ffiRead(base + 40, "f64"), Math.PI, "$vm.ffiRead f64"); + $vm.ffiWrite(base + 32, "f32", 1.5); + check(new Float32Array(buffer, 32, 1)[0], 1.5, "$vm.ffiWrite f32"); + $vm.ffiWrite(base + 8, "i32", -123456789); + check($vm.ffiRead(base + 8, "i32"), -123456789, "$vm.ffiRead i32"); + check(new Int32Array(buffer, 8, 1)[0], -123456789, "$vm.ffiWrite i32 visible to JS"); + + // ---- cstring arguments: JS strings are transcoded to NUL-terminated UTF-8. + check(strlen("hello"), 5n, 'strlen("hello")'); + check(strlen(""), 0n, 'strlen("")'); + check(strlen("héllo"), 6n, "strlen of a Latin-1 string counts UTF-8 bytes"); + check(strlen("\u{1D11E}"), 4n, "strlen of an astral character counts 4 UTF-8 bytes"); + check(strlen("→←"), 6n, "strlen of two BMP arrows"); + check(strlen("mixed é \u{1F600} end"), BigInt(6 + 2 + 1 + 4 + 4), "strlen mixed"); + let rope = ""; + for (let i = 0; i < 200; ++i) + rope += "ab"; // built by concatenation -> rope until resolved + check(strlen(rope), 400n, "strlen of a rope"); + let astralRope = ""; + for (let i = 0; i < 50; ++i) + astralRope += "\u{1D11E}x"; + check(strlen(astralRope), 250n, "strlen of an astral rope"); + // A NUL-containing string is truncated at the NUL by strlen (the copy is faithful). + check(strlen("abc\0def"), 3n, "strlen stops at embedded NUL"); + // TypedArrays are also accepted for cstring parameters (pointer semantics). + const cstringBytes = new Uint8Array([0x66, 0x66, 0x69, 0x00, 0x21]); // "ffi\0!" + check(strlen(cstringBytes), 3n, "strlen of a Uint8Array cstring"); + check(strlen(cstringBytes.subarray(1)), 2n, "strlen of a Uint8Array subarray cstring"); + + const utf8Bytes = new Uint8Array([0x68, 0xc3, 0xa9, 0x21, 0x00]); // "hé!" + check(echoCString(utf8Bytes), "hé!", "ffi_echo_cstring decodes the returned UTF-8 to a string"); + check($vm.ffiCString(ptrIdentity(utf8Bytes)), "hé!", "$vm.ffiCString decodes UTF-8"); + check($vm.ffiCString(ptrIdentity(cstringBytes)), "ffi", "$vm.ffiCString stops at NUL"); + check(echoCString(0), null, "ffi_echo_cstring(0) is null"); + check(echoCString("round trip"), "round trip", "a JS string round-trips through cstring"); + check(echoCString(null), null, "ffi_echo_cstring(null)"); + check(echoCString("transient"), "transient", "arena-copied cstring argument round-trips"); + + // ---- ffi_high_ptr: full 47-bit user-space pointer round trip. + check(highPtr(), 0x00007fffdeadbee0, "ffi_high_ptr()"); + check(ptrIdentity(highPtr()), 0x00007fffdeadbee0, "ffi_high_ptr round trip through ffi_ptr_identity"); + check(ptrIdentity(0x00007fffdeadbee0), 0x00007fffdeadbee0, "high pointer literal round trip"); + // Sign-extension of int32 pointer arguments: -1 becomes all-ones. That + // address exceeds 2^53, so it comes back as an EXACT BigInt rather than a + // lossy double (SPEC section 5 pointer rule, oven-sh/bun#28068). + check(ptrIdentity(-1), 18446744073709551615n, "ffi_ptr_identity(-1) reads back as exactly 0xFFFFFFFFFFFFFFFF"); + check(ptrIdentity(-4096), 18446744073709547520n, "ffi_ptr_identity(-4096) sign-extends (exact)"); + // ...and a BigInt address round-trips back into a pointer argument unchanged. + check(ptrIdentity(18446744073709551615n), 18446744073709551615n, "BigInt pointer argument round trip"); + check(ptrIdentity(0x123456789abn), 0x123456789ab, "small BigInt pointer comes back as a plain number"); + + // ---- Detached buffers: vector() is null, so the pointer is 0 -> null (SPEC section 5). + { + const detachable = new ArrayBuffer(16); + const detachedView = new Uint8Array(detachable); + if (typeof transferArrayBuffer === "function") + transferArrayBuffer(detachable); + else + detachable.transfer(); + if (detachedView.length !== 0) + throw new Error("expected a detached view"); + for (let i = 0; i < 3; ++i) + check(ptrIdentity(detachedView), null, "detached TypedArray converts to a null pointer (iteration " + i + ")"); + } + + // ---- Hot loops: cell arguments through the JIT tiers with GC pressure. + const hot = new Uint32Array(4); + for (let i = 0; i < 3e4; ++i) { + ptrWriteU32(hot, i); + if (hot[0] !== (i >>> 0)) + throw new Error("hot ffi_ptr_write_u32 iteration " + i); + if (ptrReadU32(hot) !== (i >>> 0)) + throw new Error("hot ffi_ptr_read_u32 iteration " + i); + } + for (let i = 0; i < 5000; ++i) { + // Temporary view: must stay alive for the duration of the call even + // though nothing but the call references it (conservative scan / + // DFG keep-alive, SPEC section 15.1). + ptrWriteU32(new Uint32Array(2), i); + if ((i & 511) === 0) + gc(); + } + for (let i = 0; i < 2e4; ++i) { + if (strlen("tier " + (i & 7)) !== 6n) + throw new Error("hot strlen iteration " + i); + } + let stableString = "stable string"; + for (let i = 0; i < 3e4; ++i) { + if (strlen(stableString) !== 13n) + throw new Error("hot strlen (stable) iteration " + i); + } + const hotAddress = ptrIdentity(hot); + for (let i = 0; i < 2e4; ++i) { + if (highPtr() !== 0x00007fffdeadbee0) + throw new Error("hot ffi_high_ptr iteration " + i); + if (retNullPtr() !== null) + throw new Error("hot ffi_ret_null_ptr iteration " + i); + if (ptrIdentity(hot) !== hotAddress) + throw new Error("hot typed array address changed at iteration " + i); + if (ptrIdentity(hotAddress) !== hotAddress) + throw new Error("hot numeric pointer round trip iteration " + i); + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-ptr-object-arg.js b/JSTests/stress/ffi-ptr-object-arg.js new file mode 100644 index 0000000000000..4295f968fb36b --- /dev/null +++ b/JSTests/stress/ffi-ptr-object-arg.js @@ -0,0 +1,38 @@ +//@ requireOptions("--useDollarVM=1") +// Pointer-family arguments accept an object carrying a numeric/BigInt `ptr` property (Bun's +// documented FFIType.function / pointer forms accept a JSCallback / Pointer / CString object). +// The property get may run a getter, so its exceptions must propagate; a non-numeric `ptr` +// falls through to the normal type error. Tier-differential against a noDFG oracle. +if (!$vm.useJIT()) quit(); + +const identity = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, $vm.ffiFixture("ffi_ptr_identity"), "ffi_ptr_identity"); +function ref(v) { try { return identity(v); } catch (e) { return "threw:" + e.constructor.name; } } +function hot(v) { try { return identity(v); } catch (e) { return "threw:" + e.constructor.name; } } +noDFG(ref); noInline(ref); noInline(hot); + +let failures = 0; +const check = (l, got, want) => { if (!Object.is(got, want)) { print(`FAIL ${l}: got ${String(got)} want ${String(want)}`); if (++failures > 8) throw new Error("too many"); } }; + +const buf = new Uint8Array(8); +const addr = ref(buf); // a real address (number) +const withPtr = { ptr: addr }; // JSCallback / Pointer-style wrapper +const withBigPtr = { ptr: 4294967297n }; // BigInt ptr (> 2^32) +class Wrapper { get ptr() { return addr; } } // getter on the prototype (like CString) +const viaGetter = new Wrapper(); +const throwing = { get ptr() { throw new RangeError("ptr getter"); } }; +const badPtr = { ptr: "not a number" }; // must still be a TypeError +const noPtr = {}; + +for (let i = 0; i < 30000; ++i) { + check(`obj#${i}`, hot(withPtr), addr); + check(`bigint#${i}`, hot(withBigPtr), 4294967297); + check(`getter#${i}`, hot(viaGetter), addr); + check(`getter-throws#${i}`, hot(throwing), "threw:RangeError"); + check(`badptr#${i}`, hot(badPtr), "threw:TypeError"); + check(`noptr#${i}`, hot(noPtr), "threw:TypeError"); + // and the reference tier agrees on every one + check(`agree-obj#${i}`, hot(withPtr), ref(withPtr)); + check(`agree-throw#${i}`, hot(throwing), ref(throwing)); +} +if (failures) throw new Error(`${failures} failure(s)`); +print("ffi ptr-object arg: all checks passed"); diff --git a/JSTests/stress/ffi-signature-errors.js b/JSTests/stress/ffi-signature-errors.js new file mode 100644 index 0000000000000..7753f8b3e3b8f --- /dev/null +++ b/JSTests/stress/ffi-signature-errors.js @@ -0,0 +1,134 @@ +//@ requireOptions("--useDollarVM=1") + +// Signature validation (SPEC sections 2, 3, 11.2): every invalid descriptor +// is a TypeError; valid ones intern to canonical strings. + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + String(expected) + " but got " + String(actual)); +} + +function expectTypeError(fn, label) { + let error = null; + try { + fn(); + } catch (e) { + error = e; + } + if (error === null) + throw new Error(label + ": expected a TypeError, nothing thrown"); + if (!(error instanceof TypeError)) + throw new Error(label + ": expected a TypeError, got " + error); +} + +function main() { + const target = $vm.ffiFixture("ffi_echo_i32"); + const dummy = () => 1; + + // ---- Invalid descriptors: TypeError from ffiFunction, ffiCallback and ffiSignatureString alike. + const invalidDescriptors = [ + [{ args: ["void"], returns: "i32" }, "void as an argument"], + [{ args: ["i32", "void", "i32"], returns: "i32" }, "void as a middle argument"], + [{ args: ["int128"], returns: "i32" }, "unknown type string"], + [{ args: ["i33"], returns: "i32" }, "unknown type string i33"], + [{ args: ["I32"], returns: "i32" }, "type strings are case sensitive"], + [{ args: [""], returns: "i32" }, "empty type string"], + [{ args: ["i32 "], returns: "i32" }, "type string with trailing space"], + [{ args: ["i32"], returns: "napi_env" }, "napi_env (removed type) as return type"], + [{ args: ["napi_env", "i32"], returns: "i32" }, "napi_env (removed type) as an argument"], + [{ args: [18], returns: "i32" }, "reserved tag 18 (was napi_env) as an argument"], + [{ args: ["i32"], returns: 18 }, "reserved tag 18 (was napi_env) as return type"], + [{ args: ["i32"], returns: "buffer" }, "buffer as return type"], + [{ args: ["i32"], returns: "buffer_length" }, "buffer_length as return type"], + [{ args: ["i32"], returns: 21 }, "buffer_length tag (21) as return type"], + [{ args: ["i32"], returns: "unknown" }, "unknown return type string"], + [{ args: ["i32"], returns: 22 }, "return tag out of range"], + [{ args: [22], returns: "i32" }, "argument tag out of range"], + [{ args: [-1], returns: "i32" }, "negative argument tag"], + [{ args: [1.5], returns: "i32" }, "fractional argument tag"], + [{ args: [13], returns: "i32" }, "void tag (13) as an argument"], + [{ args: [{}], returns: "i32" }, "object as a type"], + [{ args: [null], returns: "i32" }, "null as a type"], + [{ args: [Symbol("i32")], returns: "i32" }, "symbol as a type"], + [{ args: new Array(33).fill("i32"), returns: "i32" }, "33 arguments"], + [{ args: new Array(64).fill("f64"), returns: "f64" }, "64 arguments"], + [{ args: "i32", returns: "i32" }, "args is not an array"], + [{ args: [true], returns: "i32" }, "boolean as a type"], + [{ args: [[]], returns: "i32" }, "array as a type"], + [null, "null descriptor"], + [undefined, "undefined descriptor"], + [42, "number descriptor"], + ["f64(i32)", "string descriptor"], + ]; + for (const [descriptor, label] of invalidDescriptors) { + expectTypeError(() => $vm.ffiFunction(descriptor, target, "bad"), "ffiFunction: " + label); + expectTypeError(() => $vm.ffiCallback(descriptor, dummy), "ffiCallback: " + label); + expectTypeError(() => $vm.ffiSignatureString(descriptor), "ffiSignatureString: " + label); + } + + // The 32-argument boundary is exact. + const thirtyTwo = { args: new Array(32).fill("i32"), returns: "i64" }; + const fn32 = $vm.ffiFunction(thirtyTwo, $vm.ffiFixture("ffi_sum_i32_16"), "arity32"); + check(fn32.length, 32, "length of a 32-argument FFI function"); + + // ---- Canonical signature strings: interning smoke test + aliases + numeric tags. + check($vm.ffiSignatureString({ args: ["i32", "f64"], returns: "f64" }), "f64(i32,f64)", "canonical string"); + check($vm.ffiSignatureString({ args: ["int32_t", "double"], returns: "double" }), "f64(i32,f64)", "aliases canonicalize"); + check($vm.ffiSignatureString({ args: [5, 9], returns: 9 }), "f64(i32,f64)", "numeric tags canonicalize"); + check($vm.ffiSignatureString({ args: [], returns: "void" }), "void()", "empty signature"); + check($vm.ffiSignatureString({ args: [], returns: 13 }), "void()", "numeric void return tag"); + check($vm.ffiSignatureString({ args: ["char"], returns: "char" }), "char(char)", "char keeps its own name"); + check($vm.ffiSignatureString({ args: ["int8_t"], returns: "int8_t" }), "i8(i8)", "int8_t is i8, not char"); + check($vm.ffiSignatureString({ args: ["napi_value", "jsvalue"], returns: "napi_value" }), "jsvalue(jsvalue,jsvalue)", "napi_value is the legacy spelling of jsvalue"); + check($vm.ffiSignatureString({ args: ["buffer", "cstring", "function"], returns: "ptr" }), "ptr(buffer,cstring,function)", "pointer family names"); + check($vm.ffiSignatureString({ args: ["i64_fast", "u64_fast"], returns: "u64_fast" }), "u64_fast(i64_fast,u64_fast)", "fast 64-bit names"); + const everyAlias = [ + ["int8_t", "i8"], ["uint8_t", "u8"], ["int16_t", "i16"], ["uint16_t", "u16"], ["int32_t", "i32"], ["int", "i32"], + ["c_int", "i32"], ["uint32_t", "u32"], ["c_uint", "u32"], ["int64_t", "i64"], ["isize", "i64"], ["uint64_t", "u64"], + ["usize", "u64"], ["size_t", "u64"], ["double", "f64"], ["float", "f32"], ["void*", "ptr"], ["pointer", "ptr"], + // "char*" is a POINTER alias (tag 12, Bun's FFIType parity), not cstring. + ["char*", "ptr"], ["callback", "function"], ["fn", "function"], ["bool", "bool"], ["char", "char"], + ["ptr", "ptr"], ["cstring", "cstring"], ["jsvalue", "jsvalue"], ["napi_value", "jsvalue"], + ]; + for (const [alias, canonical] of everyAlias) + check($vm.ffiSignatureString({ args: [alias], returns: "i32" }), "i32(" + canonical + ")", "alias " + alias); + // Every numeric tag in order. Tag 13 (void) is not a valid argument and tag 18 is the + // reserved (formerly napi_env) tag, invalid in every position; both are skipped here (18's + // rejection is covered by the invalid-descriptor table above). + const canonicalByTag = ["char", "i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64", "f64", "f32", "bool", "ptr", "void", "cstring", "i64_fast", "u64_fast", "function", null, "jsvalue", "buffer", "buffer_length"]; + for (let tag = 0; tag < canonicalByTag.length; ++tag) { + if (canonicalByTag[tag] === "void" || canonicalByTag[tag] === null) + continue; // void is not a valid argument; 18 is reserved + check($vm.ffiSignatureString({ args: [tag], returns: 5 }), "i32(" + canonicalByTag[tag] + ")", "numeric tag " + tag); + } + + // Structural interning: equal shapes give the same string, order matters. + check($vm.ffiSignatureString({ args: ["i32", "f64"], returns: "f64" }) === $vm.ffiSignatureString({ args: [5, "double"], returns: 9 }), true, "interning agrees across spellings"); + if ($vm.ffiSignatureString({ args: ["f64", "i32"], returns: "f64" }) === $vm.ffiSignatureString({ args: ["i32", "f64"], returns: "f64" })) + throw new Error("argument order must matter"); + + // ---- The `ptr` parameter of $vm.ffiFunction must be a pointer number or a JSFFICallback. + expectTypeError(() => $vm.ffiFunction({ args: ["i32"], returns: "i32" }, "not a pointer", "bad"), "string as ptr"); + expectTypeError(() => $vm.ffiFunction({ args: ["i32"], returns: "i32" }, {}, "bad"), "object as ptr"); + expectTypeError(() => $vm.ffiFunction({ args: ["i32"], returns: "i32" }, Symbol("p"), "bad"), "symbol as ptr"); + expectTypeError(() => $vm.ffiFunction({ args: ["i32"], returns: "i32" }, dummy, "bad"), "raw JS function as ptr"); + expectTypeError(() => $vm.ffiCallback({ args: ["i32"], returns: "i32" }, 42), "non-callable callback target"); + expectTypeError(() => $vm.ffiCallback({ args: ["i32"], returns: "i32" }, {}), "object callback target"); + + // ---- Unknown fixture names throw (but not TypeError necessarily). + let threw = false; + try { + $vm.ffiFixture("ffi_no_such_fixture"); + } catch (e) { + threw = true; + } + check(threw, true, "unknown fixture name throws"); + const names = $vm.ffiFixtures(); + check(Array.isArray(names), true, "$vm.ffiFixtures() returns an array"); + check(names.includes("ffi_echo_i32"), true, "fixture list contains ffi_echo_i32"); + check(names.includes("ffi_canary_call"), true, "fixture list contains ffi_canary_call"); + check(names.length >= 90, true, "fixture list is complete"); +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-subword-and-returns.js b/JSTests/stress/ffi-subword-and-returns.js new file mode 100644 index 0000000000000..193865ef1d7e1 --- /dev/null +++ b/JSTests/stress/ffi-subword-and-returns.js @@ -0,0 +1,160 @@ +//@ requireOptions("--useDollarVM=1") + +// Sub-word extension probes (caller side) and return-value normalization +// probes (callee side): widen fixtures, ffi_ret_neg_one_*, float edge +// returners, bool and char rules (SPEC sections 2, 4, 5, 7.2). + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (Object.is(value, -0)) + return "-0"; + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (name, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(name), name + "->" + ret); + + // ---- Caller-side extension: the callee widens whatever low bits it got. + const widenChar = bind("ffi_widen_char", ["char"], "i64"); + const widenI8 = bind("ffi_widen_i8", ["i8"], "i64"); + const widenU8 = bind("ffi_widen_u8", ["u8"], "i64"); + const widenI16 = bind("ffi_widen_i16", ["i16"], "i64"); + const widenU16 = bind("ffi_widen_u16", ["u16"], "i64"); + const widenCases = [ + [widenChar, "widen_char", [[0, 0n], [-1, -1n], [255, -1n], [0xff, -1n], [127, 127n], [128, -128n], [-128, -128n], [0x1ff, -1n], [0x180, -128n], [-129, 127n]]], + [widenI8, "widen_i8", [[0, 0n], [-1, -1n], [255, -1n], [127, 127n], [128, -128n], [-128, -128n], [0x17f, 127n], [0x180, -128n], [-129, 127n], [0x7fffffff, -1n]]], + [widenU8, "widen_u8", [[0, 0n], [-1, 255n], [255, 255n], [256, 0n], [511, 255n], [128, 128n], [-128, 128n], [0x101, 1n], [0x7fffffff, 255n]]], + [widenI16, "widen_i16", [[0, 0n], [-1, -1n], [65535, -1n], [32767, 32767n], [32768, -32768n], [-32768, -32768n], [-32769, 32767n], [0x12345, 9029n], [65536, 0n]]], + [widenU16, "widen_u16", [[0, 0n], [-1, 65535n], [65535, 65535n], [65536, 0n], [32768, 32768n], [-32768, 32768n], [0x18000, 32768n], [70000, 4464n]]], + ]; + for (const [fn, name, cases] of widenCases) { + for (const [input, expected] of cases) + check(fn(input), expected, name + "(" + input + ")"); + } + + // ---- Callee-side return normalization: -1 through every integer width. + const negOnes = [ + ["ffi_ret_neg_one_i8", "i8", -1], + ["ffi_ret_neg_one_i16", "i16", -1], + ["ffi_ret_neg_one_i32", "i32", -1], + ["ffi_ret_neg_one_i64", "i64", -1n], + ["ffi_ret_neg_one_u8", "u8", 255], + ["ffi_ret_neg_one_u16", "u16", 65535], + ["ffi_ret_neg_one_u32", "u32", 4294967295], + ["ffi_ret_neg_one_u64", "u64", 18446744073709551615n], + // Reinterpretations of the same all-ones bit pattern: + ["ffi_ret_neg_one_i8", "u8", 255], + ["ffi_ret_neg_one_u8", "i8", -1], + ["ffi_ret_neg_one_i32", "u32", 4294967295], + ["ffi_ret_neg_one_u32", "i32", -1], + ["ffi_ret_neg_one_i64", "u64", 18446744073709551615n], + ["ffi_ret_neg_one_u64", "i64", -1n], + ["ffi_ret_neg_one_i64", "i64_fast", -1], + ["ffi_ret_neg_one_u64", "u64_fast", 18446744073709551615n], + ]; + for (let i = 0; i < negOnes.length; ++i) { + const [name, type, expected] = negOnes[i]; + const fn = bind(name, [], type); + check(fn(), expected, name + " as " + type); + for (let iteration = 0; iteration < 5000; ++iteration) { + const result = fn(); + if (!Object.is(result, expected)) + throw new Error(name + " as " + type + " hot iteration " + iteration + ": got " + describe(result)); + } + } + + // ---- bool normalization: only the low byte of a native bool return is + // defined; a callee returning 2 in an int8 register declared as bool must + // still surface as `true`. + const twoAsBool = bind("ffi_ret_two_as_bool", [], "bool"); + check(twoAsBool(), true, "ffi_ret_two_as_bool"); + for (let i = 0; i < 2e4; ++i) { + if (twoAsBool() !== true) + throw new Error("ffi_ret_two_as_bool hot iteration " + i); + } + const twoAsI8 = bind("ffi_ret_two_as_bool", [], "i8"); + check(twoAsI8(), 2, "ffi_ret_two_as_bool declared i8"); + + const echoBool = bind("ffi_echo_bool", ["bool"], "bool"); + for (const [input, expected] of [[2, true], [-1, true], [0, false], [0.5, true], [-0, false], [NaN, false], [1e-300, true], [true, true], [false, false], [null, false], [undefined, false], [256, true], [65536, true]]) + check(echoBool(input), expected, "ffi_echo_bool(" + describe(input) + ")"); + for (let i = 0; i < 2e4; ++i) { + // 256 has a zero low byte: `and32(0xff)` or `and32(1)` mis-conversions would return false. + if (echoBool(256) !== true) + throw new Error("ffi_echo_bool(256) hot iteration " + i); + if (echoBool(2) !== true) + throw new Error("ffi_echo_bool(2) hot iteration " + i); + if (echoBool(0) !== false) + throw new Error("ffi_echo_bool(0) hot iteration " + i); + } + + // ---- char is signed on every target (SPEC section 2). + const echoChar = bind("ffi_echo_char", ["char"], "char"); + check(echoChar(-1), -1, "ffi_echo_char(-1)"); + check(echoChar(255), -1, "ffi_echo_char(255)"); + check(echoChar(0x80), -128, "ffi_echo_char(0x80)"); + check(echoChar(0x7f), 127, "ffi_echo_char(0x7f)"); + check(widenChar(-1), -1n, "ffi_widen_char(-1)"); + for (let i = 0; i < 2e4; ++i) { + if (echoChar(255) !== -1) + throw new Error("ffi_echo_char(255) hot iteration " + i); + } + + // ---- Floating-point edge returns (purifyNaN, sign of zero, denormals, infinity). + const retNanF32 = bind("ffi_ret_nan_f32", [], "f32"); + const retImpureNanF64 = bind("ffi_ret_impure_nan_f64", [], "f64"); + const retNegZeroF64 = bind("ffi_ret_neg_zero_f64", [], "f64"); + const retDenormalF32 = bind("ffi_ret_denormal_f32", [], "f32"); + const retInfF64 = bind("ffi_ret_inf_f64", [], "f64"); + const echoF32 = bind("ffi_echo_f32", ["f32"], "f32"); + const echoF64 = bind("ffi_echo_f64", ["f64"], "f64"); + for (let i = 0; i < 1e4; ++i) { + if (!Number.isNaN(retNanF32())) + throw new Error("ffi_ret_nan_f32 iteration " + i); + if (!Number.isNaN(retImpureNanF64())) + throw new Error("ffi_ret_impure_nan_f64 iteration " + i); + if (!Object.is(retNegZeroF64(), -0)) + throw new Error("ffi_ret_neg_zero_f64 iteration " + i); + if (retDenormalF32() !== 2 ** -149) + throw new Error("ffi_ret_denormal_f32 iteration " + i + ": " + retDenormalF32()); + if (retInfF64() !== Infinity) + throw new Error("ffi_ret_inf_f64 iteration " + i); + if (!Number.isNaN(echoF32(NaN))) + throw new Error("ffi_echo_f32(NaN) iteration " + i); + if (!Number.isNaN(echoF64(NaN))) + throw new Error("ffi_echo_f64(NaN) iteration " + i); + if (!Object.is(echoF32(-0), -0)) + throw new Error("ffi_echo_f32(-0) iteration " + i); + if (!Object.is(echoF64(-0), -0)) + throw new Error("ffi_echo_f64(-0) iteration " + i); + } + check(echoF32(2 ** -149), 2 ** -149, "denormal f32 argument round trip"); + check(echoF64(5e-324), 5e-324, "denormal f64 argument round trip"); + check(echoF32(3.4028234663852886e38), 3.4028234663852886e38, "FLT_MAX round trip"); + check(echoF32(1e39), Infinity, "f32 overflow to +inf"); + check(echoF32(-1e39), -Infinity, "f32 overflow to -inf"); + + // u32 returns above INT32_MAX become doubles, not negative int32s. + const echoU32 = bind("ffi_echo_u32", ["u32"], "u32"); + const echoI32 = bind("ffi_echo_i32", ["i32"], "i32"); + for (let i = 0; i < 2e4; ++i) { + if (echoU32(-1) !== 4294967295) + throw new Error("u32 return of 0xffffffff iteration " + i + ": " + echoU32(-1)); + if (echoU32(2147483648) !== 2147483648) + throw new Error("u32 return of 0x80000000 iteration " + i); + if (echoI32(-1) !== -1) + throw new Error("i32 return of -1 iteration " + i); + if (echoI32(2147483648) !== -2147483648) + throw new Error("i32 wrap of 0x80000000 iteration " + i); + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-tailcall.js b/JSTests/stress/ffi-tailcall.js new file mode 100644 index 0000000000000..703f14b994350 --- /dev/null +++ b/JSTests/stress/ffi-tailcall.js @@ -0,0 +1,51 @@ +//@ requireOptions("--useDollarVM=1") +"use strict"; +// An FFI call in TAIL position (arrow expression body / `return sym(...)` in strict code) is a +// bytecode TailCall. The parser emits it as a plain Call so it can become CallFFI; that must not +// change semantics: the value returned, exceptions from the FFI conversion, and deep call chains +// (no tail-call frame reuse assumed) all behave identically to the interpreter. +if (!$vm.useJIT()) quit(); +const fixture = name => $vm.ffiFixture(name); +const identity = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, fixture("ffi_ptr_identity"), "ffi_ptr_identity"); +const addI32 = $vm.ffiFunction({ args: ["i32", "i32"], returns: "i32" }, fixture("ffi_add_i32"), "ffi_add_i32"); + +// Tail position via arrow expression bodies (strict => TailCall bytecode). +const tailIdentity = (v) => identity(v); +const tailAdd = (a, b) => addI32(a, b); +// Tail position via explicit `return` in a strict function. +function returnsCall(a, b) { return addI32(a, b); } +noInline(tailIdentity); noInline(tailAdd); noInline(returnsCall); + +// Interpreter/baseline oracles pinned below the DFG. +function refIdentity(v) { return identity(v); } +function refAdd(a, b) { return addI32(a, b); } +noDFG(refIdentity); noDFG(refAdd); noInline(refIdentity); noInline(refAdd); + +let failures = 0; +const check = (label, got, want) => { if (!Object.is(got, want)) { print(`FAIL ${label}: got ${String(got)} want ${String(want)}`); if (++failures > 8) throw new Error("too many failures"); } }; + +for (let i = 0; i < 40000; ++i) { + const a = (i * 7) | 0, b = -(i % 101); + check(`tailAdd#${i}`, tailAdd(a, b), refAdd(a, b)); + check(`returnsCall#${i}`, returnsCall(a, b), refAdd(a, b)); + check(`tailIdentity#${i}`, tailIdentity(i), refIdentity(i)); + // The tail-position result must be USABLE (not lost): compose it. + check(`compose#${i}`, addI32(tailAdd(a, b), 1), (a + b + 1) | 0); +} + +// Exceptions raised by the FFI conversion must propagate out of the tail-call arrow correctly. +const sym = Symbol("no-coerce"); +let threw = 0; +for (let i = 0; i < 40000; ++i) { + try { tailAdd(sym, 1); print("FAIL: symbol arg did not throw at " + i); ++failures; } + catch (e) { if (e instanceof TypeError) ++threw; else { print("FAIL: wrong error " + e); ++failures; } } +} +check("threw-count", threw, 40000); + +// A DEEP chain of tail-position FFI calls: with real tail calls this reuses frames; converted to +// plain Calls it grows the stack per level. It must complete (depth is modest) with the same value. +const step = (n) => n === 0 ? 0 : (addI32(step(n - 1), 1) | 0); +check("deep-chain", step(2000), 2000); + +if (failures) throw new Error(`${failures} failure(s)`); +print("ffi tail-call: all checks passed"); diff --git a/JSTests/stress/ffi-threadsafe-callback-burst.js b/JSTests/stress/ffi-threadsafe-callback-burst.js new file mode 100644 index 0000000000000..8383172de44f7 --- /dev/null +++ b/JSTests/stress/ffi-threadsafe-callback-burst.js @@ -0,0 +1,30 @@ +//@ requireOptions("--useDollarVM=1") +// Burst: many foreign-thread invocations queue up (each on its own OS thread) BEFORE a single +// drain. Records must all survive queuing (refcounted C data), and one drain must deliver every +// one, in some order, with exact values. Also multiple distinct threadsafe callbacks interleaved. +if (!$vm.useJIT()) quit(); +const callFromThread = $vm.ffiFunction({ args: ["ptr", "i32", "i64", "u64", "f64"], returns: "void" }, + $vm.ffiFixture("ffi_call_cb_from_thread"), "call_cb_from_thread"); +const seenA = [], seenB = []; +const cbA = $vm.ffiCallback({ args: ["i32", "i64", "u64", "f64"], returns: "void" }, (a) => seenA.push(a), { threadsafe: true }); +const cbB = $vm.ffiCallback({ args: ["i32", "i64", "u64", "f64"], returns: "void" }, (a, b) => seenB.push([a, b]), { threadsafe: true }); +const N = 300; +for (let i = 0; i < N; ++i) { + callFromThread(cbA.ptr, i, 1n, 1n, 0); // queued, not run + callFromThread(cbB.ptr, i * 2, BigInt(i) - 500n, 1n, 0); +} +if (seenA.length !== 0 || seenB.length !== 0) throw new Error("ran inline"); +const delivered = $vm.drainThreadsafeCallbacks(); +if (delivered !== 2 * N) throw new Error("expected " + (2 * N) + " delivered, got " + delivered); +if (seenA.length !== N || seenB.length !== N) throw new Error("counts: " + seenA.length + "/" + seenB.length); +seenA.sort((x, y) => x - y); +for (let i = 0; i < N; ++i) if (seenA[i] !== i) throw new Error("A[" + i + "]=" + seenA[i]); +seenB.sort((x, y) => x[0] - y[0]); +for (let i = 0; i < N; ++i) { + if (seenB[i][0] !== i * 2) throw new Error("B i32 " + seenB[i][0]); + if (seenB[i][1] !== BigInt(i) - 500n) throw new Error("B i64 " + String(seenB[i][1])); +} +// A second drain finds nothing left. +if ($vm.drainThreadsafeCallbacks() !== 0) throw new Error("queue not empty after drain"); +cbA.close(); cbB.close(); +print("ffi threadsafe burst: all checks passed"); diff --git a/JSTests/stress/ffi-threadsafe-callback-throw.js b/JSTests/stress/ffi-threadsafe-callback-throw.js new file mode 100644 index 0000000000000..f89cf0c2d8a65 --- /dev/null +++ b/JSTests/stress/ffi-threadsafe-callback-throw.js @@ -0,0 +1,34 @@ +//@ requireOptions("--useDollarVM=1") +// A THROWING threadsafe callback: the drain must stop running further invocations once one +// throws, propagate the exception, and still RETIRE the counts of the un-run records -- so a +// callback close()d while records were queued (the deferred-unroot path) is not leaked/rooted +// forever, and the queue is empty afterwards. (Regression for the drain-loop early-return.) +if (!$vm.useJIT()) quit(); +const callFromThread = $vm.ffiFunction({ args: ["ptr", "i32", "i64", "u64", "f64"], returns: "void" }, + $vm.ffiFixture("ffi_call_cb_from_thread"), "call_cb_from_thread"); +let ran = 0; +const boom = $vm.ffiCallback({ args: ["i32", "i64", "u64", "f64"], returns: "void" }, + (a) => { ran++; throw new RangeError("cb-throw " + a); }, { threadsafe: true }); +const other = $vm.ffiCallback({ args: ["i32", "i64", "u64", "f64"], returns: "void" }, + () => { ran++; }, { threadsafe: true }); +// Queue three records: boom (throws first), then two more that must NOT run this drain. +callFromThread(boom.ptr, 1, 2n, 3n, 4.5); +callFromThread(other.ptr, 2, 2n, 3n, 4.5); +callFromThread(boom.ptr, 3, 2n, 3n, 4.5); +// Close one callback WHILE its records are queued (deferred-unroot path). +other.close(); +let threw = false; +try { $vm.drainThreadsafeCallbacks(); } catch (e) { threw = e instanceof RangeError && /cb-throw 1/.test(e.message); } +if (!threw) throw new Error("expected the first callback's RangeError to propagate from drain"); +if (ran !== 1) throw new Error("invocations after the throw must not run this drain: ran=" + ran); +// The queue was fully consumed (the un-run records were retired, not left queued). +if ($vm.drainThreadsafeCallbacks() !== 0) throw new Error("queue not empty after the throwing drain"); +// GC must be fine: no leaked-root cell, no swept-cell record. +for (let i = 0; i < 4; ++i) { fullGC(); edenGC(); } +// And the surviving callback still works after all that. +callFromThread(boom.ptr, 9, 2n, 3n, 4.5); +let threw2 = false; +try { $vm.drainThreadsafeCallbacks(); } catch (e) { threw2 = /cb-throw 9/.test(e.message); } +if (!threw2) throw new Error("post-recovery throwing invocation did not propagate"); +boom.close(); +print("ffi threadsafe throwing callback: all checks passed"); diff --git a/JSTests/stress/ffi-threadsafe-callback.js b/JSTests/stress/ffi-threadsafe-callback.js new file mode 100644 index 0000000000000..90f3a016ab02c --- /dev/null +++ b/JSTests/stress/ffi-threadsafe-callback.js @@ -0,0 +1,76 @@ +//@ requireOptions("--useDollarVM=1") +// Threadsafe callbacks. The C caller invokes the callback from a FOREIGN OS thread; the engine +// must NOT run JS there. It copies the raw argument slots into a record and hands it to the +// registered dispatch function (here $vm's queue); the JS thread later drains the queue, and only +// THEN are the raw slots converted to JS values (so i64/u64 BigInt boxing happens on the JS +// thread -- the shape of oven-sh/bun#35406). Assertions: +// 1) the callback does NOT run inline during the FFI call (queued, count 0 before drain); +// 2) drain delivers it exactly once with exact values: i32, i64->BigInt, u64->BigInt, f64; +// 3) values that don't fit an int32 (u64 = 2^64-1) round-trip exactly as BigInt; +// 4) a callback close()d before the drain still DELIVERS its accepted invocations, and the +// pending count keeps the (JS-unreachable) cell + callable rooted until they drain. +if (!$vm.useJIT()) quit(); + +const callFromThread = $vm.ffiFunction({ args: ["ptr", "i32", "i64", "u64", "f64"], returns: "void" }, + $vm.ffiFixture("ffi_call_cb_from_thread"), "call_cb_from_thread"); + +let seen = []; +const cb = $vm.ffiCallback({ args: ["i32", "i64", "u64", "f64"], returns: "void" }, + (a, b, c, d) => seen.push([a, b, c, d]), { threadsafe: true }); + +if (cb.threadsafe !== true) throw new Error("expected .threadsafe === true, got " + cb.threadsafe); + +for (let i = 0; i < 200; ++i) { + seen = []; + // (1) invoked from a foreign thread: must be queued, NOT run inline. + callFromThread(cb.ptr, -7 - i, -9007199254740993n, 18446744073709551615n, 2.5); + if (seen.length !== 0) throw new Error("threadsafe callback ran INLINE (should be queued): " + JSON.stringify(seen)); + + // (2) the JS thread drains: exactly one invocation delivered, exact values. + const delivered = $vm.drainThreadsafeCallbacks(); + if (delivered !== 1) throw new Error("expected 1 delivered, got " + delivered); + if (seen.length !== 1) throw new Error("expected 1 seen after drain, got " + seen.length); + const [a, b, c, d] = seen[0]; + if (a !== -7 - i) throw new Error("i32 wrong: " + a); + if (b !== -9007199254740993n) throw new Error("i64 wrong: " + String(b) + " (typeof " + typeof b + ")"); + // (3) 2^64-1 must be an exact BigInt, boxed on the JS thread. + if (c !== 18446744073709551615n) throw new Error("u64 wrong: " + String(c) + " (typeof " + typeof c + ")"); + if (d !== 2.5) throw new Error("f64 wrong: " + d); +} + +// (4) close() before drain -- WITH a full GC in between and NO JS reference to the callback. +// Two guarantees are exercised at once. LIFETIME: the queued records hold a raw pointer to the +// cell, so the pending-invocation count must keep the cell (and, through its barrier, the +// callable) rooted until every record drains -- close() while records are queued must NOT unroot +// (that was a use-after-free), even across full GCs with no JS reference. DELIVERY: an invocation +// accepted while the callback was open is a commitment, so it still RUNS after close(); close() +// only refuses NEW foreign-thread calls. So the drained records execute the callable on a cell no +// JS code can reach any more, and must produce the right values. +seen = []; +{ + let victim = $vm.ffiCallback({ args: ["i32", "i64", "u64", "f64"], returns: "void" }, + (a) => seen.push(a), { threadsafe: true }); + callFromThread(victim.ptr, 111, 2n, 3n, 4.5); // queued, not run + callFromThread(victim.ptr, 222, 2n, 3n, 4.5); // a second record for the same cell + victim.close(); // close while records are queued + victim = null; // drop the only JS reference +} +for (let i = 0; i < 5; ++i) { fullGC(); edenGC(); } // cell must survive: it is rooted until drain +const late = $vm.drainThreadsafeCallbacks(); // must NOT crash / touch a swept cell +if (late !== 2) throw new Error("expected 2 records drained even when closed, got " + late); +// Accepted-while-open invocations are delivered even after close(): the callable ran on the +// unreachable (but rooted) cell and observed the right arguments, in order. +if (seen.length !== 2 || seen[0] !== 111 || seen[1] !== 222) + throw new Error("post-close delivery wrong: " + JSON.stringify(seen)); +// A closed callback that is fully drained is now collectible; more GC must not resurrect issues. +for (let i = 0; i < 3; ++i) { fullGC(); } +if ($vm.drainThreadsafeCallbacks() !== 0) throw new Error("queue not empty"); +seen = []; // reset before section (5); it is the shared sink of the still-open `cb` + +// (5) the other test callback still works after all that. +callFromThread(cb.ptr, 5, 6n, 7n, 8.5); +if ($vm.drainThreadsafeCallbacks() !== 1) throw new Error("post-race delivery failed"); +if (seen.length !== 1 || seen[0][0] !== 5) throw new Error("wrong post-race value: " + JSON.stringify(seen)); +cb.close(); + +print("ffi threadsafe callback: all checks passed"); diff --git a/JSTests/stress/ffi-tier-differential.js b/JSTests/stress/ffi-tier-differential.js new file mode 100644 index 0000000000000..fa79abb178ffd --- /dev/null +++ b/JSTests/stress/ffi-tier-differential.js @@ -0,0 +1,222 @@ +//@ requireOptions("--useDollarVM=1") + +// One battery, one HARDCODED expected table, no reference implementation: +// the harness runs this file under all of its option matrices (LLInt-only, +// baseline, DFG-eager, FTL-eager, no-cjit, ...) and the sibling +// ffi-host-path.js runs the identical battery with --useFFIICStub=0 +// --useFFICallInDFG=0, so any tier that disagrees with any other tier fails +// against the same literal constants (SPEC section 11.4). + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (Object.is(value, -0)) + return "-0"; + if (typeof value === "symbol") + return value.toString(); + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +// Returns a NEW caller function every time (a distinct FunctionExecutable / +// CodeBlock via `new Function`), so the FFI call site inside it is +// monomorphic, exact-arity and non-spread -- the only call-site shape the DFG +// ByteCodeParser constant-callee feed and the strength-reduction +// Call -> CallFFI conversion accept (SPEC section 10.2). A shared +// `callable(...args)` warm site would stay polymorphic (and a spread call is +// never converted), so the typed CallFFI path would never join the differential. +function makeMonomorphicCaller(arity) { + const argumentList = Array.from({ length: arity }, (_, i) => "args[" + i + "]").join(", "); + return new Function("callable", "args", "return callable(" + argumentList + ");"); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const bind = (name, args, ret) => $vm.ffiFunction({ args, returns: ret }, fixture(name), name); + + const echoChar = bind("ffi_echo_char", ["char"], "char"); + const echoI8 = bind("ffi_echo_i8", ["i8"], "i8"); + const echoU8 = bind("ffi_echo_u8", ["u8"], "u8"); + const echoI16 = bind("ffi_echo_i16", ["i16"], "i16"); + const echoU16 = bind("ffi_echo_u16", ["u16"], "u16"); + const echoI32 = bind("ffi_echo_i32", ["i32"], "i32"); + const echoU32 = bind("ffi_echo_u32", ["u32"], "u32"); + const echoI64 = bind("ffi_echo_i64", ["i64"], "i64"); + const echoU64 = bind("ffi_echo_u64", ["u64"], "u64"); + const echoI64Fast = $vm.ffiFunction({ args: ["i64_fast"], returns: "i64_fast" }, fixture("ffi_echo_i64"), "ffi_echo_i64:fast"); + const echoU64Fast = $vm.ffiFunction({ args: ["u64_fast"], returns: "u64_fast" }, fixture("ffi_echo_u64"), "ffi_echo_u64:fast"); + const echoF32 = bind("ffi_echo_f32", ["f32"], "f32"); + const echoF64 = bind("ffi_echo_f64", ["f64"], "f64"); + const echoBool = bind("ffi_echo_bool", ["bool"], "bool"); + const echoPtr = bind("ffi_echo_ptr", ["ptr"], "ptr"); + const echoNapiValue = bind("ffi_echo_jsvalue", ["napi_value"], "napi_value"); + const addI32 = bind("ffi_add_i32", ["i32", "i32"], "i32"); + const addF64 = bind("ffi_add_f64", ["f64", "f64"], "f64"); + const addI64 = bind("ffi_add_i64", ["i64", "i64"], "i64"); + const addU64 = bind("ffi_add_u64", ["u64", "u64"], "u64"); + const addF32 = bind("ffi_add_f32", ["f32", "f32"], "f32"); + const sumI32_9 = bind("ffi_sum_i32_9", ["i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32", "i32"], "i64"); + const sumF64_9 = bind("ffi_sum_f64_9", ["f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64", "f64"], "f64"); + const sumU8_12 = bind("ffi_sum_u8_12", ["u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8", "u8"], "i64"); + const sumI16_10 = bind("ffi_sum_i16_10", ["i16", "i16", "i16", "i16", "i16", "i16", "i16", "i16", "i16", "i16"], "i64"); + const mix1 = bind("ffi_mix_1", ["i32", "f64", "i64", "f32", "ptr", "u8", "f64", "i16", "f64", "i32"], "f64"); + const mix6 = bind("ffi_mix_6", ["bool", "bool", "i32", "bool", "f64", "bool", "f32", "bool", "bool", "bool", "bool", "bool", "bool"], "f64"); + const widenChar = bind("ffi_widen_char", ["char"], "i64_fast"); + const widenU16 = bind("ffi_widen_u16", ["u16"], "i64_fast"); + const twoAsBool = bind("ffi_ret_two_as_bool", [], "bool"); + const retNullPtr = bind("ffi_ret_null_ptr", [], "ptr"); + const highPtr = bind("ffi_high_ptr", [], "ptr"); + const retNegOneI8 = bind("ffi_ret_neg_one_i8", [], "i8"); + const retNegOneU32 = bind("ffi_ret_neg_one_u32", [], "u32"); + const retNegOneU64 = bind("ffi_ret_neg_one_u64", [], "u64"); + const retDenormalF32 = bind("ffi_ret_denormal_f32", [], "f32"); + const retNegZeroF64 = bind("ffi_ret_neg_zero_f64", [], "f64"); + const retInfF64 = bind("ffi_ret_inf_f64", [], "f64"); + + const sharedObject = { shared: true }; + + // Each row: [callable, [arguments...], expectedLiteral, label] + // Every expected value below is a literal, not a computed reference. + const battery = [ + [echoChar, [-1], -1, "char(-1)"], + [echoChar, [255], -1, "char(255)"], + [echoChar, [0x80], -128, "char(0x80)"], + [echoI8, [127], 127, "i8(127)"], + [echoI8, [128], -128, "i8(128)"], + [echoI8, [0x1ff], -1, "i8(0x1ff)"], + [echoU8, [-1], 255, "u8(-1)"], + [echoU8, [511], 255, "u8(511)"], + [echoU8, [256], 0, "u8(256)"], + [echoI16, [32768], -32768, "i16(32768)"], + [echoI16, [-32769], 32767, "i16(-32769)"], + [echoU16, [-1], 65535, "u16(-1)"], + [echoU16, [70000], 4464, "u16(70000)"], + [echoI32, [2147483648], -2147483648, "i32(2^31)"], + [echoI32, [-2147483649], 2147483647, "i32(-2^31-1)"], + [echoI32, [4294967301], 5, "i32(2^32+5)"], + [echoI32, [-1.9], -1, "i32(-1.9)"], + [echoI32, [NaN], 0, "i32(NaN)"], + [echoI32, [Infinity], 0, "i32(Infinity)"], + [echoI32, [undefined], 0, "i32(undefined)"], + [echoI32, [true], 1, "i32(true)"], + [echoU32, [-1], 4294967295, "u32(-1)"], + [echoU32, [2147483648], 2147483648, "u32(2^31)"], + [echoU32, [4294967296], 0, "u32(2^32)"], + [echoI64, [0], 0n, "i64(0)"], + [echoI64, [-1], -1n, "i64(-1)"], + [echoI64, [4294967296], 4294967296n, "i64(2^32)"], + [echoI64, [2n ** 63n - 1n], 9223372036854775807n, "i64(2^63-1)"], + [echoI64, [2n ** 63n], -9223372036854775808n, "i64(2^63)"], + [echoI64, [-1.5], -1n, "i64(-1.5)"], + [echoI64, [9007199254740992], 9007199254740992n, "i64(2^53 as number)"], + [echoU64, [-1], 18446744073709551615n, "u64(-1)"], + [echoU64, [-2147483648], 18446744071562067968n, "u64(-2^31)"], + [echoU64, [2n ** 64n + 3n], 3n, "u64(2^64+3)"], + [echoI64Fast, [9007199254740991], 9007199254740991, "i64_fast(2^53-1)"], + [echoI64Fast, [-9007199254740991], -9007199254740991, "i64_fast(-(2^53-1))"], + [echoI64Fast, [2n ** 53n], 9007199254740992n, "i64_fast(2^53)"], + [echoI64Fast, [-(2n ** 53n)], -9007199254740992n, "i64_fast(-2^53)"], + [echoI64Fast, [-1], -1, "i64_fast(-1)"], + [echoU64Fast, [9007199254740990], 9007199254740990, "u64_fast(2^53-2)"], + [echoU64Fast, [2n ** 53n - 1n], 9007199254740991n, "u64_fast(2^53-1)"], + [echoU64Fast, [-1], 18446744073709551615n, "u64_fast(-1)"], + [echoF32, [1.1], 1.100000023841858, "f32(1.1)"], + [echoF32, [-0], -0, "f32(-0)"], + [echoF32, [NaN], NaN, "f32(NaN)"], + [echoF32, [1e39], Infinity, "f32(1e39)"], + [echoF32, [16777217], 16777216, "f32(2^24+1)"], + [echoF64, [-0], -0, "f64(-0)"], + [echoF64, [NaN], NaN, "f64(NaN)"], + [echoF64, [Number.MIN_VALUE], 5e-324, "f64(min denormal)"], + [echoF64, [undefined], NaN, "f64(undefined) -> NaN"], + [echoBool, [2], true, "bool(2)"], + [echoBool, [-1], true, "bool(-1)"], + [echoBool, [0], false, "bool(0)"], + [echoBool, [0.5], true, "bool(0.5)"], + [echoBool, [-0], false, "bool(-0)"], + [echoBool, [NaN], false, "bool(NaN)"], + [echoBool, [256], true, "bool(256)"], + [echoBool, [null], false, "bool(null)"], + [echoPtr, [0], null, "ptr(0)"], + [echoPtr, [null], null, "ptr(null)"], + [echoPtr, [undefined], null, "ptr(undefined)"], + [echoPtr, [-1], 18446744073709551615n, "ptr(-1) (exact BigInt, > 2^53)"], + [echoPtr, [1099511627776], 1099511627776, "ptr(2^40)"], + [echoNapiValue, [sharedObject], sharedObject, "napi_value(object)"], + [echoNapiValue, ["x"], "x", "napi_value(string)"], + [echoNapiValue, [-0], -0, "napi_value(-0)"], + [addI32, [2147483647, 1], -2147483648, "add_i32 overflow"], + [addI32, [-2147483648, -1], 2147483647, "add_i32 underflow"], + [addI32, [7], 7, "add_i32 missing argument"], + [addI32, [7, 8, 9], 15, "add_i32 extra argument"], + [addF64, [0.1, 0.2], 0.30000000000000004, "add_f64(0.1, 0.2)"], + [addF64, [-0, -0], -0, "add_f64(-0, -0)"], + [addF64, [Infinity, -Infinity], NaN, "add_f64(inf, -inf)"], + [addI64, [2n ** 63n - 1n, 1n], -9223372036854775808n, "add_i64 wrap"], + [addI64, [-1, -1], -2n, "add_i64(-1,-1)"], + [addU64, [-1, 2], 1n, "add_u64 wrap"], + [addU64, [2n ** 32n, 2n ** 32n], 8589934592n, "add_u64(2^32,2^32)"], + [addF32, [16777216, 1], 16777216, "add_f32 precision loss"], + [addF32, [0.5, 0.25], 0.75, "add_f32 dyadics"], + [addF32, [3.4e38, 3.4e38], Infinity, "add_f32 overflow to +inf"], + [sumI32_9, [1, -2, 3, -4, 5, -6, 7, -8, 100000], 99996n, "sum_i32_9"], + [sumI32_9, [2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647, 2147483647], 19327352823n, "sum_i32_9 max"], + [sumF64_9, [1, 0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, 0.0078125, 0.00390625], 1.99609375, "sum_f64_9 dyadics"], + [sumU8_12, [255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255], 3060n, "sum_u8_12 max"], + [sumU8_12, [1, 2, 4, 8, 16, 32, 64, 128, -1, 256, 257, 511], 766n, "sum_u8_12 wrapped powers"], + [sumI16_10, [-32768, -32768, 32767, 32767, -1, 1, 40000, -40000, 65535, 65536], -3n, "sum_i16_10 edges"], + [mix1, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 385, "mix_1 identity ramp"], + [mix1, [-2147483648, -0.5, -1000000, -1.5, 4096, 255, 0, -32768, 2, 2147483647], 19324112699, "mix_1 edges"], + [mix6, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 91, "mix_6 all ones"], + [mix6, [2, 0, -3, -1, -0.25, NaN, 0.5, 256, true, false, null, undefined, true], 28.25, "mix_6 truthiness edges"], + [widenChar, [-1], -1, "widen_char(-1)"], + [widenChar, [200], -56, "widen_char(200)"], + [widenU16, [-1], 65535, "widen_u16(-1)"], + [twoAsBool, [], true, "ret_two_as_bool"], + [retNullPtr, [], null, "ret_null_ptr"], + [highPtr, [], 0x00007fffdeadbee0, "high_ptr"], + [retNegOneI8, [], -1, "ret_neg_one_i8"], + [retNegOneU32, [], 4294967295, "ret_neg_one_u32"], + [retNegOneU64, [], 18446744073709551615n, "ret_neg_one_u64"], + [retDenormalF32, [], 2 ** -149, "ret_denormal_f32"], + [retNegZeroF64, [], -0, "ret_neg_zero_f64"], + [retInfF64, [], Infinity, "ret_inf_f64"], + ]; + + function runBattery(phase) { + for (const [callable, args, expected, label] of battery) { + const actual = callable(...args); + if (!Object.is(actual, expected)) + throw new Error(phase + " " + label + ": expected " + describe(expected) + " but got " + describe(actual)); + } + } + + // Cold pass (whatever tier the harness starts in). + runBattery("cold"); + // Warm each row through its OWN exact-arity, non-spread, single-callee + // caller so that caller tiers up with a monomorphic CallFFI site, then + // re-run the whole battery. + for (const [callable, args, expected, label] of battery) { + const caller = makeMonomorphicCaller(args.length); + for (let i = 0; i < 4000; ++i) { + const actual = caller(callable, args); + if (!Object.is(actual, expected)) + throw new Error("warm " + label + " iteration " + i + ": expected " + describe(expected) + " but got " + describe(actual)); + } + } + runBattery("hot"); + // A few thousand mixed iterations across every row (megamorphic-ish). + for (let i = 0; i < 6000; ++i) { + const [callable, args, expected, label] = battery[i % battery.length]; + const actual = callable(...args); + if (!Object.is(actual, expected)) + throw new Error("mixed " + label + " iteration " + i + ": expected " + describe(expected) + " but got " + describe(actual)); + } +} + +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-typedarray-storage-modes.js b/JSTests/stress/ffi-typedarray-storage-modes.js new file mode 100644 index 0000000000000..a6fffc26f4dd6 --- /dev/null +++ b/JSTests/stress/ffi-typedarray-storage-modes.js @@ -0,0 +1,139 @@ +//@ requireOptions("--useDollarVM=1") + +// TypedArray storage modes vs. the ptr/buffer argument fast paths (SPEC +// sections 5 and 8.3): a JSArrayBufferView's vector lives in different +// places depending on its mode -- +// FastTypedArray small array, GC-auxiliary storage, no ArrayBuffer +// OversizeTypedArray large array, malloc'd (Gigacage) storage, no ArrayBuffer +// WastefulTypedArray ArrayBuffer-backed (created up front, or MATERIALIZED +// on demand by touching .buffer, which MOVES the storage: +// JSArrayBufferView::slowDownAndWasteMemory) +// The IC stub / DFG / FTL read the vector with a single offsetOfVector() load, +// so every mode must yield the right pointer, and a mode transition between +// two calls must be picked up (the freshly materialized buffer, not the stale +// pre-transition storage). None of this may crash the engine at any tier. + +// The stress harness also runs every file with the JIT disabled (lockdown / +// no-jit configs), where bun:ffi creation throws by design (SPEC section +// 0.1); like every ffi-*.js file, gate the body on $vm.useJIT() +// (ffi-no-jit.js covers the no-JIT behavior explicitly). +if (!$vm.useJIT()) + quit(); + +const identity = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, $vm.ffiFixture("ffi_ptr_identity"), "ffi_ptr_identity"); +const readU32 = $vm.ffiFunction({ args: ["ptr"], returns: "u32" }, $vm.ffiFixture("ffi_ptr_read_u32"), "ffi_ptr_read_u32"); +const writeU32 = $vm.ffiFunction({ args: ["ptr", "u32"], returns: "void" }, $vm.ffiFixture("ffi_ptr_write_u32"), "ffi_ptr_write_u32"); +const bufferIdentity = $vm.ffiFunction({ args: ["buffer"], returns: "ptr" }, $vm.ffiFixture("ffi_ptr_identity"), "ffi_buffer_identity"); +const callVoid = $vm.ffiFunction({ args: ["ptr"], returns: "void" }, $vm.ffiFixture("ffi_call_cb_void"), "ffi_call_cb_void"); + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + String(expected) + " but got " + String(actual)); +} + +function currentPtr(view) { return identity(view); } + +function main() { + // ---- 1. Fast (small) vs Oversize (large) vs Wasteful (buffer-backed) views: + // pointer identity and read-through must agree across many iterations + // (hot enough for baseline -> DFG -> FTL on the call sites below). + const fast = new Uint32Array(4); // FastTypedArray + fast[0] = 0xF00D; + const oversize = new Uint32Array(1 << 16); // OversizeTypedArray (256KB) + oversize[0] = 0x0517E; + const wasteful = new Uint32Array(new ArrayBuffer(64)); // Wasteful from birth + wasteful[0] = 0xBEEF; + + for (let i = 0; i < 2e4; ++i) { + check(readU32(fast), 0xF00D, "fast read"); + check(readU32(wasteful), 0xBEEF, "wasteful read"); + // The buffer-typed argument path (requires a view; §5) sees the same storage. + check(bufferIdentity(fast) === currentPtr(fast), true, "buffer arg == ptr arg (fast)"); + // Write then read back through the oversize (malloc'd Gigacage) storage. + writeU32(oversize, i & 0xffff); + check(oversize[0], i & 0xffff, "oversize write visible to JS"); + check(readU32(oversize), (i & 0xffff) >>> 0, "oversize read after write"); + } + + // ---- 2. Materialize .buffer AFTER the call site is hot: the storage + // MOVES (slowDownAndWasteMemory). Subsequent calls must see the new + // location, and reads/writes must round-trip through it. + const migrant = new Uint32Array(4); // starts Fast + migrant[0] = 0xAAAA; + for (let i = 0; i < 2e4; ++i) + check(readU32(migrant), 0xAAAA, "pre-materialization read"); + const before = currentPtr(migrant); + const buffer = migrant.buffer; // <-- materialize: mode transition happens here + const after = currentPtr(migrant); + check(migrant[0], 0xAAAA, "contents survive materialization"); + check(readU32(migrant), 0xAAAA, "read after materialization (same hot site)"); + if (buffer.byteLength !== 16) + throw new Error("materialized buffer has wrong length: " + buffer.byteLength); + // The engine reads the vector fresh on every call, so writes through the + // NEW storage are visible; a stale cached pre-transition pointer would not be. + writeU32(migrant, 0xC0FFEE); + check(migrant[0], 0xC0FFEE, "write after materialization lands in the new storage"); + check(new Uint32Array(buffer)[0], 0xC0FFEE, "write visible through the materialized buffer"); + // (before/after may or may not differ numerically depending on the + // allocator; the invariant is behavior, asserted above, not the address.) + void before; void after; + + // Keep the hot site polymorphic across modes so one compiled body sees all three. + const rotation = [fast, oversize, wasteful, migrant]; + for (let i = 0; i < 4e4; ++i) { + const view = rotation[i & 3]; + const expected = view[0]; + check(readU32(view), expected, "rotating-mode read " + (i & 3)); + } + + // ---- 3. A view over a SLICE of a buffer (byteOffset != 0): the pointer + // must include the offset (vector() already does; this pins it). + const slab = new ArrayBuffer(64); + const whole = new Uint32Array(slab); + whole[3] = 0xDEAD10; + const sliced = new Uint32Array(slab, 12, 4); // byteOffset 12 -> element index 3 + for (let i = 0; i < 2e4; ++i) + check(readU32(sliced), 0xDEAD10, "sliced view reads at its byteOffset"); + writeU32(sliced, 0x51CE); + check(whole[3], 0x51CE, "write through sliced view lands at the offset"); + + // ---- 4. Mode transition triggered INSIDE the FFI call by a JS callback, + // while native code (conceptually) still holds the pre-call vector. This + // is the documented user-error case (like detach-during-callback); the + // engine's contract is only that it does not crash and stays consistent + // AFTER the call. The callback materializes .buffer on a Fast view. + const trigger = new Uint32Array(4); + trigger[0] = 0x1EAD; + const materializeInCallback = $vm.ffiCallback({ args: [], returns: "void" }, () => { + trigger.buffer; // slowDownAndWasteMemory mid-FFI-call + }); + for (let i = 0; i < 5e3; ++i) { + // Pass the callback's native entrypoint as a plain pointer; the fixture calls it. + callVoid(materializeInCallback.ptr); + check(readU32(trigger), 0x1EAD, "read of the (already-materialized) view after callback " + i); + } + + // ---- 5. Detach hot: the same site sees a live view, then that view is + // detached; conversion yields a null pointer (0) from then on (§5), at every tier. + const doomed = new ArrayBuffer(32); + const doomedView = new Uint32Array(doomed); + for (let i = 0; i < 2e4; ++i) + currentPtr(doomedView); + check(currentPtr(doomedView) !== null && currentPtr(doomedView) !== 0, true, "live view yields a non-null pointer"); + if (typeof doomed.transfer === "function") + doomed.transfer(); + else if (typeof transferArrayBuffer === "function") + transferArrayBuffer(doomed); + else + return; // No detach primitive in this shell; sections 1-4 still ran. + check(doomedView.length, 0, "view is detached"); + for (let i = 0; i < 100; ++i) + check(currentPtr(doomedView), null, "detached view -> null pointer after tier-up (iteration " + i + ")"); +} +noInline(main); + +main(); +// A full GC and a second run make the storage-lifetime story exercise +// reclamation of the (now-unreferenced) pre-materialization storage. +gc(); +main(); diff --git a/JSTests/stress/ffi-types-echo.js b/JSTests/stress/ffi-types-echo.js new file mode 100644 index 0000000000000..54204acd8c42e --- /dev/null +++ b/JSTests/stress/ffi-types-echo.js @@ -0,0 +1,210 @@ +//@ requireOptions("--useDollarVM=1") + +// Every FFI type through its ffi_echo_* fixture with edge values, cold and +// in a loop hot enough to reach the optimizing tiers; results must be +// identical across tiers (SPEC sections 5 and 11.4). + +function describe(value) { + if (typeof value === "bigint") + return String(value) + "n"; + if (typeof value === "symbol") + return value.toString(); + if (Object.is(value, -0)) + return "-0"; + return String(value); +} + +function check(actual, expected, message) { + if (!Object.is(actual, expected)) + throw new Error(message + ": expected " + describe(expected) + " but got " + describe(actual)); +} + +// Returns a NEW caller function every time (a distinct FunctionExecutable / +// CodeBlock via `new Function`), so the FFI call site inside it is +// monomorphic, exact-arity and non-spread -- the only call-site shape the DFG +// ByteCodeParser constant-callee feed and the strength-reduction +// Call -> CallFFI conversion accept (SPEC section 10.2). Warming the FFI +// functions through one shared `fn(x)` site in a table loop would leave that +// site polymorphic, and the typed CallFFI path would never be compiled. +function makeMonomorphicCaller(arity) { + const argumentList = Array.from({ length: arity }, (_, i) => "args[" + i + "]").join(", "); + return new Function("callable", "args", "return callable(" + argumentList + ");"); +} + +function main() { + const fixture = name => $vm.ffiFixture(name); + const echo = (fixtureName, type, name) => $vm.ffiFunction({ args: [type], returns: type }, fixture(fixtureName), name || (fixtureName + ":" + type)); + + const echoChar = echo("ffi_echo_char", "char"); + const echoI8 = echo("ffi_echo_i8", "i8"); + const echoU8 = echo("ffi_echo_u8", "u8"); + const echoI16 = echo("ffi_echo_i16", "i16"); + const echoU16 = echo("ffi_echo_u16", "u16"); + const echoI32 = echo("ffi_echo_i32", "i32"); + const echoU32 = echo("ffi_echo_u32", "u32"); + const echoI64 = echo("ffi_echo_i64", "i64"); + const echoU64 = echo("ffi_echo_u64", "u64"); + const echoI64Fast = echo("ffi_echo_i64", "i64_fast", "ffi_echo_i64:i64_fast"); + const echoU64Fast = echo("ffi_echo_u64", "u64_fast", "ffi_echo_u64:u64_fast"); + const echoF32 = echo("ffi_echo_f32", "f32"); + const echoF64 = echo("ffi_echo_f64", "f64"); + const echoBool = echo("ffi_echo_bool", "bool"); + const echoPtr = echo("ffi_echo_ptr", "ptr"); + const echoCString = echo("ffi_echo_cstring", "cstring"); + const echoNapiValue = echo("ffi_echo_jsvalue", "napi_value"); + + const object = { tag: "object" }; + const symbol = Symbol("napi"); + + const batteries = [ + [echoChar, "char", [ + [0, 0], [1, 1], [-1, -1], [127, 127], [-128, -128], [128, -128], [255, -1], [0x1ff, -1], + [-129, 127], [200, -56], [true, 1], [false, 0], [undefined, 0], [null, 0], [1.9, 1], [-1.9, -1], + ]], + [echoI8, "i8", [ + [0, 0], [127, 127], [-128, -128], [128, -128], [255, -1], [256, 0], [-129, 127], [0.9, 0], + [true, 1], [undefined, 0], [null, 0], [NaN, 0], [Infinity, 0], [-Infinity, 0], + ]], + [echoU8, "u8", [ + [0, 0], [255, 255], [256, 0], [-1, 255], [511, 255], [-129, 127], [300.7, 44], [128, 128], + [true, 1], [false, 0], [undefined, 0], [null, 0], [NaN, 0], [-0, 0], + ]], + [echoI16, "i16", [ + [0, 0], [32767, 32767], [32768, -32768], [-32769, 32767], [-1, -1], [65535, -1], [0x12345, 0x2345], + [-32768, -32768], [65536, 0], [1.5, 1], [undefined, 0], [true, 1], + ]], + [echoU16, "u16", [ + [0, 0], [65535, 65535], [65536, 0], [-1, 65535], [70000, 4464], [32768, 32768], [-32768, 32768], + [undefined, 0], [null, 0], [NaN, 0], [true, 1], + ]], + [echoI32, "i32", [ + [0, 0], [1, 1], [-1, -1], [2147483647, 2147483647], [-2147483648, -2147483648], + [2147483648, -2147483648], [-2147483649, 2147483647], [4294967301, 5], [4294967296, 0], + [-1.5, -1], [1.9, 1], [-0.9, 0], [0.5, 0], [-0, 0], [NaN, 0], [Infinity, 0], [-Infinity, 0], + [1e10, 1410065408], [undefined, 0], [null, 0], [true, 1], [false, 0], + ]], + [echoU32, "u32", [ + [0, 0], [-1, 4294967295], [4294967295, 4294967295], [4294967296, 0], [2147483648, 2147483648], + [2147483647, 2147483647], [-0.5, 0], [1e10, 1410065408], [-2147483648, 2147483648], [undefined, 0], + [null, 0], [true, 1], [NaN, 0], + ]], + [echoI64, "i64", [ + [0, 0n], [1, 1n], [-1, -1n], [2 ** 53, 9007199254740992n], [-(2 ** 53), -9007199254740992n], + [123n, 123n], [-123n, -123n], [2n ** 63n - 1n, 9223372036854775807n], [-(2n ** 63n), -9223372036854775808n], + [2n ** 64n + 7n, 7n], [2n ** 63n, -9223372036854775808n], [-1.5, -1n], [2.9, 2n], [-0, 0n], + [2147483647, 2147483647n], [-2147483648, -2147483648n], [4294967296, 4294967296n], + ]], + [echoU64, "u64", [ + [0, 0n], [1, 1n], [-1, 18446744073709551615n], [4294967295, 4294967295n], [2n ** 64n - 1n, 18446744073709551615n], + [2n ** 64n + 7n, 7n], [-2n, 18446744073709551614n], [2.5, 2n], [2 ** 53, 9007199254740992n], + [-2147483648, 18446744071562067968n], [9007199254740991, 9007199254740991n], + ]], + [echoI64Fast, "i64_fast", [ + [0, 0], [-1, -1], [42, 42], [2 ** 53 - 1, 9007199254740991], [-(2 ** 53 - 1), -9007199254740991], + [2n ** 53n, 9007199254740992n], [-(2n ** 53n), -9007199254740992n], [1n << 62n, 4611686018427387904n], + [2n ** 63n - 1n, 9223372036854775807n], [-2, -2], [3.7, 3], [-3.7, -3], + ]], + [echoU64Fast, "u64_fast", [ + [0, 0], [123, 123], [2 ** 53 - 2, 9007199254740990], [2n ** 53n - 1n, 9007199254740991n], + [2n ** 53n, 9007199254740992n], [-1, 18446744073709551615n], [2n ** 64n - 1n, 18446744073709551615n], + [4.9, 4], + ]], + [echoF32, "f32", [ + [0, 0], [-0, -0], [1.5, 1.5], [1.1, Math.fround(1.1)], [-1.1, Math.fround(-1.1)], [NaN, NaN], + [Infinity, Infinity], [-Infinity, -Infinity], [3.4e38, Math.fround(3.4e38)], [3.5e38, Infinity], + [1e-45, Math.fround(1e-45)], [16777217, 16777216], [-16777217, -16777216], [2 ** -149, 2 ** -149], + ]], + [echoF64, "f64", [ + [0, 0], [-0, -0], [1.5, 1.5], [NaN, NaN], [Infinity, Infinity], [-Infinity, -Infinity], + [Number.MAX_VALUE, Number.MAX_VALUE], [Number.MIN_VALUE, Number.MIN_VALUE], [Number.EPSILON, Number.EPSILON], + [Math.PI, Math.PI], [-1e308, -1e308], [undefined, NaN], [123456789.123456789, 123456789.123456789], + ]], + [echoBool, "bool", [ + [0, false], [1, true], [2, true], [-1, true], [0.5, true], [-0, false], [NaN, false], [Infinity, true], + [true, true], [false, false], [undefined, false], [null, false], [255, true], [256, true], + ]], + [echoPtr, "ptr", [ + [0, null], [null, null], [undefined, null], [4096, 4096], [1, 1], [-1, 18446744073709551615n], + [2 ** 40, 2 ** 40], [0.9, null], [-0, null], [65536.7, 65536], + ]], + [echoCString, "cstring", [ + [0, null], [null, null], [undefined, null], + ["hello", "hello"], ["", ""], ["\u00e9\u2603 utf8", "\u00e9\u2603 utf8"], + ]], + [echoNapiValue, "napi_value", [ + [0, 0], [42, 42], [-0, -0], [NaN, NaN], [1.5, 1.5], [undefined, undefined], [null, null], [true, true], + [false, false], ["string", "string"], [object, object], [symbol, symbol], [9007199254740993n, 9007199254740993n], + ]], + ]; + + // Cold: every case exactly once, warm-up free. + for (const [fn, label, cases] of batteries) { + for (const [input, expected] of cases) + check(fn(input), expected, label + " cold echo(" + describe(input) + ")"); + } + + // Hot: enough iterations to tier the per-battery caller through baseline + // into DFG/FTL, where the exact-arity monomorphic call site becomes a + // typed CallFFI node (ffi-callffi-was-compiled.js machine-checks that + // conversion via $vm.ffiCompileCounts()). + for (const [fn, label, cases] of batteries) { + // (a) Monomorphic input through a dedicated single-callee caller: the + // typed CallFFI fast path is what the loop settles on. + const monoCaller = makeMonomorphicCaller(1); + const [monoInput, monoExpected] = cases[1]; + const monoArgs = [monoInput]; + for (let i = 0; i < 4e4; ++i) { + const result = monoCaller(fn, monoArgs); + if (!Object.is(result, monoExpected)) + throw new Error(label + " hot mono iteration " + i + ": expected " + describe(monoExpected) + " but got " + describe(result)); + } + // (b) Mixed inputs cycling through every edge case at another dedicated + // single-callee site (forces exits, slow paths and re-optimization). + const mixedCaller = makeMonomorphicCaller(1); + const inputs = cases.map(c => [c[0]]); + for (let i = 0; i < 1.5e4; ++i) { + const k = i % cases.length; + const result = mixedCaller(fn, inputs[k]); + if (!Object.is(result, cases[k][1])) + throw new Error(label + " hot mixed iteration " + i + " echo(" + describe(cases[k][0]) + "): expected " + describe(cases[k][1]) + " but got " + describe(result)); + } + } + + // char signedness lock across a dedicated widening fixture. + const widenChar = $vm.ffiFunction({ args: ["char"], returns: "i64_fast" }, fixture("ffi_widen_char"), "ffi_widen_char"); + check(widenChar(-1), -1, "ffi_widen_char(-1)"); + check(widenChar(255), -1, "ffi_widen_char(255)"); + check(widenChar(128), -128, "ffi_widen_char(128)"); + check(widenChar(127), 127, "ffi_widen_char(127)"); + for (let i = 0; i < 2e4; ++i) { + if (widenChar(-1) !== -1) + throw new Error("ffi_widen_char(-1) !== -1 in hot loop"); + } + + // typeof edges of the fast 64-bit variants. + check(typeof echoI64Fast(2 ** 53 - 2), "number", "i64_fast typeof at 2^53-2"); + check(typeof echoI64Fast(2 ** 53 - 1), "number", "i64_fast typeof at 2^53-1"); + check(typeof echoI64Fast(2n ** 53n), "bigint", "i64_fast typeof at 2^53"); + check(typeof echoI64Fast(-(2 ** 53 - 1)), "number", "i64_fast typeof at -(2^53-1)"); + check(typeof echoI64Fast(-(2n ** 53n)), "bigint", "i64_fast typeof at -2^53"); + check(typeof echoU64Fast(2 ** 53 - 2), "number", "u64_fast typeof at 2^53-2"); + check(typeof echoU64Fast(2n ** 53n - 1n), "bigint", "u64_fast typeof at 2^53-1 (strict < quirk)"); + check(typeof echoU64Fast(2n ** 53n), "bigint", "u64_fast typeof at 2^53"); + check(typeof echoI64(0), "bigint", "i64 is always a BigInt"); + check(typeof echoU64(0), "bigint", "u64 is always a BigInt"); + + // Debug builds must not assert on NaN through f32/f64 (purifyNaN). + for (let i = 0; i < 1e4; ++i) { + if (!Number.isNaN(echoF32(NaN))) + throw new Error("echo f32 NaN not NaN"); + if (!Number.isNaN(echoF64(NaN))) + throw new Error("echo f64 NaN not NaN"); + } +} + +// FFI-SPEC-GAP: the stress harness also runs every file under --useJIT=false +// (lockdown/no-jit configs), where bun:ffi creation throws by design (SPEC +// section 0.1). Every ffi-*.js file therefore gates its body on $vm.useJIT(); +// ffi-no-jit.js covers the no-JIT behavior explicitly. +if ($vm.useJIT()) + main(); diff --git a/JSTests/stress/ffi-untyped-float-args.js b/JSTests/stress/ffi-untyped-float-args.js new file mode 100644 index 0000000000000..8d7d8b797638a --- /dev/null +++ b/JSTests/stress/ffi-untyped-float-args.js @@ -0,0 +1,51 @@ +//@ requireOptions("--useDollarVM=1") +// An f32/f64 parameter whose call site does NOT speculate a number (FFIDFG falls back to +// UntypedUse) has no single SSA operand: the value is converted into the canonical slot by +// operationFFIWriteSlot. The FTL direct call must reload that slot AS A FLOAT so the value goes +// out in an FPR -- reloading it as an integer sends the bits in a GPR and the callee reads junk. +if (!$vm.useJIT()) quit(); + +const fixture = name => $vm.ffiFixture(name); +const addF64 = $vm.ffiFunction({ args: ["f64", "f64"], returns: "f64" }, fixture("ffi_add_f64"), "add_f64"); +const addF32 = $vm.ffiFunction({ args: ["f32", "f32"], returns: "f32" }, fixture("ffi_add_f32"), "add_f32"); +const echoF64 = $vm.ffiFunction({ args: ["f64"], returns: "f64" }, fixture("ffi_echo_f64"), "echo_f64"); + +// Oracles pinned below the DFG. +function refAddF64(a, b) { return addF64(a, b); } +function refAddF32(a, b) { return addF32(a, b); } +function refEchoF64(a) { return echoF64(a); } +noDFG(refAddF64); noDFG(refAddF32); noDFG(refEchoF64); +noInline(refAddF64); noInline(refAddF32); noInline(refEchoF64); + +function hotAddF64(a, b) { return addF64(a, b); } +function hotAddF32(a, b) { return addF32(a, b); } +function hotEchoF64(a) { return echoF64(a); } +noInline(hotAddF64); noInline(hotAddF32); noInline(hotEchoF64); + +let failures = 0; +function agree(label, hot, ref) { + const same = Object.is(hot, ref); + if (!same) { print(`MISMATCH [${label}]: hot=${String(hot)} ref=${String(ref)}`); if (++failures > 8) throw new Error("too many mismatches"); } +} + +// Poison the argument prediction so FFIDFG picks UntypedUse: feed values that are numbers most of +// the time but sometimes null/undefined/boolean, so neither shouldSpeculateDoubleReal() nor +// shouldSpeculateNumber() holds at the call site. +const poison = [1.5, 2.25, null, undefined, true, false, -0.5, 1e300, NaN, 0]; +const iterations = 30000; +for (let i = 0; i < iterations; ++i) { + const a = poison[i % poison.length]; + const b = poison[(i + 3) % poison.length]; + agree(`addF64#${i}`, hotAddF64(a, b), refAddF64(a, b)); + agree(`addF32#${i}`, hotAddF32(a, b), refAddF32(a, b)); + agree(`echoF64#${i}`, hotEchoF64(a), refEchoF64(a)); +} + +// And the plain numeric case must still be exact after all that. +for (let i = 0; i < 5000; ++i) { + agree(`exact#${i}`, hotAddF64(1.5, 2.25), 3.75); + agree(`exactEcho#${i}`, hotEchoF64(1e300), 1e300); +} + +if (failures) throw new Error(`${failures} mismatch(es)`); +print("ffi untyped float args: all checks passed"); diff --git a/JSTests/stress/ffi-untyped-int-stack-args.js b/JSTests/stress/ffi-untyped-int-stack-args.js new file mode 100644 index 0000000000000..e4d17a4eb3d65 --- /dev/null +++ b/JSTests/stress/ffi-untyped-int-stack-args.js @@ -0,0 +1,25 @@ +//@ requireOptions("--useDollarVM=1") +// Regression: with int32 speculation gated on profiling, an i32/u32 parameter can carry an +// UntypedUse edge. On the FTL DIRECT-call path that operand must be reloaded as a B3 Int32; as an +// Int64 CCallValue lays a STACK argument at 8-byte stride, but Darwin/arm64 packs a stacked +// int32_t at 4-byte natural stride, so the 9th/10th arguments (spilled past the 8 GPRs) would +// shift. Poison the profile so the args stay UntypedUse, pass values whose position matters +// (weighted sum), and compare the FTL-hot twin against a noDFG oracle. +if (!$vm.useJIT()) quit(); +const sum10 = $vm.ffiFunction({ args: ["i32","i32","i32","i32","i32","i32","i32","i32","i32","i32"], returns: "i64" }, + $vm.ffiFixture("ffi_sum_i32_x10"), "sum_i32_x10"); +function ref(a,b,c,d,e,f,g,h,i,j) { return sum10(a,b,c,d,e,f,g,h,i,j); } +function hot(a,b,c,d,e,f,g,h,i,j) { return sum10(a,b,c,d,e,f,g,h,i,j); } +noDFG(ref); noInline(ref); noInline(hot); +let failures = 0; +// Mix ints with booleans / null / doubles so FFIDFG's shouldSpeculateInt32() gate falls to +// UntypedUse for these arguments (the exact edge kind the direct-call reload must handle). +const vals = [7, true, null, 3, false, 2.0, 9, 1, 12345, -6, 100000, 4]; +for (let it = 0; it < 60000; ++it) { + const a = vals[it % vals.length], b = vals[(it + 1) % vals.length], c = 5, d = -1, e = it & 7, f = 8, + g = vals[(it + 3) % vals.length], h = 2, i = it & 3, j = vals[(it + 5) % vals.length]; + const hv = hot(a,b,c,d,e,f,g,h,i,j), rv = ref(a,b,c,d,e,f,g,h,i,j); + if (hv !== rv) { print(`MISMATCH it=${it}: hot=${hv} ref=${rv} args=${[a,b,c,d,e,f,g,h,i,j]}`); if (++failures > 5) throw new Error("stack arg stride mismatch"); } +} +if (failures) throw new Error(failures + " mismatches"); +print("ffi untyped int stack args: all checks passed"); diff --git a/JSTests/stress/ffi-view-args.js b/JSTests/stress/ffi-view-args.js new file mode 100644 index 0000000000000..2f63c9e7b4867 --- /dev/null +++ b/JSTests/stress/ffi-view-args.js @@ -0,0 +1,191 @@ +//@ requireOptions("--useDollarVM=1") +// Pointer-family FFI arguments accept typed-array / DataView VIEWS directly, resolved inline in the +// DFG and (now) the FTL to the view's data pointer. This is a per-call tier-differential test: every +// hot (FTL-bound) function has a noDFG-pinned twin as the interpreter/baseline oracle, and the two +// must agree on EVERY iteration -- for every view type, storage mode, and the guard cases (detached, +// shared / resizable) that must punt to the C++ conversion in every tier. +if (!$vm.useJIT()) quit(); + +const fixture = name => $vm.ffiFixture(name); +const identity = $vm.ffiFunction({ args: ["ptr"], returns: "ptr" }, fixture("ffi_ptr_identity"), "ffi_ptr_identity"); +const strlen = $vm.ffiFunction({ args: ["cstring"], returns: "u64" }, fixture("ffi_strlen"), "ffi_strlen"); +const bufIdentity = $vm.ffiFunction({ args: ["buffer"], returns: "ptr" }, fixture("ffi_ptr_identity"), "ffi_ptr_identity(buffer)"); +const readU32 = $vm.ffiFunction({ args: ["ptr"], returns: "u32" }, fixture("ffi_ptr_read_u32"), "ffi_ptr_read_u32"); + +// Oracle twins: same call, pinned below the DFG, so their result is the interpreter/baseline +// (out-of-line C++ conversion) answer that the JIT tiers must match exactly. +function refIdentity(v) { return identity(v); } +function refBufIdentity(v) { return bufIdentity(v); } +function refStrlen(v) { return strlen(v); } +noDFG(refIdentity); noDFG(refBufIdentity); noDFG(refStrlen); +noInline(refIdentity); noInline(refBufIdentity); noInline(refStrlen); + +function hotIdentity(v) { return identity(v); } +function hotBufIdentity(v) { return bufIdentity(v); } +function hotStrlen(v) { return strlen(v); } +noInline(hotIdentity); noInline(hotBufIdentity); noInline(hotStrlen); + +let failures = 0; +function agree(label, hot, ref) { + // A pointer is a number, jsNull for 0, or an exact BigInt above 2^53 -- compare exactly. + if (hot !== ref) { + print(`TIER MISMATCH [${label}]: hot=${String(hot)} ref=${String(ref)}`); + if (++failures > 8) throw new Error("too many tier mismatches"); + } +} + +const iterations = 30000; + +// --------------------------------------------------------------------------------------------- +// 1. Every view type resolves to base + byteOffset, and stays tier-stable. +// --------------------------------------------------------------------------------------------- +const backing = new ArrayBuffer(512); +const views = [ + ["Int8Array", new Int8Array(backing, 8)], + ["Uint8Array", new Uint8Array(backing, 16)], + ["Uint8ClampedArray", new Uint8ClampedArray(backing, 24)], + ["Int16Array", new Int16Array(backing, 32)], + ["Uint16Array", new Uint16Array(backing, 40)], + ["Int32Array", new Int32Array(backing, 48)], + ["Uint32Array", new Uint32Array(backing, 56)], + ["Float32Array", new Float32Array(backing, 64)], + ["Float64Array", new Float64Array(backing, 72)], + ["BigInt64Array", new BigInt64Array(backing, 80)], + ["BigUint64Array", new BigUint64Array(backing, 88)], + ["DataView", new DataView(backing, 96)], +]; +const basePtr = refIdentity(new Uint8Array(backing)); +if (typeof basePtr !== "number" || basePtr === 0) + throw new Error("bad base pointer: " + String(basePtr)); +for (const [name, view] of views) { + const expected = basePtr + view.byteOffset; + if (refIdentity(view) !== expected) + throw new Error(`${name}: byteOffset not applied by the reference path: ${refIdentity(view)} vs ${expected}`); +} +for (let i = 0; i < iterations; ++i) { + for (const [name, view] of views) { + agree(`identity(${name})#${i}`, hotIdentity(view), refIdentity(view)); + agree(`buffer(${name})#${i}`, hotBufIdentity(view), refBufIdentity(view)); + } +} + +// --------------------------------------------------------------------------------------------- +// 2. Storage modes: a Fast (GC-heap vector) view whose .buffer is materialized MID-LOOP moves to +// Wasteful storage (the vector may be re-pointed); an Oversize (Gigacage) view; a subarray. +// Whatever the pointer is at each instant, both tiers must agree on it -- and the pointer must +// still address the LIVE bytes (proven by reading through it). +// --------------------------------------------------------------------------------------------- +const fastView = new Uint32Array(64); // Fast: small, GC-heap vector +const oversize = new Uint8Array(4 * 1024 * 1024); // Oversize: Gigacage-allocated +const sub = oversize.subarray(4096, 8192); +fastView[0] = 0xdeadbeef; +oversize[4096] = 0x7f; +let materialized = false; +for (let i = 0; i < iterations; ++i) { + agree(`fast#${i}`, hotIdentity(fastView), refIdentity(fastView)); + agree(`oversize#${i}`, hotIdentity(oversize), refIdentity(oversize)); + agree(`subarray#${i}`, hotIdentity(sub), refIdentity(sub)); + // The pointer must address live memory: read the value we stored, through the returned pointer. + if (readU32(hotIdentity(fastView)) !== 0xdeadbeef) + throw new Error(`fast view pointer does not address live data at ${i}`); + if (hotIdentity(sub) !== hotIdentity(oversize) + 4096) + throw new Error(`subarray offset lost at ${i}`); + if (!materialized && i === (iterations >> 1)) { + void fastView.buffer; // Fast -> Wasteful: storage may move + materialized = true; + } +} +// After materialization the read-through invariant must still hold at the (possibly new) pointer. +if (readU32(hotIdentity(fastView)) !== 0xdeadbeef || readU32(refIdentity(fastView)) !== 0xdeadbeef) + throw new Error("materialized wasteful view lost its data or its pointer"); + +// --------------------------------------------------------------------------------------------- +// 3. cstring parameter with a view: the inline path hands over the vector and C reads real, +// NUL-terminated bytes -- including through a byteOffset'd subarray. +// --------------------------------------------------------------------------------------------- +const strBuf = new Uint8Array(64); // "abc\0" at 0, then "engine-native\0" at 16 +strBuf.set([97, 98, 99, 0], 0); +strBuf.set([101, 110, 103, 105, 110, 101, 45, 110, 97, 116, 105, 118, 101, 0], 16); +const strView = strBuf.subarray(16); +for (let i = 0; i < iterations; ++i) { + agree(`strlen(buf)#${i}`, hotStrlen(strBuf), refStrlen(strBuf)); + agree(`strlen(sub)#${i}`, hotStrlen(strView), refStrlen(strView)); + if (hotStrlen(strBuf) !== 3n || hotStrlen(strView) !== 13n) + throw new Error(`cstring-from-view read wrong bytes at ${i}: ${hotStrlen(strBuf)}, ${hotStrlen(strView)}`); +} + +// --------------------------------------------------------------------------------------------- +// 4. The number paths of the same untyped conversion must be undisturbed by the new view checks +// (regression cover): int32 / double / negative / null / large all agree tier-to-tier. +// --------------------------------------------------------------------------------------------- +const numberArgs = [0, 1, 4096, -1, 2147483647, -2147483648, 4294967296, 1.5e9, null, undefined]; +for (let i = 0; i < iterations; ++i) + for (const n of numberArgs) + agree(`number(${String(n)})#${i}`, hotIdentity(n), refIdentity(n)); + +// --------------------------------------------------------------------------------------------- +// 5. GUARD: a DETACHED view must never leak its stale pointer through the inline path (null +// vector -> slow path); whatever the C++ conversion decides (0 / null / throw), the tiers agree. +// --------------------------------------------------------------------------------------------- +function tryHot(fn, v) { try { return fn(v); } catch (e) { return "threw:" + e.constructor.name; } } +const detached = new Uint8Array(new ArrayBuffer(64)); +const stalePtr = refIdentity(detached); +if (typeof detached.buffer.transfer === "function") detached.buffer.transfer(); +else if (typeof transferArrayBuffer === "function") transferArrayBuffer(detached.buffer); +else throw new Error("no way to detach an ArrayBuffer in this shell"); +for (let i = 0; i < iterations; ++i) { + const hot = tryHot(hotIdentity, detached), ref = tryHot(refIdentity, detached); + agree(`detached#${i}`, hot, ref); + if (hot === stalePtr && stalePtr !== 0) + throw new Error(`detached view leaked its stale pointer at ${i}: ${String(hot)}`); +} + +// --------------------------------------------------------------------------------------------- +// 6. GUARD: RESIZABLE and GROWABLE-SHARED backed views carry the isResizableOrGrowableShared +// mode bits and must take the C++ path in every tier; a plain (fixed) SharedArrayBuffer view +// does not carry them. In all cases the requirement is tier AGREEMENT, whatever the C++ +// conversion's policy for these buffers is. +// --------------------------------------------------------------------------------------------- +const guardedViews = []; +guardedViews.push(["resizable", new Uint8Array(new ArrayBuffer(64, { maxByteLength: 256 }))]); +if (typeof SharedArrayBuffer === "function") { + guardedViews.push(["shared-fixed", new Uint8Array(new SharedArrayBuffer(64))]); + let growable; + try { growable = new SharedArrayBuffer(64, { maxByteLength: 256 }); } catch (e) { growable = null; } + if (growable) + guardedViews.push(["shared-growable", new Uint8Array(growable)]); +} +for (let i = 0; i < iterations; ++i) + for (const [name, view] of guardedViews) + agree(`${name}#${i}`, tryHot(hotIdentity, view), tryHot(refIdentity, view)); + +// --------------------------------------------------------------------------------------------- +// 7. buffer-typed parameter rejects NON-view values consistently in every tier (numbers throw +// in C++; the inline path must not accept them either). +// --------------------------------------------------------------------------------------------- +for (let i = 0; i < iterations; ++i) { + const hot = tryHot(hotBufIdentity, 1234), ref = tryHot(refBufIdentity, 1234); + agree(`buffer(number)#${i}`, hot, ref); + if (!String(hot).startsWith("threw:")) + throw new Error(`buffer param accepted a number in the JIT at ${i}: ${String(hot)}`); +} + +// --------------------------------------------------------------------------------------------- +// 8. A view argument followed by a THROWING argument: the exception propagates cleanly (the +// partially-written slot buffer is never observed) and identically across tiers. +// --------------------------------------------------------------------------------------------- +const add = $vm.ffiFunction({ args: ["ptr", "i32"], returns: "ptr" }, fixture("ffi_ptr_identity"), "identity2"); +function hotAdd(v, x) { return add(v, x); } +function refAdd(v, x) { return add(v, x); } +noDFG(refAdd); noInline(refAdd); noInline(hotAdd); +const poison = { valueOf() { throw new RangeError("poison"); } }; +const sym = Symbol("s"); +function tryCall(fn, a, b) { try { return fn(a, b); } catch (e) { return "threw:" + e.constructor.name; } } +for (let i = 0; i < iterations; ++i) { + agree(`view+poison#${i}`, tryCall(hotAdd, strBuf, poison), tryCall(refAdd, strBuf, poison)); + agree(`view+symbol#${i}`, tryCall(hotAdd, strBuf, sym), tryCall(refAdd, strBuf, sym)); +} + +if (failures) + throw new Error(`${failures} tier mismatch(es) reported above`); +print("ffi view args: all checks passed"); diff --git a/Source/JavaScriptCore/CMakeLists.txt b/Source/JavaScriptCore/CMakeLists.txt index 4e044a61d9e52..0479eb5d056da 100644 --- a/Source/JavaScriptCore/CMakeLists.txt +++ b/Source/JavaScriptCore/CMakeLists.txt @@ -28,6 +28,8 @@ set(JavaScriptCore_PRIVATE_INCLUDE_DIRECTORIES "${JAVASCRIPTCORE_DIR}/disassembler/ARM64" "${JAVASCRIPTCORE_DIR}/disassembler/zydis" "${JAVASCRIPTCORE_DIR}/domjit" + "${JAVASCRIPTCORE_DIR}/ffi" + "${JAVASCRIPTCORE_DIR}/ffi/tests" "${JAVASCRIPTCORE_DIR}/ftl" "${JAVASCRIPTCORE_DIR}/fuzzilli" "${JAVASCRIPTCORE_DIR}/heap" @@ -2049,6 +2051,21 @@ set(JavaScriptCore_PRIVATE_FRAMEWORK_HEADERS heap/BunV8HeapSnapshotBuilder.h heap/HeapProfiler.h + ffi/BunFFI.h + ffi/FFICallHost.h + ffi/FFICallbackThunk.h + ffi/FFICallingConvention.h + ffi/FFIContext.h + ffi/FFIConversions.h + ffi/FFIDFG.h + ffi/FFIICStub.h + ffi/FFIInvokeThunk.h + ffi/FFISignature.h + ffi/FFIType.h + ffi/JSFFICallback.h + ffi/JSFFIFunction.h + ffi/tests/FFITestFixtures.h + bytecode/GlobalCodeBlock.h bytecode/ModuleProgramCodeBlock.h bytecode/ProgramCodeBlock.h diff --git a/Source/JavaScriptCore/Sources.txt b/Source/JavaScriptCore/Sources.txt index 50505d40a3586..06bea7fd82993 100644 --- a/Source/JavaScriptCore/Sources.txt +++ b/Source/JavaScriptCore/Sources.txt @@ -1317,4 +1317,19 @@ yarr/YarrCanonicalizeUnicode.cpp runtime/InternalFieldTuple.cpp -heap/BunV8HeapSnapshotBuilder.cpp \ No newline at end of file +heap/BunV8HeapSnapshotBuilder.cpp + +ffi/BunFFI.cpp +ffi/FFICallHost.cpp +ffi/FFICallbackThunk.cpp +ffi/FFICallingConvention.cpp +ffi/FFIContext.cpp +ffi/FFIConversions.cpp +ffi/FFIDFG.cpp +ffi/FFIDFGCodegen.cpp +ffi/FFIICStub.cpp +ffi/FFIInvokeThunk.cpp +ffi/FFISignature.cpp +ffi/JSFFICallback.cpp +ffi/JSFFIFunction.cpp +ffi/tests/FFITestFixtures.cpp diff --git a/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h b/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h index fdceccaacdf04..1a8df3862e2e0 100644 --- a/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h +++ b/Source/JavaScriptCore/dfg/DFGAbstractInterpreterInlines.h @@ -37,6 +37,9 @@ #include "DOMJITCallDOMGetterSnippet.h" #include "DOMJITGetterSetter.h" #include "DOMJITSignature.h" +#if USE(BUN_JSC_ADDITIONS) +#include "FFIDFG.h" +#endif #include "FunctionPrototype.h" #include "GetByStatus.h" #include "GetterSetter.h" @@ -5747,6 +5750,16 @@ bool AbstractInterpreter::executeEffects(unsigned clobberLimi break; } + case CallFFI: { +#if USE(BUN_JSC_ADDITIONS) + clobberWorld(); + setTypeForNode(node, FFI::speculatedResultTypeForCallFFI(node)); +#else + DFG_CRASH(m_graph, node, "Unexpected node type"); +#endif + break; + } + case ForceOSRExit: case CheckBadValue: m_state.setIsValid(false); diff --git a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp index f7c96e814cdfb..25f0894b55b62 100644 --- a/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp +++ b/Source/JavaScriptCore/dfg/DFGByteCodeParser.cpp @@ -69,6 +69,10 @@ #include "JSBoundFunctionInlines.h" #include "JSCInlines.h" #include "JSCellButterfly.h" +#if USE(BUN_JSC_ADDITIONS) +#include "FFISignature.h" +#include "JSFFIFunction.h" +#endif #include "JSIteratorHelper.h" #include "JSMapIterator.h" #include "JSModuleEnvironment.h" @@ -2347,6 +2351,21 @@ ByteCodeParser::CallOptimizationResult ByteCodeParser::handleInlining( if (executable->intrinsic() == WasmFunctionIntrinsic && !Options::forceICFailure()) return inliningResult; +#if USE(BUN_JSC_ADDITIONS) + if (Options::useFFICallInDFG() && (callOp == Call || callOp == TailCall) && callee.function() && callee.function()->inherits() + && !uncheckedDowncast(callee.function())->isHostPathOnly() // hooked => host path only + && uncheckedDowncast(callee.function())->signature().invokeThunk()) { + auto* ffiFunction = uncheckedDowncast(callee.function()); + m_graph.m_plan.recordedStatuses().addCallLinkStatus(currentNodeOrigin().semantic, CallLinkStatus(callee)); + auto* frozenFunction = m_graph.freeze(ffiFunction); + addToGraph(CheckIsConstant, OpInfo(frozenFunction), Edge(callTargetNode, CellUse)); + m_parameterSlots = std::max(m_parameterSlots, Graph::parameterSlotsForArgCount( + std::max(ffiFunction->signature().slotCount() + 1, argumentCountIncludingThis))); + addCall(result, Call, OpInfo(), jsConstant(frozenFunction), argumentCountIncludingThis, registerOffset, prediction); + return CallOptimizationResult::Inlined; + } +#endif + if (executable->intrinsic() == BoundFunctionCallIntrinsic) return inliningResult; diff --git a/Source/JavaScriptCore/dfg/DFGClobberize.h b/Source/JavaScriptCore/dfg/DFGClobberize.h index 9f5d120b6edc9..0e7d689bce1f2 100644 --- a/Source/JavaScriptCore/dfg/DFGClobberize.h +++ b/Source/JavaScriptCore/dfg/DFGClobberize.h @@ -834,6 +834,7 @@ void clobberize(Graph& graph, Node* node, const ReadFunctor& read, const WriteFu case CallDirectEval: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case CallCustomAccessorGetter: case CallCustomAccessorSetter: case ToPrimitive: diff --git a/Source/JavaScriptCore/dfg/DFGDoesGC.cpp b/Source/JavaScriptCore/dfg/DFGDoesGC.cpp index 02e2e0a8862c1..59908b02ae6db 100644 --- a/Source/JavaScriptCore/dfg/DFGDoesGC.cpp +++ b/Source/JavaScriptCore/dfg/DFGDoesGC.cpp @@ -322,6 +322,7 @@ bool doesGC(Graph& graph, Node* node) case DirectTailCallInlinedCaller: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case CallCustomAccessorGetter: case CallCustomAccessorSetter: case ForceOSRExit: diff --git a/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp b/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp index 0563563cd6664..9a09c196b6347 100644 --- a/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp +++ b/Source/JavaScriptCore/dfg/DFGFixupPhase.cpp @@ -3798,6 +3798,7 @@ class FixupPhase : public Phase { case TailCallForwardVarargsInlinedCaller: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case ProfileControlFlow: case NewObject: case NewInternalFieldObject: diff --git a/Source/JavaScriptCore/dfg/DFGMayExit.cpp b/Source/JavaScriptCore/dfg/DFGMayExit.cpp index 3e81adb2dac45..48a7e2f02977c 100644 --- a/Source/JavaScriptCore/dfg/DFGMayExit.cpp +++ b/Source/JavaScriptCore/dfg/DFGMayExit.cpp @@ -222,6 +222,7 @@ ExitMode mayExitImpl(Graph& graph, Node* node, StateType& state) case RegExpMatchFastGlobal: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case CallCustomAccessorGetter: case CallCustomAccessorSetter: case AllocatePropertyStorage: diff --git a/Source/JavaScriptCore/dfg/DFGNode.cpp b/Source/JavaScriptCore/dfg/DFGNode.cpp index 777e1b10d981e..7bab35c9ea9fc 100644 --- a/Source/JavaScriptCore/dfg/DFGNode.cpp +++ b/Source/JavaScriptCore/dfg/DFGNode.cpp @@ -34,6 +34,11 @@ #include "DOMJITSignature.h" #include "JSCellButterfly.h" +#if USE(BUN_JSC_ADDITIONS) +#include "FFISignature.h" +#include "JSFFIFunction.h" +#endif + WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN namespace JSC { namespace DFG { @@ -327,6 +332,26 @@ void Node::convertToCallWasm(FrozenValue* callee) m_opInfo = callee; } +#if USE(BUN_JSC_ADDITIONS) +void Node::convertToCallFFI(FrozenValue* callee) +{ + ASSERT(m_op == Call); + m_op = CallFFI; + m_opInfo = callee; +} + +JSFFIFunction* Node::ffiFunction() +{ + ASSERT(op() == CallFFI); + return castOperand(); +} + +FFI::Signature& Node::ffiSignature() +{ + return ffiFunction()->signature(); +} +#endif + void Node::convertToCallDOM(Graph& graph) { ASSERT(op() == Call); diff --git a/Source/JavaScriptCore/dfg/DFGNode.h b/Source/JavaScriptCore/dfg/DFGNode.h index 699aaf4605818..5f7d48988d32e 100644 --- a/Source/JavaScriptCore/dfg/DFGNode.h +++ b/Source/JavaScriptCore/dfg/DFGNode.h @@ -83,6 +83,13 @@ class ExecutionCounter; class Snippet; +#if USE(BUN_JSC_ADDITIONS) +class JSFFIFunction; +namespace FFI { +class Signature; +} // namespace FFI +#endif + namespace DFG { class BasicBlock; @@ -964,6 +971,13 @@ struct Node { void NODELETE convertToCallWasm(FrozenValue*); +#if USE(BUN_JSC_ADDITIONS) + void NODELETE convertToCallFFI(FrozenValue*); + + JSFFIFunction* ffiFunction(); + FFI::Signature& ffiSignature(); +#endif + void NODELETE convertToCallDOM(Graph&); void NODELETE convertToRegExpExecNonGlobalOrStickyWithoutChecks(FrozenValue* regExp); @@ -2136,6 +2150,7 @@ struct Node { case TailCallForwardVarargsInlinedCaller: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case CallCustomAccessorGetter: case GetByOffset: case MultiGetByOffset: @@ -2254,6 +2269,7 @@ struct Node { case DirectTailCallInlinedCaller: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case RegExpExecNonGlobalOrSticky: case RegExpExecSticky: case RegExpMatchFastGlobal: diff --git a/Source/JavaScriptCore/dfg/DFGNodeType.h b/Source/JavaScriptCore/dfg/DFGNodeType.h index 41df5d9ebdf80..3e4d205035999 100644 --- a/Source/JavaScriptCore/dfg/DFGNodeType.h +++ b/Source/JavaScriptCore/dfg/DFGNodeType.h @@ -415,6 +415,8 @@ namespace JSC { namespace DFG { macro(CallDirectEval, NodeResultJS | NodeMustGenerate | NodeHasVarArgs) \ macro(CallWasm, NodeResultJS | NodeMustGenerate | NodeHasVarArgs) \ macro(TailCallInlinedCallerWasm, NodeResultJS | NodeMustGenerate | NodeHasVarArgs) \ + /* bun:ffi (USE(BUN_JSC_ADDITIONS)); the enumerators are unconditional, only their uses are guarded. */\ + macro(CallFFI, NodeResultJS | NodeMustGenerate | NodeHasVarArgs) \ \ macro(CallCustomAccessorGetter, NodeResultJS | NodeMustGenerate) \ macro(CallCustomAccessorSetter, NodeMustGenerate) \ diff --git a/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp b/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp index cf42a81910923..3fc3052da559a 100644 --- a/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp +++ b/Source/JavaScriptCore/dfg/DFGPredictionPropagationPhase.cpp @@ -1080,6 +1080,7 @@ class PredictionPropagationPhase : public Phase { case TailCallForwardVarargsInlinedCaller: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case CallCustomAccessorGetter: case GetGlobalVar: case GetGlobalLexicalVariable: diff --git a/Source/JavaScriptCore/dfg/DFGSafeToExecute.h b/Source/JavaScriptCore/dfg/DFGSafeToExecute.h index 0ab636444a3b7..e7f128b177651 100644 --- a/Source/JavaScriptCore/dfg/DFGSafeToExecute.h +++ b/Source/JavaScriptCore/dfg/DFGSafeToExecute.h @@ -667,6 +667,7 @@ bool safeToExecute(AbstractStateType& state, Graph& graph, Node* node, bool igno case ConstructVarargs: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case CallCustomAccessorGetter: case CallCustomAccessorSetter: case VarargsLength: diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h index 4d3b1750527ac..a6ebb28efcffe 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT.h @@ -1516,6 +1516,9 @@ class SpeculativeJIT : public JITCompiler { void compileGetButterfly(Node*); void compileCallDOMGetter(Node*); void compileCallDOM(Node*); +#if USE(BUN_JSC_ADDITIONS) + void compileCallFFI(Node*); // Defined out-of-line in ffi/FFIDFGCodegen.cpp (JSVALUE64 only). +#endif void compileCheckJSCast(Node*); void compileCallCustomAccessorGetter(Node*); void compileCallCustomAccessorSetter(Node*); diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp index 28c1a609fab88..98c5dd0b891ec 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp @@ -4623,6 +4623,7 @@ void SpeculativeJIT::compile(Node* node) case DateSetTime: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case FunctionBind: case NewBoundFunction: case EnumeratorPutByVal: diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp index 2b85ae7e464a0..afb9d4c0f2519 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp @@ -6318,6 +6318,14 @@ void SpeculativeJIT::compile(Node* node) compileCallDOM(node); break; + case CallFFI: +#if USE(BUN_JSC_ADDITIONS) + compileCallFFI(node); +#else + DFG_CRASH(m_graph, node, "Unexpected node"); +#endif + break; + case CallDOMGetter: compileCallDOMGetter(node); break; diff --git a/Source/JavaScriptCore/dfg/DFGStrengthReductionPhase.cpp b/Source/JavaScriptCore/dfg/DFGStrengthReductionPhase.cpp index e65c31e703e83..3771b12b5e105 100644 --- a/Source/JavaScriptCore/dfg/DFGStrengthReductionPhase.cpp +++ b/Source/JavaScriptCore/dfg/DFGStrengthReductionPhase.cpp @@ -35,6 +35,9 @@ #include "DFGInsertionSet.h" #include "DFGJITCode.h" #include "DFGPhase.h" +#if USE(BUN_JSC_ADDITIONS) +#include "FFIDFG.h" +#endif #include "JSBoundFunctionInlines.h" #include "JSObjectInlines.h" #include "JSWebAssemblyInstance.h" @@ -2002,6 +2005,13 @@ class StrengthReductionPhase : public Phase { } #endif +#if USE(BUN_JSC_ADDITIONS) + if (FFI::tryConvertCallToCallFFI(m_graph, m_insertionSet, m_nodeIndex, m_node, function)) { + m_changed = true; + break; + } +#endif + // We gave up inlining a wrapped function, but still, we can inline bound function's wrapper by extracting it. // This also wipes bound-function thunk call which is suboptimal compared to directly calling a wrapped function here. if (executable->intrinsic() == BoundFunctionCallIntrinsic && function && (m_node->op() == Call || m_node->op() == TailCall || m_node->op() == TailCallInlinedCaller)) { diff --git a/Source/JavaScriptCore/ffi/BunFFI.cpp b/Source/JavaScriptCore/ffi/BunFFI.cpp new file mode 100644 index 0000000000000..5fabf8402fe95 --- /dev/null +++ b/Source/JavaScriptCore/ffi/BunFFI.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "BunFFI.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "Error.h" +#include "FFIContext.h" +#include "IdentifierInlines.h" +#include "JSArrayInlines.h" +#include "JSCInlines.h" +#include "JSFFICallback.h" +#include "JSFFIFunction.h" +#include "JSGlobalObject.h" +#include "JSObjectInlines.h" +#include +#include + +namespace JSC { namespace FFI { + +std::optional typeFromJS(JSGlobalObject* globalObject, JSValue value) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (value.isNumber()) { + double tag = value.asNumber(); + if (tag == std::trunc(tag) && tag >= 0 && tag < numberOfTypes) + return static_cast(static_cast(tag)); + throwTypeError(globalObject, scope, makeString("Unknown FFI type tag "_s, tag)); + return std::nullopt; + } + + if (value.isString()) { + String string = value.toWTFString(globalObject); + RETURN_IF_EXCEPTION(scope, std::nullopt); + if (std::optional type = parseType(string)) + return type; + throwTypeError(globalObject, scope, makeString("Unknown FFI type '"_s, string, '\'')); + return std::nullopt; + } + + throwTypeError(globalObject, scope, "An FFI type must be a string or a number"_s); + return std::nullopt; +} + +RefPtr signatureFromJS(JSGlobalObject* globalObject, JSValue descriptor) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + JSObject* object = descriptor.getObject(); + if (!object) { + throwTypeError(globalObject, scope, "Expected an FFI signature descriptor object of the form { args, returns }"_s); + return nullptr; + } + + Vector argumentTypes; + JSValue argsValue = object->get(globalObject, Identifier::fromString(vm, "args"_s)); + RETURN_IF_EXCEPTION(scope, nullptr); + if (!argsValue.isUndefinedOrNull()) { + JSObject* argsObject = argsValue.getObject(); + if (!argsObject) { + throwTypeError(globalObject, scope, "FFI signature 'args' must be an array of types"_s); + return nullptr; + } + uint64_t length = toLength(globalObject, argsObject); + RETURN_IF_EXCEPTION(scope, nullptr); + if (length > Signature::maxArguments) { + throwTypeError(globalObject, scope, makeString("FFI signatures support at most "_s, Signature::maxArguments, " arguments"_s)); + return nullptr; + } + argumentTypes.reserveInitialCapacity(static_cast(length)); + for (unsigned i = 0; i < length; ++i) { + JSValue element = argsObject->getIndex(globalObject, i); + RETURN_IF_EXCEPTION(scope, nullptr); + std::optional type = typeFromJS(globalObject, element); + RETURN_IF_EXCEPTION(scope, nullptr); + ASSERT(type); + if (!isValidArgumentType(*type)) { + throwTypeError(globalObject, scope, makeString("FFI argument type '"_s, name(*type), "' is not allowed"_s)); + return nullptr; + } + argumentTypes.append(*type); + } + } + + Type returnType = Type::Void; + JSValue returnsValue = object->get(globalObject, Identifier::fromString(vm, "returns"_s)); + RETURN_IF_EXCEPTION(scope, nullptr); + if (!returnsValue.isUndefinedOrNull()) { + std::optional type = typeFromJS(globalObject, returnsValue); + RETURN_IF_EXCEPTION(scope, nullptr); + ASSERT(type); + if (!isValidReturnType(*type)) { + throwTypeError(globalObject, scope, makeString("FFI return type '"_s, name(*type), "' is not allowed"_s)); + return nullptr; + } + returnType = *type; + } + + RefPtr signature = Signature::tryCreate(argumentTypes.span(), returnType); + if (!signature) { + throwTypeError(globalObject, scope, "Invalid FFI signature"_s); + return nullptr; + } + return signature; +} + +JSFFICallback* createCallback(JSGlobalObject* globalObject, Ref&& signature, JSObject* callable, bool threadsafe, void* embedderContext) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (!callable || !JSValue(callable).isCallable()) [[unlikely]] { + throwTypeError(globalObject, scope, "bun:ffi callback requires a callable JS function"_s); + return nullptr; + } + + if (!Options::useJIT()) [[unlikely]] { + throwTypeError(globalObject, scope, "bun:ffi requires the JIT"_s); + return nullptr; + } + + RELEASE_AND_RETURN(scope, JSFFICallback::create(vm, globalObject, globalObject->ffiCallbackStructure(), callable, WTF::move(signature), threadsafe, embedderContext)); +} + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/BunFFI.h b/Source/JavaScriptCore/ffi/BunFFI.h new file mode 100644 index 0000000000000..0853e4b197955 --- /dev/null +++ b/Source/JavaScriptCore/ffi/BunFFI.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#include "FFISignature.h" +#include "FFIType.h" +#include "JSCJSValue.h" +#include +#include +#include +#include + +namespace JSC { + +class JSFFICallback; +class JSGlobalObject; +class JSObject; + +namespace FFI { + +JS_EXPORT_PRIVATE std::optional typeFromJS(JSGlobalObject*, JSValue); + +JS_EXPORT_PRIVATE RefPtr signatureFromJS(JSGlobalObject*, JSValue descriptor); + +class ThreadsafeInvocation; +JS_EXPORT_PRIVATE JSFFICallback* createCallback(JSGlobalObject*, Ref&&, JSObject* callable, bool threadsafe, void* embedderContext); +JS_EXPORT_PRIVATE void runThreadsafeInvocation(ThreadsafeInvocation&); + +} // namespace FFI + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFICallHost.cpp b/Source/JavaScriptCore/ffi/FFICallHost.cpp new file mode 100644 index 0000000000000..399bd361afcbc --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFICallHost.cpp @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFICallHost.h" + +#if USE(BUN_JSC_ADDITIONS) + +#if USE(JSVALUE64) + +#include "CallFrame.h" +#include "ExceptionHelpers.h" +#include "FFIContext.h" +#include "FFIConversions.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "FrameTracers.h" +#include "JSCInlines.h" +#include "JSFFIFunction.h" +#include "JSGlobalObject.h" +#include "TopExceptionScope.h" + +namespace JSC { + +namespace FFI { + +static ALWAYS_INLINE EncodedJSValue ffiCall(JSGlobalObject* globalObject, CallFrame* callFrame) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + auto* function = uncheckedDowncast(callFrame->jsCallee()); + Signature& signature = function->signature(); + FFIContext& context = globalObject->ffiContext(); + StringArena::Scope arenaScope(context); + + unsigned argumentCount = signature.argumentCount(); + ASSERT(argumentCount <= Signature::maxArguments); + uint64_t slots[Signature::maxArguments + 1]; + + for (unsigned i = 0; i < argumentCount; ++i) { + Type type = signature.argumentType(i); + writeSlotFromJSValue(globalObject, context, type, callFrame->argument(i), slots[i], &context.stringArena()); + RETURN_IF_EXCEPTION(scope, { }); + } + slots[argumentCount] = 0; + + CodePtr thunk = signature.invokeThunk(); + if (!thunk) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope, "bun:ffi failed to allocate executable memory for the invoke thunk"_s); + return { }; + } + + if (const CallHooks* hooks = function->hooks()) [[unlikely]] { + Exception* pending = nullptr; + { + auto hookScope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + void* hookToken = hooks->before ? hooks->before(globalObject, callFrame) : nullptr; + if (!hookScope.exception()) [[likely]] + thunk.taggedPtr()(function->target(), slots); + pending = hookScope.exception(); // from a throwing before-hook or an in-call callback + if (pending) [[unlikely]] + hookScope.clearException(); + if (hooks->after) + hooks->after(globalObject, callFrame, hookToken); // runs on a clean VM + if (hookScope.exception()) [[unlikely]] { + if (pending) + hookScope.clearException(); + else { + pending = hookScope.exception(); + hookScope.clearException(); + } + } + } + if (pending) [[unlikely]] { + throwException(globalObject, scope, pending); + return { }; + } + } else + thunk.taggedPtr()(function->target(), slots); + RETURN_IF_EXCEPTION(scope, { }); + + RELEASE_AND_RETURN(scope, JSValue::encode(jsValueFromSlot(globalObject, context, signature.returnType(), slots[argumentCount]))); +} + +JSC_DEFINE_HOST_FUNCTION(ffiHostCall, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + return ffiCall(globalObject, callFrame); +} + +} // namespace FFI + +JSC_DEFINE_JIT_OPERATION(operationFFICallSlowPath, EncodedJSValue, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + VM& vm = globalObject->vm(); + JITOperationPrologueCallFrameTracer tracer(vm, callFrame); + auto scope = DECLARE_THROW_SCOPE(vm); + OPERATION_RETURN(scope, FFI::ffiCall(globalObject, callFrame)); +} + +} // namespace JSC + +#endif // USE(JSVALUE64) + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFICallHost.h b/Source/JavaScriptCore/ffi/FFICallHost.h new file mode 100644 index 0000000000000..5aae3e9612083 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFICallHost.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#if USE(JSVALUE64) + +#include "JSCJSValue.h" +#include "OperationResult.h" + +namespace JSC { + +class CallFrame; +class JSGlobalObject; + +namespace FFI { + +JSC_DECLARE_HOST_FUNCTION(ffiHostCall); + +} // namespace FFI + +JSC_DECLARE_JIT_OPERATION(operationFFICallSlowPath, EncodedJSValue, (JSGlobalObject*, CallFrame*)); + +} // namespace JSC + +#endif // USE(JSVALUE64) + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFICallbackThunk.cpp b/Source/JavaScriptCore/ffi/FFICallbackThunk.cpp new file mode 100644 index 0000000000000..4b897557779ad --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFICallbackThunk.cpp @@ -0,0 +1,500 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFICallbackThunk.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "ArgList.h" +#include "CCallHelpers.h" +#include "CallData.h" +#include "Error.h" +#include "ExceptionHelpers.h" +#include "FFICallingConvention.h" +#include "FFIContext.h" +#include "FFIConversions.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "JSCJSValueInlines.h" +#include "JSFFICallback.h" +#include "JSGlobalObject.h" +#include "JSLock.h" +#include "JSObjectInlines.h" +#include "LinkBuffer.h" +#include "MarkedVector.h" +#include "Options.h" +#include + +namespace JSC { + +#if FFI_CALLBACK_THUNK_SUPPORTED + +namespace FFI { + +namespace { + +#if CPU(X86_64) +constexpr GPRReg thunkScratchGPR = X86Registers::eax; +#else +constexpr GPRReg thunkScratchGPR = ARM64Registers::x9; +#endif + +#if OS(WINDOWS) && CPU(X86_64) +constexpr unsigned win64XMMSaveCount = 10; // xmm6 .. xmm15 +constexpr unsigned win64SaveAreaBytes = win64XMMSaveCount * 16 + 2 * 8; // 176, a multiple of 16 +constexpr FPRReg win64SaveTempFPR = X86Registers::xmm4; +#else +constexpr unsigned win64SaveAreaBytes = 0; +#endif +static_assert(!(win64SaveAreaBytes % 16), "the Win64 save area keeps rsp 16-byte aligned"); + +#if OS(WINDOWS) && CPU(X86_64) +static void emitSaveWin64Nonvolatiles(CCallHelpers& jit, int saveAreaOffsetFromFP) +{ + for (unsigned i = 0; i < win64XMMSaveCount; ++i) { + FPRReg reg = static_cast(X86Registers::xmm6 + i); + CCallHelpers::Address low(GPRInfo::callFrameRegister, saveAreaOffsetFromFP + static_cast(i * 16)); + jit.storeDouble(reg, low); + jit.vectorExtractLaneFloat64(CCallHelpers::TrustedImm32(1), reg, win64SaveTempFPR); + jit.storeDouble(win64SaveTempFPR, low.withOffset(8)); + } + jit.storePtr(X86Registers::esi, CCallHelpers::Address(GPRInfo::callFrameRegister, saveAreaOffsetFromFP + static_cast(win64XMMSaveCount * 16))); + jit.storePtr(X86Registers::edi, CCallHelpers::Address(GPRInfo::callFrameRegister, saveAreaOffsetFromFP + static_cast(win64XMMSaveCount * 16 + 8))); +} + +static void emitRestoreWin64Nonvolatiles(CCallHelpers& jit, int saveAreaOffsetFromFP) +{ + for (unsigned i = 0; i < win64XMMSaveCount; ++i) { + FPRReg reg = static_cast(X86Registers::xmm6 + i); + CCallHelpers::Address low(GPRInfo::callFrameRegister, saveAreaOffsetFromFP + static_cast(i * 16)); + jit.loadDouble(low, reg); + jit.loadDouble(low.withOffset(8), win64SaveTempFPR); + jit.vectorReplaceLaneFloat64(CCallHelpers::TrustedImm32(1), win64SaveTempFPR, reg); + } + jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, saveAreaOffsetFromFP + static_cast(win64XMMSaveCount * 16)), X86Registers::esi); + jit.loadPtr(CCallHelpers::Address(GPRInfo::callFrameRegister, saveAreaOffsetFromFP + static_cast(win64XMMSaveCount * 16 + 8)), X86Registers::edi); +} +#endif // OS(WINDOWS) && CPU(X86_64) + +static void emitStoreIncomingArgumentGPR(CCallHelpers& jit, Type type, GPRReg source, CCallHelpers::Address slot) +{ + GPRReg scratchGPR = thunkScratchGPR; + switch (type) { + case Type::Char: + case Type::Int8: + jit.signExtend8To64(source, scratchGPR); + break; + case Type::Uint8: + jit.zeroExtend8To64(source, scratchGPR); + break; + case Type::Int16: + jit.signExtend16To64(source, scratchGPR); + break; + case Type::Uint16: + jit.zeroExtend16To64(source, scratchGPR); + break; + case Type::Int32: + jit.signExtend32To64(source, scratchGPR); + break; + case Type::Uint32: + jit.zeroExtend32ToWord(source, scratchGPR); + break; + case Type::Bool: + jit.and32(CCallHelpers::TrustedImm32(0xff), source, scratchGPR); + jit.compare32(CCallHelpers::NotEqual, scratchGPR, CCallHelpers::TrustedImm32(0), scratchGPR); + break; + case Type::Int64: + case Type::Uint64: + case Type::Int64Fast: + case Type::Uint64Fast: + case Type::Pointer: + case Type::CString: + case Type::Function: + case Type::Buffer: + case Type::BufferLength: // a callback parameter is a plain unsigned 64-bit length (like Uint64) + case Type::JSValue: + jit.store64(source, slot); + return; + case Type::Float: + case Type::Double: + case Type::Void: + case Type::RESERVED_WasNapiEnv: + RELEASE_ASSERT_NOT_REACHED(); // Never GPR-classed / never a valid argument. + } + jit.store64(scratchGPR, slot); +} + +static void emitStoreIncomingArgumentStack(CCallHelpers& jit, Type type, CCallHelpers::Address source, CCallHelpers::Address slot) +{ + GPRReg scratchGPR = thunkScratchGPR; + switch (type) { + case Type::Char: + case Type::Int8: + jit.load8SignedExtendTo32(source, scratchGPR); + jit.signExtend32To64(scratchGPR, scratchGPR); + break; + case Type::Uint8: + jit.load8(source, scratchGPR); + break; + case Type::Int16: + jit.load16SignedExtendTo32(source, scratchGPR); + jit.signExtend32To64(scratchGPR, scratchGPR); + break; + case Type::Uint16: + jit.load16(source, scratchGPR); + break; + case Type::Int32: + jit.load32(source, scratchGPR); + jit.signExtend32To64(scratchGPR, scratchGPR); + break; + case Type::Uint32: + jit.load32(source, scratchGPR); // A 32-bit load zero-extends the full register. + break; + case Type::Bool: + jit.load8(source, scratchGPR); + jit.compare32(CCallHelpers::NotEqual, scratchGPR, CCallHelpers::TrustedImm32(0), scratchGPR); + break; + case Type::Float: + jit.load32(source, scratchGPR); + break; + case Type::Int64: + case Type::Uint64: + case Type::Int64Fast: + case Type::Uint64Fast: + case Type::Double: + case Type::Pointer: + case Type::CString: + case Type::Function: + case Type::Buffer: + case Type::BufferLength: // a callback parameter is a plain unsigned 64-bit length (like Uint64) + case Type::JSValue: + jit.load64(source, scratchGPR); + break; + case Type::Void: + case Type::RESERVED_WasNapiEnv: + RELEASE_ASSERT_NOT_REACHED(); // Never an argument. + } + jit.store64(scratchGPR, slot); +} + +static void emitStoreIncomingArgumentFPR(CCallHelpers& jit, Type type, FPRReg source, CCallHelpers::Address slot) +{ + switch (type) { + case Type::Float: + jit.storeFloat(source, slot); + jit.store32(CCallHelpers::TrustedImm32(0), slot.withOffset(4)); + break; + case Type::Double: + jit.storeDouble(source, slot); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); // Only Float / Double are FPR-classed. + } +} + +} // anonymous namespace + +MacroAssemblerCodeRef generateCallbackThunk(VM&, JSFFICallback& callback) +{ + Signature& signature = callback.signature(); + const NativeCC cc = hostNativeCC(); + const CallLayout layout = computeCallLayout(cc, signature, Direction::Incoming); + const auto integerArgumentGPRs = integerArgumentRegisters(cc); + const auto floatArgumentFPRs = floatArgumentRegisters(cc); + +#if ASSERT_ENABLED + for (GPRReg argumentGPR : integerArgumentGPRs) + ASSERT(argumentGPR != thunkScratchGPR); +#endif + + const unsigned argumentCount = signature.argumentCount(); + const unsigned slotBufferBytes = static_cast(signature.slotBufferBytes()); + ASSERT(slotBufferBytes == signature.slotCount() * slotSize); + + const unsigned frameBytes = win64SaveAreaBytes + WTF::roundUpToMultipleOf<16>(slotBufferBytes); + const int slotsOffsetFromFP = -static_cast(frameBytes); +#if OS(WINDOWS) && CPU(X86_64) + const int saveAreaOffsetFromFP = -static_cast(win64SaveAreaBytes); +#endif + + CCallHelpers jit; + + jit.emitFunctionPrologue(); + + jit.subPtr(CCallHelpers::TrustedImm32(frameBytes), CCallHelpers::stackPointerRegister); + +#if OS(WINDOWS) && CPU(X86_64) + emitSaveWin64Nonvolatiles(jit, saveAreaOffsetFromFP); +#endif + + for (unsigned i = 0; i < argumentCount; ++i) { + const ArgLocation& location = layout.arguments[i]; + const Type type = signature.argumentType(i); + ASSERT(location.type == type); + const CCallHelpers::Address slot(GPRInfo::callFrameRegister, slotsOffsetFromFP + static_cast(i * slotSize)); + + switch (location.kind) { + case ArgLocation::Kind::GPR: + RELEASE_ASSERT(location.regIndex < integerArgumentGPRs.size()); + emitStoreIncomingArgumentGPR(jit, type, integerArgumentGPRs[location.regIndex], slot); + break; + case ArgLocation::Kind::FPR: + RELEASE_ASSERT(location.regIndex < floatArgumentFPRs.size()); + emitStoreIncomingArgumentFPR(jit, type, floatArgumentFPRs[location.regIndex], slot); + break; + case ArgLocation::Kind::Stack: { + const CCallHelpers::Address source(GPRInfo::callFrameRegister, static_cast(incomingStackOffset(layout, i))); + emitStoreIncomingArgumentStack(jit, type, source, slot); + break; + } + } + } + + jit.addPtr(CCallHelpers::TrustedImm32(slotsOffsetFromFP), GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImmPtr(&callback), GPRInfo::argumentGPR0); + auto dispatchOperation = callback.isThreadsafe() ? tagCFunction(ffiCallbackDispatchThreadsafe) : tagCFunction(ffiCallbackDispatch); + jit.move(CCallHelpers::TrustedImmPtr(dispatchOperation), thunkScratchGPR); + jit.call(thunkScratchGPR, OperationPtrTag); + + const CCallHelpers::Address returnSlot(GPRInfo::callFrameRegister, slotsOffsetFromFP + static_cast(argumentCount * slotSize)); + switch (layout.returnClass) { + case ArgClass::Void: + break; + case ArgClass::Int: + jit.load64(returnSlot, GPRInfo::returnValueGPR); + break; + case ArgClass::Float: + jit.loadFloat(returnSlot, FPRInfo::returnValueFPR); + break; + case ArgClass::Double: + jit.loadDouble(returnSlot, FPRInfo::returnValueFPR); + break; + } + +#if OS(WINDOWS) && CPU(X86_64) + emitRestoreWin64Nonvolatiles(jit, saveAreaOffsetFromFP); +#endif + jit.emitFunctionEpilogue(); + jit.ret(); + + LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID, LinkBuffer::Profile::Thunk, JITCompilationCanFail); + if (patchBuffer.didFailToAllocate()) [[unlikely]] + return { }; + + patchBuffer.setIsThunk(); + return FINALIZE_CODE_IF(Options::dumpDisassembly() || Options::dumpFFIDisassembly(), patchBuffer, JITThunkPtrTag, "FFICallbackThunk"_s, "FFI callback %s", signature.toString().ascii().data()); +} + +} // namespace FFI + +namespace FFI { + +class CallbackEntryScope { + WTF_MAKE_NONCOPYABLE(CallbackEntryScope); +public: + explicit CallbackEntryScope(VM& vm) + : m_vm(vm) + { +#if ENABLE(EXCEPTION_SCOPE_VERIFICATION) + m_savedNeedExceptionCheck = m_vm.m_needExceptionCheck; + if (m_savedNeedExceptionCheck) { + m_savedThrowPointRecursionDepth = m_vm.m_simulatedThrowPointRecursionDepth; + m_savedThrowPointLocation = m_vm.m_simulatedThrowPointLocation; + m_savedNativeStackTraceOfLastSimulatedThrow = WTF::move(m_vm.m_nativeStackTraceOfLastSimulatedThrow); + m_vm.m_needExceptionCheck = false; + } +#endif + } + + ~CallbackEntryScope() + { +#if ENABLE(EXCEPTION_SCOPE_VERIFICATION) + if (m_vm.m_needExceptionCheck || !m_savedNeedExceptionCheck) + return; + m_vm.m_needExceptionCheck = true; + m_vm.m_simulatedThrowPointRecursionDepth = m_savedThrowPointRecursionDepth; + m_vm.m_simulatedThrowPointLocation = m_savedThrowPointLocation; + m_vm.m_nativeStackTraceOfLastSimulatedThrow = WTF::move(m_savedNativeStackTraceOfLastSimulatedThrow); +#endif + } + +private: + VM& m_vm; +#if ENABLE(EXCEPTION_SCOPE_VERIFICATION) + bool m_savedNeedExceptionCheck { false }; + unsigned m_savedThrowPointRecursionDepth { 0 }; + ExceptionEventLocation m_savedThrowPointLocation; + std::unique_ptr m_savedNativeStackTraceOfLastSimulatedThrow; +#endif +}; + +} // namespace FFI + +JSC_DEFINE_JIT_OPERATION(ffiCallbackDispatchThreadsafe, EncodedJSValue, (JSFFICallback* callback, uint64_t* slots)) +{ + ASSERT(callback->isThreadsafe()); + FFI::Signature& signature = callback->signature(); + const unsigned argumentCount = signature.argumentCount(); + auto dispatch = FFI::FFIContext::threadsafeDispatch(); + RELEASE_ASSERT(dispatch); + if (callback->tryBeginThreadsafeInvocation()) [[likely]] { + auto invocation = FFI::ThreadsafeInvocation::create(callback, callback->embedderContext(), std::span(slots, argumentCount)); + dispatch(invocation.get()); + } + slots[argumentCount] = 0; + return { encodedJSUndefined(), nullptr }; +} + +JSC_DEFINE_JIT_OPERATION(ffiCallbackDispatch, EncodedJSValue, (JSFFICallback* callback, uint64_t* slots)) +{ + ASSERT(!callback->isThreadsafe()); + JSGlobalObject* globalObject = callback->globalObject(); + VM& vm = globalObject->vm(); + JSLockHolder locker(vm); + + FFI::Signature& signature = callback->signature(); + const unsigned argumentCount = signature.argumentCount(); + uint64_t& returnSlot = slots[argumentCount]; + + if (Exception* pendingException = vm.exceptionForInspection()) [[unlikely]] { + returnSlot = 0; + return { encodedJSUndefined(), pendingException }; + } + + FFI::CallbackEntryScope entryScope(vm); + auto scope = DECLARE_THROW_SCOPE(vm); + + FFI::FFIContext& context = globalObject->ffiContext(); + + MarkedArgumentBuffer arguments; + for (unsigned i = 0; i < argumentCount; ++i) { + FFI::Type type = signature.argumentType(i); + arguments.append(FFI::jsValueFromSlot(globalObject, context, type, slots[i])); + if (scope.exception()) [[unlikely]] { + returnSlot = 0; + OPERATION_RETURN(scope, encodedJSUndefined()); + } + } + if (arguments.hasOverflowed()) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + returnSlot = 0; + OPERATION_RETURN(scope, encodedJSUndefined()); + } + + JSObject* callable = callback->callable(); + CallData callData = JSC::getCallData(callable); + if (callData.type == CallData::Type::None) [[unlikely]] { + throwTypeError(globalObject, scope, "FFI callback target is not callable"_s); + returnSlot = 0; + OPERATION_RETURN(scope, encodedJSUndefined()); + } + + JSValue result = profiledCall(globalObject, ProfilingReason::API, callable, callData, jsUndefined(), arguments); + if (scope.exception()) [[unlikely]] { + returnSlot = 0; + OPERATION_RETURN(scope, encodedJSUndefined()); + } + + if (signature.returnType() != FFI::Type::Void) { + if (signature.returnType() == FFI::Type::CString && result.isString()) { + String string = result.toWTFString(globalObject); + if (scope.exception()) [[unlikely]] { + returnSlot = 0; + OPERATION_RETURN(scope, encodedJSUndefined()); + } + returnSlot = static_cast(reinterpret_cast(callback->setReturnCString(string.utf8()))); + } else { + FFI::writeSlotFromJSValue(globalObject, context, signature.returnType(), result, returnSlot, nullptr); + if (scope.exception()) [[unlikely]] { + returnSlot = 0; + OPERATION_RETURN(scope, encodedJSUndefined()); + } + } + } + + OPERATION_RETURN(scope, JSValue::encode(result)); +} + +namespace FFI { + +void runThreadsafeInvocation(ThreadsafeInvocation& invocation) +{ + JSFFICallback* callback = invocation.callback(); + struct RetireInvocation { + JSFFICallback* callback; + ~RetireInvocation() + { + if (callback->endThreadsafeInvocation()) + callback->unroot(); + } + } retire { callback }; + + JSGlobalObject* globalObject = callback->globalObject(); + VM& vm = globalObject->vm(); + JSLockHolder locker(vm); + auto scope = DECLARE_THROW_SCOPE(vm); + + Signature& signature = callback->signature(); + FFIContext& context = globalObject->ffiContext(); + + std::span slots = invocation.slots(); + ASSERT(slots.size() == signature.argumentCount()); + MarkedArgumentBuffer arguments; + for (unsigned i = 0; i < signature.argumentCount(); ++i) { + arguments.append(jsValueFromSlot(globalObject, context, signature.argumentType(i), slots[i])); + RETURN_IF_EXCEPTION(scope, void()); + } + if (arguments.hasOverflowed()) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + return; + } + + JSObject* callable = callback->callable(); + CallData callData = JSC::getCallData(callable); + if (callData.type == CallData::Type::None) [[unlikely]] { + throwTypeError(globalObject, scope, "FFI callback target is not callable"_s); + return; + } + profiledCall(globalObject, ProfilingReason::API, callable, callData, jsUndefined(), arguments); + RETURN_IF_EXCEPTION(scope, void()); +} + +} // namespace FFI + +#else // !FFI_CALLBACK_THUNK_SUPPORTED + +namespace FFI { +void runThreadsafeInvocation(ThreadsafeInvocation&) +{ + RELEASE_ASSERT_NOT_REACHED(); // unreachable: no threadsafe callback exists to have queued this +} +} // namespace FFI + +#endif // FFI_CALLBACK_THUNK_SUPPORTED + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFICallbackThunk.h b/Source/JavaScriptCore/ffi/FFICallbackThunk.h new file mode 100644 index 0000000000000..7d4b5235f6d66 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFICallbackThunk.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#include "JITOperationValidation.h" +#include "JSCJSValue.h" +#include "JSCPtrTag.h" +#include "MacroAssemblerCodeRef.h" +#include "OperationResult.h" + +#if ENABLE(JIT) && USE(JSVALUE64) && !ENABLE(JIT_CAGE) && (CPU(X86_64) || CPU(ARM64)) +#define FFI_CALLBACK_THUNK_SUPPORTED 1 +#else +#define FFI_CALLBACK_THUNK_SUPPORTED 0 +#endif + +namespace JSC { + +class JSFFICallback; +class VM; + +namespace FFI { + +#if FFI_CALLBACK_THUNK_SUPPORTED +MacroAssemblerCodeRef generateCallbackThunk(VM&, JSFFICallback&); +#endif // FFI_CALLBACK_THUNK_SUPPORTED + +} // namespace FFI + +#if FFI_CALLBACK_THUNK_SUPPORTED +JSC_DECLARE_JIT_OPERATION(ffiCallbackDispatch, EncodedJSValue, (JSFFICallback*, uint64_t*)); +JSC_DECLARE_JIT_OPERATION(ffiCallbackDispatchThreadsafe, EncodedJSValue, (JSFFICallback*, uint64_t*)); +#endif // FFI_CALLBACK_THUNK_SUPPORTED + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFICallingConvention.cpp b/Source/JavaScriptCore/ffi/FFICallingConvention.cpp new file mode 100644 index 0000000000000..8d187e4464d98 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFICallingConvention.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFICallingConvention.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "FFISignature.h" +#include + +namespace JSC::FFI { + +#if ENABLE(ASSEMBLER) && (CPU(X86_64) || CPU(ARM64)) +static constexpr bool listContains(std::span list, GPRReg reg) +{ + for (GPRReg candidate : list) { + if (candidate == reg) + return true; + } + return false; +} +#endif + +#if ENABLE(ASSEMBLER) && CPU(X86_64) +static_assert(!listContains(integerArgumentRegisters(NativeCC::SysV64), X86Registers::ebx)); +static_assert(!listContains(integerArgumentRegisters(NativeCC::SysV64), X86Registers::r10)); +static_assert(!listContains(integerArgumentRegisters(NativeCC::Win64), X86Registers::ebx)); +static_assert(!listContains(integerArgumentRegisters(NativeCC::Win64), X86Registers::r10)); +static_assert(X86Registers::ebx != GPRInfo::returnValueGPR); +static_assert(X86Registers::r10 != GPRInfo::returnValueGPR); +static_assert(X86Registers::ebx != MacroAssembler::s_scratchRegister); +static_assert(X86Registers::r10 != MacroAssembler::s_scratchRegister); +static_assert(integerArgumentRegisters(NativeCC::SysV64).size() == integerArgumentRegisterCount(NativeCC::SysV64)); +static_assert(floatArgumentRegisters(NativeCC::SysV64).size() == floatArgumentRegisterCount(NativeCC::SysV64)); +static_assert(integerArgumentRegisters(NativeCC::Win64).size() == integerArgumentRegisterCount(NativeCC::Win64)); +static_assert(floatArgumentRegisters(NativeCC::Win64).size() == floatArgumentRegisterCount(NativeCC::Win64)); +#endif + +#if ENABLE(ASSEMBLER) && CPU(ARM64) +static_assert(!listContains(integerArgumentRegisters(NativeCC::AAPCS64), ARM64Registers::x19)); +static_assert(!listContains(integerArgumentRegisters(NativeCC::AAPCS64), ARM64Registers::x9)); +static_assert(ARM64Registers::x19 != GPRInfo::returnValueGPR); +static_assert(ARM64Registers::x9 != GPRInfo::returnValueGPR); +static_assert(ARM64Registers::x19 != MacroAssembler::dataTempRegister); +static_assert(ARM64Registers::x19 != MacroAssembler::memoryTempRegister); +static_assert(ARM64Registers::x9 != MacroAssembler::dataTempRegister); +static_assert(ARM64Registers::x9 != MacroAssembler::memoryTempRegister); +static_assert(integerArgumentRegisters(NativeCC::AAPCS64).size() == integerArgumentRegisterCount(NativeCC::AAPCS64)); +static_assert(floatArgumentRegisters(NativeCC::AAPCS64).size() == floatArgumentRegisterCount(NativeCC::AAPCS64)); +#endif + +std::array scratchGPRsForInvoke(NativeCC cc) +{ +#if ENABLE(ASSEMBLER) && CPU(X86_64) + switch (cc) { + case NativeCC::SysV64: + case NativeCC::Win64: + return { X86Registers::ebx, X86Registers::r10 }; + case NativeCC::AAPCS64: + break; + } +#elif ENABLE(ASSEMBLER) && CPU(ARM64) + if (cc == NativeCC::AAPCS64) + return { ARM64Registers::x19, ARM64Registers::x9 }; +#else + UNUSED_PARAM(cc); +#endif + RELEASE_ASSERT_NOT_REACHED(); + return { InvalidGPRReg, InvalidGPRReg }; +} + +CallLayout computeCallLayout(NativeCC cc, const Signature& signature, Direction direction) +{ + return computeCallLayout(cc, stackPackingForNativeCC(cc), signature, direction); +} + +CallLayout computeCallLayout(NativeCC cc, StackPacking packing, const Signature& signature, Direction direction) +{ + UNUSED_PARAM(direction); + + if (cc != NativeCC::AAPCS64) + packing = StackPacking::EightByteSlots; + + CallLayout layout; + layout.cc = cc; + layout.packing = packing; + layout.returnClass = argClass(signature.returnType()); + + const unsigned integerRegisterCount = integerArgumentRegisterCount(cc); + const unsigned floatRegisterCount = floatArgumentRegisterCount(cc); + unsigned gprIndex = 0; + unsigned fprIndex = 0; + unsigned nextStackOffset = shadowStackBytes(cc); + + unsigned argumentCount = signature.argumentCount(); + layout.arguments.reserveInitialCapacity(argumentCount); + for (unsigned i = 0; i < argumentCount; ++i) { + Type type = signature.argumentType(i); + ASSERT(isValidArgumentType(type)); + ArgClass klass = argClass(type); + bool isFloatingPoint = klass == ArgClass::Float || klass == ArgClass::Double; + + ArgLocation location; + location.type = type; + + if (cc == NativeCC::Win64) { + constexpr unsigned win64RegisterSlots = 4; + if (i < win64RegisterSlots) { + location.kind = isFloatingPoint ? ArgLocation::Kind::FPR : ArgLocation::Kind::GPR; + location.regIndex = static_cast(i); + } else { + location.kind = ArgLocation::Kind::Stack; + location.stackOffset = shadowStackBytes(NativeCC::Win64) + (i - win64RegisterSlots) * 8; + nextStackOffset = location.stackOffset + 8; + } + } else if (isFloatingPoint && fprIndex < floatRegisterCount) { + location.kind = ArgLocation::Kind::FPR; + location.regIndex = static_cast(fprIndex++); + } else if (!isFloatingPoint && gprIndex < integerRegisterCount) { + location.kind = ArgLocation::Kind::GPR; + location.regIndex = static_cast(gprIndex++); + } else { + location.kind = ArgLocation::Kind::Stack; + switch (packing) { + case StackPacking::EightByteSlots: + location.stackOffset = nextStackOffset; + nextStackOffset += 8; + break; + case StackPacking::Natural: { + unsigned size = nativeSizeInBytes(type); + location.stackOffset = roundUpToMultipleOf(size, nextStackOffset); + nextStackOffset = location.stackOffset + size; + break; + } + } + } + + layout.arguments.append(location); + } + + layout.stackBytes = static_cast(roundUpToMultipleOf<16>(static_cast(nextStackOffset))); + ASSERT(!(layout.stackBytes % 16)); + return layout; +} + +unsigned incomingStackOffset(const CallLayout& layout, unsigned argIndex) +{ + RELEASE_ASSERT(argIndex < layout.arguments.size()); + const ArgLocation& location = layout.arguments[argIndex]; + RELEASE_ASSERT(location.kind == ArgLocation::Kind::Stack); + constexpr unsigned savedFrameAndReturnAddressBytes = 2 * sizeof(void*); + return location.stackOffset + savedFrameAndReturnAddressBytes; +} + +} // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFICallingConvention.h b/Source/JavaScriptCore/ffi/FFICallingConvention.h new file mode 100644 index 0000000000000..2765814135467 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFICallingConvention.h @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) + +#include "FFIType.h" +#include "FPRInfo.h" +#include "GPRInfo.h" +#include "JSExportMacros.h" +#include +#include +#include + +namespace JSC::FFI { + +class Signature; + +enum class NativeCC : uint8_t { SysV64, AAPCS64, Win64 }; + +enum class StackPacking : uint8_t { EightByteSlots, Natural }; + +constexpr NativeCC hostNativeCC() +{ +#if OS(WINDOWS) && CPU(X86_64) + return NativeCC::Win64; +#elif CPU(ARM64) + return NativeCC::AAPCS64; +#else + return NativeCC::SysV64; +#endif +} + +constexpr StackPacking stackPackingForNativeCC(NativeCC cc) +{ +#if OS(DARWIN) && CPU(ARM64) + if (cc == NativeCC::AAPCS64) + return StackPacking::Natural; +#else + UNUSED_PARAM(cc); +#endif + return StackPacking::EightByteSlots; +} + +struct ArgLocation { + enum class Kind : uint8_t { GPR, FPR, Stack } kind { Kind::GPR }; + uint8_t regIndex { 0 }; + unsigned stackOffset { 0 }; + Type type { Type::Void }; +}; + +struct CallLayout { + NativeCC cc { hostNativeCC() }; + Vector arguments; + unsigned stackBytes { 0 }; + ArgClass returnClass { ArgClass::Void }; + StackPacking packing { StackPacking::EightByteSlots }; +}; + +enum class Direction : uint8_t { Outgoing, Incoming }; + +JS_EXPORT_PRIVATE CallLayout computeCallLayout(NativeCC, const Signature&, Direction = Direction::Outgoing); +JS_EXPORT_PRIVATE CallLayout computeCallLayout(NativeCC, StackPacking, const Signature&, Direction = Direction::Outgoing); + +JS_EXPORT_PRIVATE unsigned incomingStackOffset(const CallLayout&, unsigned argIndex); + +JS_EXPORT_PRIVATE std::array scratchGPRsForInvoke(NativeCC); + +constexpr unsigned shadowStackBytes(NativeCC cc) +{ + return cc == NativeCC::Win64 ? 32 : 0; +} + +constexpr unsigned integerArgumentRegisterCount(NativeCC cc) +{ + switch (cc) { + case NativeCC::SysV64: + return 6; + case NativeCC::AAPCS64: + return 8; + case NativeCC::Win64: + return 4; + } + RELEASE_ASSERT_NOT_REACHED_UNDER_CONSTEXPR_CONTEXT(); +} + +constexpr unsigned floatArgumentRegisterCount(NativeCC cc) +{ + switch (cc) { + case NativeCC::SysV64: + return 8; + case NativeCC::AAPCS64: + return 8; + case NativeCC::Win64: + return 4; + } + RELEASE_ASSERT_NOT_REACHED_UNDER_CONSTEXPR_CONTEXT(); +} + +#if ENABLE(ASSEMBLER) && CPU(X86_64) +inline constexpr std::array s_sysV64IntegerArgumentRegisters { + X86Registers::edi, X86Registers::esi, X86Registers::edx, X86Registers::ecx, X86Registers::r8, X86Registers::r9, +}; +inline constexpr std::array s_sysV64FloatArgumentRegisters { + X86Registers::xmm0, X86Registers::xmm1, X86Registers::xmm2, X86Registers::xmm3, + X86Registers::xmm4, X86Registers::xmm5, X86Registers::xmm6, X86Registers::xmm7, +}; +inline constexpr std::array s_win64IntegerArgumentRegisters { + X86Registers::ecx, X86Registers::edx, X86Registers::r8, X86Registers::r9, +}; +inline constexpr std::array s_win64FloatArgumentRegisters { + X86Registers::xmm0, X86Registers::xmm1, X86Registers::xmm2, X86Registers::xmm3, +}; +#endif // ENABLE(ASSEMBLER) && CPU(X86_64) + +#if ENABLE(ASSEMBLER) && CPU(ARM64) +inline constexpr std::array s_aapcs64IntegerArgumentRegisters { + ARM64Registers::x0, ARM64Registers::x1, ARM64Registers::x2, ARM64Registers::x3, + ARM64Registers::x4, ARM64Registers::x5, ARM64Registers::x6, ARM64Registers::x7, +}; +inline constexpr std::array s_aapcs64FloatArgumentRegisters { + ARM64Registers::q0, ARM64Registers::q1, ARM64Registers::q2, ARM64Registers::q3, + ARM64Registers::q4, ARM64Registers::q5, ARM64Registers::q6, ARM64Registers::q7, +}; +#endif // ENABLE(ASSEMBLER) && CPU(ARM64) + +constexpr std::span integerArgumentRegisters(NativeCC cc) +{ + switch (cc) { + case NativeCC::SysV64: +#if ENABLE(ASSEMBLER) && CPU(X86_64) + return std::span { s_sysV64IntegerArgumentRegisters }; +#else + return { }; +#endif + case NativeCC::Win64: +#if ENABLE(ASSEMBLER) && CPU(X86_64) + return std::span { s_win64IntegerArgumentRegisters }; +#else + return { }; +#endif + case NativeCC::AAPCS64: +#if ENABLE(ASSEMBLER) && CPU(ARM64) + return std::span { s_aapcs64IntegerArgumentRegisters }; +#else + return { }; +#endif + } + RELEASE_ASSERT_NOT_REACHED_UNDER_CONSTEXPR_CONTEXT(); +} + +constexpr std::span floatArgumentRegisters(NativeCC cc) +{ + switch (cc) { + case NativeCC::SysV64: +#if ENABLE(ASSEMBLER) && CPU(X86_64) + return std::span { s_sysV64FloatArgumentRegisters }; +#else + return { }; +#endif + case NativeCC::Win64: +#if ENABLE(ASSEMBLER) && CPU(X86_64) + return std::span { s_win64FloatArgumentRegisters }; +#else + return { }; +#endif + case NativeCC::AAPCS64: +#if ENABLE(ASSEMBLER) && CPU(ARM64) + return std::span { s_aapcs64FloatArgumentRegisters }; +#else + return { }; +#endif + } + RELEASE_ASSERT_NOT_REACHED_UNDER_CONSTEXPR_CONTEXT(); +} + +} // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIContext.cpp b/Source/JavaScriptCore/ffi/FFIContext.cpp new file mode 100644 index 0000000000000..2e272b5615cce --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIContext.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFIContext.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "JSCJSValueInlines.h" +#include "JSFFICallback.h" +#include "SlotVisitorInlines.h" +#include +#include + +namespace JSC { namespace FFI { + +WTF_MAKE_TZONE_ALLOCATED_IMPL(FFIContext); + +WTF_MAKE_TZONE_ALLOCATED_IMPL(ThreadsafeInvocation); + +FFIContext::ThreadsafeDispatchFunction FFIContext::s_threadsafeDispatch { nullptr }; + +FFIContext::FFIContext(VM& vm) + : m_vm(vm) +{ + m_vm.heap.addObserver(this); +} + +FFIContext::~FFIContext() +{ + m_vm.heap.removeObserver(this); +} + +void FFIContext::setThreadsafeDispatch(ThreadsafeDispatchFunction fn) +{ + s_threadsafeDispatch = fn; +} + +void FFIContext::addLiveCallback(VM& vm, JSGlobalObject& owner, JSFFICallback* callback) +{ + Locker locker { owner.cellLock() }; + m_liveCallbacks.append(WriteBarrier(vm, &owner, callback)); +} + +void FFIContext::removeLiveCallback(JSGlobalObject& owner, JSFFICallback* callback) +{ + Locker locker { owner.cellLock() }; + m_liveCallbacks.removeFirstMatching([&](const WriteBarrier& entry) { + return entry.get() == callback; + }); +} + +template +void FFIContext::visitLiveCallbacks(JSGlobalObject& owner, Visitor& visitor) +{ + Locker locker { owner.cellLock() }; + for (auto& callback : m_liveCallbacks) + visitor.append(callback); +} + +template void FFIContext::visitLiveCallbacks(JSGlobalObject&, AbstractSlotVisitor&); +template void FFIContext::visitLiveCallbacks(JSGlobalObject&, SlotVisitor&); + +const CString* FFIContext::cachedUTF8(StringImpl& impl) +{ + for (auto& entry : m_utf8Cache) { + if (entry.key.get() == &impl) { + entry.lastUse = ++m_utf8CacheClock; + return &entry.utf8; + } + } + return nullptr; +} + +const CString& FFIContext::cacheUTF8(StringImpl& impl, CString&& utf8) +{ + if (m_utf8Cache.size() < utf8CacheCapacity) { + UTF8CacheEntry entry; + entry.key = &impl; + entry.utf8 = WTF::move(utf8); + entry.lastUse = ++m_utf8CacheClock; + m_utf8Cache.append(WTF::move(entry)); + return m_utf8Cache.last().utf8; + } + + UTF8CacheEntry* victim = &m_utf8Cache[0]; + for (auto& entry : m_utf8Cache) { + if (entry.lastUse < victim->lastUse) + victim = &entry; + } + victim->key = &impl; + victim->utf8 = WTF::move(utf8); + victim->lastUse = ++m_utf8CacheClock; + return victim->utf8; +} + +void StringArena::enter() +{ + ++m_depth; +} + +void StringArena::reset() +{ + ASSERT(!m_depth); + m_offsetInLastChunk = 0; + if (m_chunks.isEmpty()) + return; + if (m_chunks[0].sizeInBytes() > maximumRetainedChunkBytes) { + m_chunks.clear(); + return; + } + m_chunks.shrink(1); +} + +std::span StringArena::allocate(size_t bytes) +{ + ASSERT(m_depth); + + if (!m_chunks.isEmpty()) { + auto lastChunk = m_chunks.last().mutableSpan(); + if (lastChunk.size() - m_offsetInLastChunk >= bytes) { + auto result = lastChunk.subspan(m_offsetInLastChunk, bytes); + m_offsetInLastChunk += bytes; + return result; + } + } + + size_t capacity = std::max(defaultChunkBytes, bytes); + auto chunk = MallocSpan::tryMalloc(capacity); + if (!chunk) + return { }; + m_chunks.append(WTF::move(chunk)); + m_offsetInLastChunk = bytes; + return m_chunks.last().mutableSpan().first(bytes); +} + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIContext.h b/Source/JavaScriptCore/ffi/FFIContext.h new file mode 100644 index 0000000000000..8ca22fb019a14 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIContext.h @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) + +#include "HeapObserver.h" +#include "JSExportMacros.h" +#include "WriteBarrier.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace JSC { +class JSFFICallback; +class VM; +} // namespace JSC + +namespace JSC { namespace FFI { + +class ThreadsafeInvocation final : public ThreadSafeRefCounted { + WTF_MAKE_TZONE_ALLOCATED(ThreadsafeInvocation); +public: + static Ref create(JSFFICallback* callback, void* embedderContext, std::span slots) + { + return adoptRef(*new ThreadsafeInvocation(callback, embedderContext, slots)); + } + + JSFFICallback* callback() const { return m_callback; } + void* embedderContext() const { return m_embedderContext; } + std::span slots() { return m_slots.mutableSpan(); } + std::span slots() const { return m_slots.span(); } + +private: + ThreadsafeInvocation(JSFFICallback* callback, void* embedderContext, std::span slots) + : m_callback(callback) + , m_embedderContext(embedderContext) + , m_slots(slots) + { + } + + JSFFICallback* m_callback; + void* m_embedderContext; + Vector m_slots; +}; + +class FFIContext; + +class StringArena { + WTF_MAKE_NONCOPYABLE(StringArena); +public: + class Scope { + WTF_MAKE_NONCOPYABLE(Scope); + WTF_FORBID_HEAP_ALLOCATION; + public: + explicit Scope(StringArena& arena) + : m_arena(arena) + { + m_arena.enter(); + } + + explicit Scope(FFIContext&); + + ~Scope() + { + m_arena.exit(); + } + + private: + StringArena& m_arena; + }; + + StringArena() = default; + ~StringArena() = default; + + JS_EXPORT_PRIVATE void enter(); + void exit() + { + ASSERT(m_depth); + if (!--m_depth) + reset(); + } + unsigned depth() const { return m_depth; } + + JS_EXPORT_PRIVATE std::span allocate(size_t bytes); + void shrinkWhenIdle() + { + if (m_depth) + return; + m_offsetInLastChunk = 0; + m_chunks.clear(); + } + +private: + void reset(); + + static constexpr size_t defaultChunkBytes = 4096; + static constexpr size_t maximumRetainedChunkBytes = 64 * 1024; + + Vector, 4> m_chunks; + size_t m_offsetInLastChunk { 0 }; + unsigned m_depth { 0 }; +}; + +class FFIContext final : public HeapObserver { + WTF_MAKE_TZONE_ALLOCATED(FFIContext); + WTF_MAKE_NONCOPYABLE(FFIContext); +public: + JS_EXPORT_PRIVATE explicit FFIContext(VM&); + JS_EXPORT_PRIVATE ~FFIContext(); + + void willGarbageCollect() final { } + void didGarbageCollect(CollectionScope) final { m_arena.shrinkWhenIdle(); } + + StringArena& stringArena() { return m_arena; } + StringArena& arena() { return m_arena; } + + const CString* cachedUTF8(StringImpl&); + const CString& cacheUTF8(StringImpl&, CString&&); + + static constexpr unsigned utf8CacheCapacity = 64; + + void addLiveCallback(VM&, JSGlobalObject& owner, JSFFICallback*); + void removeLiveCallback(JSGlobalObject& owner, JSFFICallback*); + template void visitLiveCallbacks(JSGlobalObject& owner, Visitor&); + + using ThreadsafeDispatchFunction = void (*)(ThreadsafeInvocation&); + JS_EXPORT_PRIVATE static void setThreadsafeDispatch(ThreadsafeDispatchFunction); + static ThreadsafeDispatchFunction threadsafeDispatch() { return s_threadsafeDispatch; } + +private: + Vector> m_liveCallbacks; + JS_EXPORT_PRIVATE static ThreadsafeDispatchFunction s_threadsafeDispatch; + struct UTF8CacheEntry { + RefPtr key; + CString utf8; + uint64_t lastUse { 0 }; + }; + + VM& m_vm; + StringArena m_arena; + Vector m_utf8Cache; + uint64_t m_utf8CacheClock { 0 }; +}; + +inline StringArena::Scope::Scope(FFIContext& context) + : Scope(context.arena()) +{ +} + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIConversions.cpp b/Source/JavaScriptCore/ffi/FFIConversions.cpp new file mode 100644 index 0000000000000..5e669561b36a4 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIConversions.cpp @@ -0,0 +1,549 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFIConversions.h" + +#if USE(BUN_JSC_ADDITIONS) + +#if USE(JSVALUE64) + +#include "CallFrame.h" +#include "Error.h" +#include "ExceptionHelpers.h" +#include "FFIContext.h" +#include "FFIType.h" +#include "FrameTracers.h" +#include "JSArrayBuffer.h" +#include "JSArrayBufferView.h" +#include "JSBigInt.h" +#include "JSBigIntInlines.h" +#include "JSCInlines.h" +#include "JSCJSValueInlines.h" +#include "JSFFICallback.h" +#include "JSGlobalObject.h" +#include "JSString.h" +#include "PureNaN.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#if CPU(X86_64) +#include +#elif CPU(ARM64) +#include +#endif + +namespace JSC { namespace FFI { + +JSValue pointerToJSValue(JSGlobalObject* globalObject, uint64_t address) +{ + if (!address) + return jsNull(); + if (address <= static_cast(9007199254740991ULL)) + return jsNumber(static_cast(address)); + return JSBigInt::createFrom(globalObject, address); +} + +static constexpr int64_t maxInt52 = 9007199254740991; + +int64_t doubleToInt64(double value) +{ +#if CPU(X86_64) + return _mm_cvttsd_si64(_mm_set_sd(value)); +#elif CPU(ARM64) + return vcvtd_s64_f64(value); +#else + if (std::isnan(value)) + return 0; + if (value >= 9223372036854775808.0) + return std::numeric_limits::max(); + if (value <= -9223372036854775808.0) + return std::numeric_limits::min(); + return static_cast(value); +#endif +} + +static bool throwCannotConvert(JSGlobalObject* globalObject, ThrowScope& scope, Type type) +{ + throwTypeError(globalObject, scope, makeString("bun:ffi cannot convert argument to '"_s, name(type), '\'')); + return false; +} + +static bool writeIntegerSlot(JSGlobalObject* globalObject, Type type, JSValue value, uint64_t& slotOut) +{ + VM& vm = getVM(globalObject); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (type == Type::Bool) { + slotOut = value.toBoolean(globalObject) ? 1 : 0; + return true; + } + + if (value.isBigInt()) [[unlikely]] { + uint64_t bits = JSBigInt::toBigUInt64(value); + switch (type) { + case Type::Char: + case Type::Int8: + slotOut = static_cast(static_cast(static_cast(bits))); + return true; + case Type::Uint8: + slotOut = static_cast(static_cast(bits)); + return true; + case Type::Int16: + slotOut = static_cast(static_cast(static_cast(bits))); + return true; + case Type::Uint16: + slotOut = static_cast(static_cast(bits)); + return true; + case Type::Int32: + slotOut = static_cast(static_cast(static_cast(bits))); + return true; + case Type::Uint32: + slotOut = static_cast(static_cast(bits)); + return true; + default: + RELEASE_ASSERT_NOT_REACHED(); + return false; + } + } + + if (value.isString() || value.isSymbol()) [[unlikely]] + return throwCannotConvert(globalObject, scope, type); + + if (type == Type::Uint32) { + uint32_t truncated = value.toUInt32(globalObject); + RETURN_IF_EXCEPTION(scope, false); + slotOut = static_cast(truncated); + return true; + } + + int32_t truncated = value.toInt32(globalObject); + RETURN_IF_EXCEPTION(scope, false); + switch (type) { + case Type::Char: + case Type::Int8: + slotOut = static_cast(static_cast(static_cast(truncated))); + break; + case Type::Uint8: + slotOut = static_cast(static_cast(truncated)); + break; + case Type::Int16: + slotOut = static_cast(static_cast(static_cast(truncated))); + break; + case Type::Uint16: + slotOut = static_cast(static_cast(truncated)); + break; + case Type::Int32: + slotOut = static_cast(static_cast(truncated)); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + return false; + } + return true; +} + +static bool writeInt64Slot(JSGlobalObject* globalObject, Type type, JSValue value, uint64_t& slotOut) +{ + VM& vm = getVM(globalObject); + auto scope = DECLARE_THROW_SCOPE(vm); + + bool isUnsigned = type == Type::Uint64 || type == Type::Uint64Fast; + if (value.isInt32()) { + slotOut = static_cast(static_cast(value.asInt32())); + return true; + } + if (value.isDouble()) { + slotOut = isUnsigned ? doubleToUInt64(value.asDouble()) : static_cast(doubleToInt64(value.asDouble())); + return true; + } + if (value.isBigInt()) { + slotOut = isUnsigned ? JSBigInt::toBigUInt64(value) : static_cast(JSBigInt::toBigInt64(value)); + return true; + } + return throwCannotConvert(globalObject, scope, type); +} + +static bool writeFloatingPointSlot(JSGlobalObject* globalObject, Type type, JSValue value, uint64_t& slotOut) +{ + VM& vm = getVM(globalObject); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (value.isString() || value.isSymbol()) [[unlikely]] + return throwCannotConvert(globalObject, scope, type); + + double number; + if (value.isNumber()) + number = value.asNumber(); + else if (value.isUndefined()) + number = PNaN; // Number(undefined) === NaN. + else if (value.isBigInt()) + number = JSBigInt::toNumber(value).asNumber(); + else { + number = value.toNumber(globalObject); + RETURN_IF_EXCEPTION(scope, false); + } + + if (type == Type::Double) { + slotOut = std::bit_cast(number); + return true; + } + + ASSERT(type == Type::Float); + float narrowed = static_cast(number); // Math.fround semantics. + slotOut = static_cast(std::bit_cast(narrowed)); + return true; +} + +static bool writeCStringSlot(JSGlobalObject* globalObject, FFIContext& context, JSString* jsString, uint64_t& slotOut, StringArena* arena) +{ + VM& vm = getVM(globalObject); + auto scope = DECLARE_THROW_SCOPE(vm); + + auto string = jsString->value(globalObject); + RETURN_IF_EXCEPTION(scope, false); + + if (!arena) { + throwTypeError(globalObject, scope, "bun:ffi: a JavaScript string is not valid here; return it from a 'cstring'-returning callback, or pass a pointer/TypedArray"_s); + return false; + } + StringArena& targetArena = *arena; + ASSERT_WITH_MESSAGE(targetArena.depth(), "bun:ffi cstring conversion requires an active FFI arena bracket"); + + StringImpl* impl = string->impl(); + if (!impl || !impl->length()) { + auto storage = targetArena.allocate(1); + if (storage.empty()) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + return false; + } + storage[0] = '\0'; + slotOut = static_cast(reinterpret_cast(storage.data())); + return true; + } + + if (impl->is8Bit()) { + auto characters = impl->span8(); + if (charactersAreAllASCII(characters)) { + auto storage = targetArena.allocate(characters.size() + 1); + if (storage.empty()) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + return false; + } + memcpySpan(storage, characters); + storage[characters.size()] = '\0'; + slotOut = static_cast(reinterpret_cast(storage.data())); + return true; + } + } + + const CString* utf8 = context.cachedUTF8(*impl); + if (!utf8) { + auto result = impl->tryGetUTF8(); + if (!result) [[unlikely]] { + if (result.error() == UTF8ConversionError::OutOfMemory) + throwOutOfMemoryError(globalObject, scope); + else + throwTypeError(globalObject, scope, "bun:ffi could not encode the string argument as UTF-8"_s); + return false; + } + utf8 = &context.cacheUTF8(*impl, WTF::move(result.value())); + } + + auto bytes = utf8->span(); + auto storage = targetArena.allocate(bytes.size() + 1); + if (storage.empty()) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + return false; + } + memcpySpan(storage, bytes); + storage[bytes.size()] = '\0'; + slotOut = static_cast(reinterpret_cast(storage.data())); + return true; +} + +static bool writeBufferLengthSlot(JSGlobalObject* globalObject, JSValue value, uint64_t& slotOut) +{ + VM& vm = getVM(globalObject); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (auto* view = dynamicDowncast(value)) { + slotOut = static_cast(view->byteLength()); + return true; + } + throwTypeError(globalObject, scope, "bun:ffi 'buffer_length' argument must be a TypedArray or DataView"_s); + return false; +} + +static bool writePointerSlot(JSGlobalObject* globalObject, FFIContext& context, Type type, JSValue value, uint64_t& slotOut, StringArena* arena) +{ + VM& vm = getVM(globalObject); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (type == Type::Buffer) { + if (auto* view = dynamicDowncast(value)) { + slotOut = static_cast(reinterpret_cast(view->vector())); + return true; + } + throwTypeError(globalObject, scope, "bun:ffi 'buffer' argument must be a TypedArray or DataView"_s); + return false; + } + + if (value.isUndefinedOrNull()) { + if (type == Type::Function) [[unlikely]] { + throwTypeError(globalObject, scope, "bun:ffi: expected a callback (a JSCallback or an FFI function) but got undefined/null"_s); + return false; + } + slotOut = 0; + return true; + } + + if (value.isInt32()) { + slotOut = static_cast(static_cast(static_cast(value.asInt32()))); + return true; + } + + if (value.isDouble()) { + slotOut = static_cast(static_cast(doubleToInt64(value.asDouble()))); + return true; + } + + if (value.isBigInt()) { + slotOut = JSBigInt::toBigUInt64(value); + RETURN_IF_EXCEPTION(scope, false); + return true; + } + + if (value.isCell()) { + JSCell* cell = value.asCell(); + + if (auto* view = dynamicDowncast(cell)) { + slotOut = static_cast(reinterpret_cast(view->vector())); + return true; + } + + if (auto* buffer = dynamicDowncast(cell)) { + slotOut = static_cast(reinterpret_cast(buffer->impl()->data())); + return true; + } + + if (auto* callback = dynamicDowncast(cell)) { + slotOut = static_cast(reinterpret_cast(callback->nativeEntrypoint())); + return true; + } + + if (cell->isString()) { + if (type == Type::CString) + RELEASE_AND_RETURN(scope, writeCStringSlot(globalObject, context, uncheckedDowncast(cell), slotOut, arena)); + throwTypeError(globalObject, scope, "To convert a string to a pointer, encode it as a buffer"_s); + return false; + } + + if (type != Type::Buffer && cell->isObject()) { + JSValue ptrValue = uncheckedDowncast(cell)->get(globalObject, Identifier::fromString(vm, "ptr"_s)); + RETURN_IF_EXCEPTION(scope, false); + if (ptrValue.isNumber() || ptrValue.isBigInt()) + RELEASE_AND_RETURN(scope, writePointerSlot(globalObject, context, type, ptrValue, slotOut, arena)); + } + } + + return throwCannotConvert(globalObject, scope, type); +} + +bool writeSlotFromJSValue(JSGlobalObject* globalObject, FFIContext& context, Type type, JSValue value, uint64_t& slotOut, StringArena* arena) +{ + switch (type) { + case Type::Char: + case Type::Int8: + case Type::Uint8: + case Type::Int16: + case Type::Uint16: + case Type::Int32: + case Type::Uint32: + case Type::Bool: + return writeIntegerSlot(globalObject, type, value, slotOut); + + case Type::Int64: + case Type::Uint64: + case Type::Int64Fast: + case Type::Uint64Fast: + return writeInt64Slot(globalObject, type, value, slotOut); + + case Type::Double: + case Type::Float: + return writeFloatingPointSlot(globalObject, type, value, slotOut); + + case Type::Pointer: + case Type::CString: + case Type::Function: + case Type::Buffer: + return writePointerSlot(globalObject, context, type, value, slotOut, arena); + + case Type::BufferLength: + return writeBufferLengthSlot(globalObject, value, slotOut); + + case Type::JSValue: + slotOut = static_cast(JSValue::encode(value)); + return true; + + case Type::RESERVED_WasNapiEnv: + RELEASE_ASSERT_NOT_REACHED(); + return false; + + case Type::Void: + return true; + } + + RELEASE_ASSERT_NOT_REACHED(); + return false; +} + +JSValue jsValueFromSlot(JSGlobalObject* globalObject, FFIContext&, Type type, uint64_t slot) +{ + switch (type) { + case Type::Char: + case Type::Int8: + return jsNumber(static_cast(static_cast(slot))); + case Type::Uint8: + return jsNumber(static_cast(static_cast(slot))); + case Type::Int16: + return jsNumber(static_cast(static_cast(slot))); + case Type::Uint16: + return jsNumber(static_cast(static_cast(slot))); + case Type::Int32: + return jsNumber(static_cast(slot)); + case Type::Uint32: + return jsNumber(static_cast(slot)); + case Type::Bool: + return jsBoolean(!!slot); + case Type::Int64: + return JSBigInt::makeHeapBigIntOrBigInt32(globalObject, static_cast(slot)); + case Type::Uint64: + case Type::BufferLength: + return JSBigInt::createFrom(globalObject, static_cast(slot)); + case Type::Int64Fast: { + int64_t value = static_cast(slot); + if (value >= -maxInt52 && value <= maxInt52) + return jsNumber(static_cast(value)); + return JSBigInt::makeHeapBigIntOrBigInt32(globalObject, value); + } + case Type::Uint64Fast: { + uint64_t value = static_cast(slot); + if (value < static_cast(maxInt52)) + return jsNumber(static_cast(value)); + return JSBigInt::createFrom(globalObject, value); + } + case Type::Double: + return jsNumber(purifyNaN(std::bit_cast(slot))); + case Type::Float: + return jsNumber(purifyNaN(static_cast(std::bit_cast(static_cast(slot))))); + case Type::CString: { + const char* string = reinterpret_cast(static_cast(slot)); + if (!string) + return jsNull(); + String decoded = String::fromUTF8WithLatin1Fallback(std::span(string, strlen(string))); + return jsString(getVM(globalObject), decoded); + } + case Type::Pointer: + case Type::Function: + case Type::Buffer: + return pointerToJSValue(globalObject, static_cast(slot)); + case Type::JSValue: + return JSValue::decode(static_cast(slot)); + case Type::Void: + return jsUndefined(); + case Type::RESERVED_WasNapiEnv: + RELEASE_ASSERT_NOT_REACHED(); + return jsUndefined(); + } + + RELEASE_ASSERT_NOT_REACHED(); + return jsUndefined(); +} + +} } // namespace JSC::FFI + +namespace JSC { + +JSC_DEFINE_JIT_OPERATION(operationFFIBoxSlot, EncodedJSValue, (JSGlobalObject* globalObject, uint32_t typeTag, uint64_t slot, int32_t exitArena)) +{ + VM& vm = globalObject->vm(); + CallFrame* callFrame = DECLARE_CALL_FRAME(vm); + JITOperationPrologueCallFrameTracer tracer(vm, callFrame); + auto scope = DECLARE_THROW_SCOPE(vm); + + ASSERT(typeTag < FFI::numberOfTypes); + JSValue boxed = FFI::jsValueFromSlot(globalObject, globalObject->ffiContext(), static_cast(typeTag), slot); + if (exitArena) + globalObject->ffiContext().stringArena().exit(); + OPERATION_RETURN(scope, JSValue::encode(boxed)); +} + +JSC_DEFINE_JIT_OPERATION(operationFFIWriteSlot, void, (JSGlobalObject* globalObject, FFI::FFIContext* context, uint32_t typeTag, EncodedJSValue value, uint64_t* slot)) +{ + VM& vm = globalObject->vm(); + CallFrame* callFrame = DECLARE_CALL_FRAME(vm); + JITOperationPrologueCallFrameTracer tracer(vm, callFrame); + auto scope = DECLARE_THROW_SCOPE(vm); + + ASSERT(context); + ASSERT(typeTag < FFI::numberOfTypes); + FFI::writeSlotFromJSValue(globalObject, *context, static_cast(typeTag), JSValue::decode(value), *slot, &context->arena()); + OPERATION_RETURN(scope); +} + +JSC_DEFINE_JIT_OPERATION(operationFFIArenaEnter, void, (JSGlobalObject* globalObject)) +{ + VM& vm = globalObject->vm(); + CallFrame* callFrame = DECLARE_CALL_FRAME(vm); + JITOperationPrologueCallFrameTracer tracer(vm, callFrame); + auto scope = DECLARE_THROW_SCOPE(vm); + + globalObject->ffiContext().arena().enter(); + OPERATION_RETURN(scope); +} + +JSC_DEFINE_JIT_OPERATION(operationFFIArenaExit, void, (JSGlobalObject* globalObject)) +{ + VM& vm = globalObject->vm(); + CallFrame* callFrame = DECLARE_CALL_FRAME(vm); + JITOperationPrologueCallFrameTracer tracer(vm, callFrame); + auto scope = DECLARE_THROW_SCOPE(vm); + + globalObject->ffiContext().arena().exit(); + OPERATION_RETURN(scope); +} + +} // namespace JSC + +#endif // USE(JSVALUE64) + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIConversions.h b/Source/JavaScriptCore/ffi/FFIConversions.h new file mode 100644 index 0000000000000..33e2ddd4c3767 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIConversions.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) + +#if USE(JSVALUE64) + +#include "FFIContext.h" +#include "FFIType.h" +#include "JSCJSValue.h" +#include "JSExportMacros.h" +#include "OperationResult.h" +#include + +namespace JSC { + +class JSGlobalObject; + +namespace FFI { + +JS_EXPORT_PRIVATE bool writeSlotFromJSValue(JSGlobalObject*, FFIContext&, Type, JSValue, uint64_t& slotOut, StringArena* arena); + +JS_EXPORT_PRIVATE JSValue jsValueFromSlot(JSGlobalObject*, FFIContext&, Type, uint64_t slot); + +JS_EXPORT_PRIVATE JSValue pointerToJSValue(JSGlobalObject*, uint64_t address); + +JS_EXPORT_PRIVATE int64_t doubleToInt64(double); + +inline uint64_t doubleToUInt64(double value) +{ + return std::bit_cast(doubleToInt64(value)); +} + +} // namespace FFI + +JSC_DECLARE_JIT_OPERATION(operationFFIBoxSlot, EncodedJSValue, (JSGlobalObject*, uint32_t typeTag, uint64_t slot, int32_t exitArena)); +JSC_DECLARE_JIT_OPERATION(operationFFIWriteSlot, void, (JSGlobalObject*, FFI::FFIContext*, uint32_t typeTag, EncodedJSValue value, uint64_t* slot)); +JSC_DECLARE_JIT_OPERATION(operationFFIArenaEnter, void, (JSGlobalObject*)); +JSC_DECLARE_JIT_OPERATION(operationFFIArenaExit, void, (JSGlobalObject*)); + +} // namespace JSC + +#endif // USE(JSVALUE64) + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIDFG.cpp b/Source/JavaScriptCore/ffi/FFIDFG.cpp new file mode 100644 index 0000000000000..cd4450ea62443 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIDFG.cpp @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFIDFG.h" + +#if USE(BUN_JSC_ADDITIONS) + +#if ENABLE(DFG_JIT) + +#include "DFGGraph.h" +#include "DFGInsertionSet.h" +#include "DFGNode.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "JSCellInlines.h" +#include "JSFFIFunction.h" +#include "Options.h" + +namespace JSC { namespace FFI { + +bool tryConvertCallToCallFFI(DFG::Graph& graph, DFG::InsertionSet& insertionSet, unsigned nodeIndex, DFG::Node* node, JSFunction* function) +{ + if (!Options::useFFICallInDFG()) + return false; + + if (node->op() != DFG::Call) + return false; + + if (!is64Bit()) + return false; + + if (!function) + return false; + auto* ffiFunction = dynamicDowncast(function); + if (!ffiFunction) + return false; + + if (ffiFunction->isHostPathOnly()) + return false; + + Signature& signature = ffiFunction->signature(); + + if (node->numChildren() - 2 != signature.argumentCount()) + return false; + + if (!signature.invokeThunk()) + return false; + + if (!node->origin.exitOK) + return false; + unsigned checkIndex = nodeIndex; + + graph.m_parameterSlots = std::max(graph.m_parameterSlots, DFG::Graph::parameterSlotsForArgCount(signature.slotCount() + 1)); + + for (unsigned index = 0; index < signature.argumentCount(); ++index) { + Type type = signature.argumentType(index); + unsigned childIndex = 2 + index; + DFG::Edge argument = graph.varArgChild(node, childIndex); + DFG::Node* argumentNode = argument.node(); + switch (type) { + case Type::Char: + case Type::Int8: + case Type::Uint8: + case Type::Int16: + case Type::Uint16: + case Type::Int32: + case Type::Uint32: { + if (argumentNode->shouldSpeculateInt32()) { + insertionSet.insertCheck(checkIndex, node->origin, DFG::Edge(argumentNode, DFG::Int32Use)); + graph.varArgChild(node, childIndex) = DFG::Edge(argumentNode, DFG::KnownInt32Use); + } else + graph.varArgChild(node, childIndex) = DFG::Edge(argumentNode, DFG::UntypedUse); + break; + } + case Type::Bool: { + if (argumentNode->shouldSpeculateBoolean()) { + insertionSet.insertCheck(checkIndex, node->origin, DFG::Edge(argumentNode, DFG::BooleanUse)); + graph.varArgChild(node, childIndex) = DFG::Edge(argumentNode, DFG::KnownBooleanUse); + } else if (argumentNode->shouldSpeculateInt32()) { + insertionSet.insertCheck(checkIndex, node->origin, DFG::Edge(argumentNode, DFG::Int32Use)); + graph.varArgChild(node, childIndex) = DFG::Edge(argumentNode, DFG::KnownInt32Use); + } else + graph.varArgChild(node, childIndex) = DFG::Edge(argumentNode, DFG::UntypedUse); + break; + } + case Type::Float: + case Type::Double: { + DFG::UseKind useKind; + if (argumentNode->shouldSpeculateDoubleReal()) + useKind = DFG::RealNumberUse; + else if (argumentNode->shouldSpeculateNumber()) + useKind = DFG::NumberUse; + else { + graph.varArgChild(node, childIndex) = DFG::Edge(argumentNode, DFG::UntypedUse); + break; + } + DFG::Node* result = insertionSet.insertNode(checkIndex, SpecBytecodeDouble, DFG::DoubleRep, node->origin, DFG::Edge(argumentNode, useKind)); + graph.varArgChild(node, childIndex) = DFG::Edge(result, DFG::DoubleRepUse); + break; + } + case Type::Int64: + case Type::Uint64: + case Type::Int64Fast: + case Type::Uint64Fast: + case Type::Pointer: + case Type::CString: + case Type::Function: + case Type::Buffer: + case Type::BufferLength: + case Type::JSValue: { + graph.varArgChild(node, childIndex) = DFG::Edge(argumentNode, DFG::UntypedUse); + break; + } + case Type::RESERVED_WasNapiEnv: + case Type::Void: + RELEASE_ASSERT_NOT_REACHED(); + break; + } + } + + graph.varArgChild(node, 0) = DFG::Edge(graph.varArgChild(node, 0).node(), DFG::KnownCellUse); + + node->convertToCallFFI(graph.freeze(ffiFunction)); + return true; +} + +SpeculatedType speculatedResultTypeForCallFFI(DFG::Node* node) +{ + switch (node->ffiSignature().returnType()) { + case Type::Char: + case Type::Int8: + case Type::Uint8: + case Type::Int16: + case Type::Uint16: + case Type::Int32: + return SpecInt32Only; + case Type::Uint32: + return SpecBytecodeNumber; + case Type::Bool: + return SpecBoolean; + case Type::Float: + case Type::Double: + return SpecBytecodeDouble; + case Type::Void: + return SpecOther; + case Type::Int64: + case Type::Uint64: + return SpecBigInt; + case Type::Int64Fast: + case Type::Uint64Fast: + return SpecBytecodeNumber | SpecHeapBigInt; + case Type::CString: + return SpecString | SpecOther; + case Type::Pointer: + case Type::Function: + case Type::Buffer: + return SpecBytecodeNumber | SpecOther | SpecBigInt; + case Type::JSValue: + return SpecBytecodeTop; + case Type::RESERVED_WasNapiEnv: + case Type::BufferLength: + RELEASE_ASSERT_NOT_REACHED(); + return SpecBytecodeTop; + } + RELEASE_ASSERT_NOT_REACHED(); + return SpecBytecodeTop; +} + +} } // namespace JSC::FFI + +#endif // ENABLE(DFG_JIT) + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIDFG.h b/Source/JavaScriptCore/ffi/FFIDFG.h new file mode 100644 index 0000000000000..7b2163f90c5a4 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIDFG.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#if ENABLE(DFG_JIT) + +#include "SpeculatedType.h" + +namespace JSC { + +class JSFunction; + +namespace DFG { +class Graph; +class InsertionSet; +struct Node; +} // namespace DFG + +namespace FFI { + +bool tryConvertCallToCallFFI(DFG::Graph&, DFG::InsertionSet&, unsigned nodeIndex, DFG::Node*, JSFunction* callee); + +SpeculatedType speculatedResultTypeForCallFFI(DFG::Node*); + +} // namespace FFI + +} // namespace JSC + +#endif // ENABLE(DFG_JIT) + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIDFGCodegen.cpp b/Source/JavaScriptCore/ffi/FFIDFGCodegen.cpp new file mode 100644 index 0000000000000..0114414e5dc41 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIDFGCodegen.cpp @@ -0,0 +1,482 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "DFGSpeculativeJIT.h" + +#if ENABLE(DFG_JIT) && USE(JSVALUE64) + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +#include "DFGSlowPathGenerator.h" +#include "FFIContext.h" +#include "FFIConversions.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "JSArrayBufferView.h" +#include "JSCast.h" +#include "JSFFIFunction.h" +#include "JSGlobalObject.h" + +namespace JSC { namespace DFG { + +static bool ffiUntypedConversionMayUseStringArena(FFI::Type type) +{ + switch (type) { + case FFI::Type::CString: + return true; + default: + return false; + } +} + +void SpeculativeJIT::compileCallFFI(Node* node) +{ + JSFFIFunction* function = node->ffiFunction(); + FFI::Signature& signature = node->ffiSignature(); + void* target = function->target(); + JSGlobalObject* globalObject = function->globalObject(); + + FFI::FFIContext* ffiContext = &globalObject->ffiContext(); + + CodePtr invokeThunk = signature.invokeThunk(); + if (!invokeThunk) [[unlikely]] { + flushRegisters(); + callOperationWithoutExceptionCheck(operationThrowOutOfMemoryError, TrustedImmPtr(&vm())); + exceptionCheck(); + GPRTemporary result(this); + move(TrustedImm64(JSValue::encode(jsUndefined())), result.gpr()); + jsValueResult(result.gpr(), node); + return; + } + + const unsigned nativeArgumentCount = signature.argumentCount(); + const unsigned returnSlotIndex = nativeArgumentCount; + + DFG_ASSERT(m_graph, node, node->numChildren() == 2 + nativeArgumentCount, node->numChildren(), nativeArgumentCount); + DFG_ASSERT(m_graph, node, m_graph.m_parameterSlots * sizeof(Register) >= signature.slotBufferBytes(), m_graph.m_parameterSlots, signature.slotCount()); + + FrozenValue* frozenGlobalObject = m_graph.freeze(globalObject); + + auto slotAddressFor = [&](unsigned slotIndex) -> Address { + return Address(stackPointerRegister, static_cast(slotIndex * FFI::slotSize)); + }; + + bool needsArenaBracket = false; + for (unsigned i = 0; i < nativeArgumentCount; ++i) { + FFI::Type type = signature.argumentType(i); + Edge edge = m_graph.varArgChild(node, 2 + i); + if (edge.useKind() == UntypedUse && ffiUntypedConversionMayUseStringArena(type)) + needsArenaBracket = true; + } + + if (needsArenaBracket) { + flushRegisters(); + setupArguments(TrustedImmPtr(frozenGlobalObject)); + appendCall(operationFFIArenaEnter); + } + + auto emitArenaExitIfExceptionPending = [this, frozenGlobalObject] { + Jump noException = emitExceptionCheck(vm(), AssemblyHelpers::InvertedExceptionCheck); + setupArguments(TrustedImmPtr(frozenGlobalObject)); + appendCall(operationFFIArenaExit); + loadPtr(vm().addressOfException(), GPRInfo::returnValueGPR); + noException.link(this); + }; + + auto storeInt32AsIntegerType = [&](FFI::Type type, GPRReg valueGPR, GPRReg scratchGPR, Address slot) { + switch (type) { + case FFI::Type::Char: + case FFI::Type::Int8: + signExtend8To32(valueGPR, scratchGPR); + signExtend32ToPtr(scratchGPR, scratchGPR); + break; + case FFI::Type::Uint8: + zeroExtend8To32(valueGPR, scratchGPR); + zeroExtend32ToWord(scratchGPR, scratchGPR); + break; + case FFI::Type::Int16: + signExtend16To32(valueGPR, scratchGPR); + signExtend32ToPtr(scratchGPR, scratchGPR); + break; + case FFI::Type::Uint16: + zeroExtend16To32(valueGPR, scratchGPR); + zeroExtend32ToWord(scratchGPR, scratchGPR); + break; + case FFI::Type::Int32: + signExtend32ToPtr(valueGPR, scratchGPR); + break; + case FFI::Type::Uint32: + zeroExtend32ToWord(valueGPR, scratchGPR); + break; + case FFI::Type::Bool: + compare32(NotEqual, valueGPR, TrustedImm32(0), scratchGPR); + zeroExtend32ToWord(scratchGPR, scratchGPR); + break; + default: + DFG_CRASH(m_graph, node, "CallFFI: not an integer-class FFI type"); + return; + } + store64(scratchGPR, slot); + }; + + for (unsigned i = 0; i < nativeArgumentCount; ++i) { + FFI::Type type = signature.argumentType(i); + Address slotAddress = slotAddressFor(i); + + Edge& edge = m_graph.varArgChild(node, 2 + i); + + switch (edge.useKind()) { + case KnownInt32Use: { + SpeculateInt32Operand value(this, edge); + GPRTemporary scratch(this); + storeInt32AsIntegerType(type, value.gpr(), scratch.gpr(), slotAddress); + break; + } + + case KnownBooleanUse: { + DFG_ASSERT(m_graph, node, type == FFI::Type::Bool, static_cast(type)); + SpeculateBooleanOperand value(this, edge); + GPRTemporary scratch(this); + GPRReg scratchGPR = scratch.gpr(); + move(value.gpr(), scratchGPR); + and64(TrustedImm32(1), scratchGPR); + store64(scratchGPR, slotAddress); + break; + } + + case DoubleRepUse: { + SpeculateDoubleOperand value(this, edge); + FPRReg valueFPR = value.fpr(); + switch (type) { + case FFI::Type::Double: + storeDouble(valueFPR, slotAddress); + break; + case FFI::Type::Float: { + FPRTemporary floatScratch(this); + FPRReg floatFPR = floatScratch.fpr(); + convertDoubleToFloat(valueFPR, floatFPR); + store32(TrustedImm32(0), slotAddress.withOffset(4)); + storeFloat(floatFPR, slotAddress); + break; + } + default: + DFG_CRASH(m_graph, node, "CallFFI: DoubleRepUse edge for a non-floating-point FFI type"); + break; + } + break; + } + + case UntypedUse: { + JSValueOperand value(this, edge); + GPRTemporary scratch(this); + GPRTemporary slotAddressTemp(this); + FPRTemporary fpScratchTemp(this); + GPRReg valueGPR = value.gpr(); + GPRReg scratchGPR = scratch.gpr(); + GPRReg slotAddressGPR = slotAddressTemp.gpr(); + FPRReg scratchFPR = fpScratchTemp.fpr(); + + addPtr(TrustedImm32(static_cast(i * FFI::slotSize)), stackPointerRegister, slotAddressGPR); + + JumpList slowCases; + JumpList stored; + + auto storeTypedArrayViewVector = [&] { + slowCases.append(branchIfNotType(valueGPR, JSTypeRange { static_cast(FirstTypedArrayType), static_cast(LastTypedArrayType) })); + slowCases.append(branchTest8(NonZero, Address(valueGPR, JSArrayBufferView::offsetOfMode()), TrustedImm32(isResizableOrGrowableSharedMode))); + loadPtr(Address(valueGPR, JSArrayBufferView::offsetOfVector()), scratchGPR); + slowCases.append(branchTestPtr(Zero, scratchGPR)); + cageTypedArrayStorage(valueGPR, scratchGPR); + store64(scratchGPR, slotAddress); + }; + + switch (type) { + case FFI::Type::Char: + case FFI::Type::Int8: + case FFI::Type::Uint8: + case FFI::Type::Int16: + case FFI::Type::Uint16: + case FFI::Type::Int32: + case FFI::Type::Uint32: + slowCases.append(branchIfNotInt32(valueGPR)); + storeInt32AsIntegerType(type, valueGPR, scratchGPR, slotAddress); + break; + + case FFI::Type::Bool: { + Jump notInt32 = branchIfNotInt32(valueGPR); + compare32(NotEqual, valueGPR, TrustedImm32(0), scratchGPR); + zeroExtend32ToWord(scratchGPR, scratchGPR); + store64(scratchGPR, slotAddress); + stored.append(jump()); + + notInt32.link(this); + slowCases.append(branchIfNotBoolean(valueGPR, scratchGPR)); + move(valueGPR, scratchGPR); + and64(TrustedImm32(1), scratchGPR); + store64(scratchGPR, slotAddress); + break; + } + + case FFI::Type::Int64: + case FFI::Type::Uint64: + case FFI::Type::Int64Fast: + case FFI::Type::Uint64Fast: { + Jump notInt32 = branchIfNotInt32(valueGPR); + signExtend32ToPtr(valueGPR, scratchGPR); + store64(scratchGPR, slotAddress); + stored.append(jump()); + + notInt32.link(this); + slowCases.append(branchIfNotCell(valueGPR)); + slowCases.append(branchIfNotHeapBigInt(valueGPR)); + toBigInt64(valueGPR, scratchGPR); + store64(scratchGPR, slotAddress); + break; + } + + case FFI::Type::Double: + case FFI::Type::Float: { + slowCases.append(branchIfNotNumber(valueGPR)); + Jump notInt32 = branchIfNotInt32(valueGPR); + convertInt32ToDouble(valueGPR, scratchFPR); + Jump converted = jump(); + notInt32.link(this); + unboxDouble(valueGPR, scratchGPR, scratchFPR); + converted.link(this); + if (type == FFI::Type::Double) + storeDouble(scratchFPR, slotAddress); + else { + convertDoubleToFloat(scratchFPR, scratchFPR); + store32(TrustedImm32(0), slotAddress.withOffset(4)); + storeFloat(scratchFPR, slotAddress); + } + break; + } + + case FFI::Type::Pointer: + case FFI::Type::CString: + case FFI::Type::Function: { + Jump notInt32 = branchIfNotInt32(valueGPR); + signExtend32ToPtr(valueGPR, scratchGPR); + store64(scratchGPR, slotAddress); + stored.append(jump()); + + notInt32.link(this); + Jump notNumber = branchIfNotNumber(valueGPR); + unboxDouble(valueGPR, scratchGPR, scratchFPR); + truncateDoubleToInt64(scratchFPR, scratchGPR); + store64(scratchGPR, slotAddress); + stored.append(jump()); + + notNumber.link(this); + slowCases.append(branchIfNotCell(valueGPR)); + storeTypedArrayViewVector(); + break; + } + + case FFI::Type::Buffer: + slowCases.append(branchIfNotCell(valueGPR)); + storeTypedArrayViewVector(); + break; + + case FFI::Type::BufferLength: + slowCases.append(jump()); + break; + + case FFI::Type::JSValue: + store64(valueGPR, slotAddress); + break; + + case FFI::Type::Void: + case FFI::Type::RESERVED_WasNapiEnv: + DFG_CRASH(m_graph, node, "CallFFI: unexpected JS argument type"); + break; + } + + stored.link(this); + + if (!slowCases.empty()) { + Label doneLabel = label(); + Vector savePlans; + silentSpillAllRegistersImpl(false, savePlans, NoResult); + uint32_t typeTag = static_cast(type); + bool exitArenaOnException = needsArenaBracket; + addSlowPathGeneratorLambda([=, this, savePlans = WTF::move(savePlans), slowCases = WTF::move(slowCases)]() mutable { + slowCases.link(this); + silentSpill(savePlans); + setupArguments(TrustedImmPtr(frozenGlobalObject), TrustedImmPtr(ffiContext), TrustedImm32(static_cast(typeTag)), JSValueRegs(valueGPR), slotAddressGPR); + appendCall(operationFFIWriteSlot); + if (exitArenaOnException) + emitArenaExitIfExceptionPending(); + std::optional exceptionReg = tryHandleOrGetExceptionUnderSilentSpill(savePlans, NoResult); + silentFill(savePlans); + if (exceptionReg) + exceptionCheck(*exceptionReg); + jump().linkTo(doneLabel, this); + }); + } + break; + } + + default: + DFG_CRASH(m_graph, node, "CallFFI: unexpected use kind on an argument edge"); + break; + } + } + + flushRegisters(); + + emitStoreCodeOrigin(node->origin.semantic); + storePtr(GPRInfo::callFrameRegister, &vm().topCallFrame); + move(TrustedImmPtr(target), GPRInfo::argumentGPR0); + move(stackPointerRegister, GPRInfo::argumentGPR1); + move(TrustedImmPtr(invokeThunk.taggedPtr()), GPRInfo::nonArgGPR0); + call(GPRInfo::nonArgGPR0, OperationPtrTag); + + if (needsArenaBracket && signature.returnType() != FFI::Type::CString) { + setupArguments(TrustedImmPtr(frozenGlobalObject)); + appendCall(operationFFIArenaExit); + } else if (needsArenaBracket) + emitArenaExitIfExceptionPending(); + + exceptionCheck(); + + Address returnSlot = slotAddressFor(returnSlotIndex); + switch (signature.returnType()) { + case FFI::Type::Void: { + GPRTemporary result(this); + move(TrustedImm64(JSValue::encode(jsUndefined())), result.gpr()); + jsValueResult(result.gpr(), node); + break; + } + + case FFI::Type::Char: + case FFI::Type::Int8: + case FFI::Type::Uint8: + case FFI::Type::Int16: + case FFI::Type::Uint16: + case FFI::Type::Int32: { + GPRTemporary result(this); + load32(returnSlot, result.gpr()); + strictInt32Result(result.gpr(), node); + break; + } + + case FFI::Type::Uint32: { + GPRTemporary result(this); + GPRTemporary value(this); + FPRTemporary fpValue(this); + GPRReg resultGPR = result.gpr(); + GPRReg valueGPR = value.gpr(); + FPRReg valueFPR = fpValue.fpr(); + load64(returnSlot, valueGPR); + Jump doesNotFitInt32 = branch32(LessThan, valueGPR, TrustedImm32(0)); + boxInt32(valueGPR, JSValueRegs(resultGPR)); + Jump done = jump(); + doesNotFitInt32.link(this); + convertUInt32ToDouble(valueGPR, valueFPR); + boxDouble(valueFPR, resultGPR); + done.link(this); + jsValueResult(resultGPR, node); + break; + } + + case FFI::Type::Bool: { + GPRTemporary result(this); + GPRReg resultGPR = result.gpr(); + load32(returnSlot, resultGPR); +#if ASSERT_ENABLED + Jump canonical = branch32(BelowOrEqual, resultGPR, TrustedImm32(1)); + breakpoint(); + canonical.link(this); +#endif + or32(TrustedImm32(JSValue::ValueFalse), resultGPR); + jsValueResult(resultGPR, node, DataFormatJSBoolean); + break; + } + + case FFI::Type::Double: + case FFI::Type::Float: { + FPRTemporary fpValue(this); + GPRTemporary result(this); + FPRReg valueFPR = fpValue.fpr(); + GPRReg resultGPR = result.gpr(); + if (signature.returnType() == FFI::Type::Double) + loadDouble(returnSlot, valueFPR); + else { + loadFloat(returnSlot, valueFPR); + convertFloatToDouble(valueFPR, valueFPR); + } + purifyNaN(valueFPR, valueFPR); + boxDouble(valueFPR, resultGPR); + jsValueResult(resultGPR, node); + break; + } + + case FFI::Type::JSValue: { + GPRTemporary result(this); + load64(returnSlot, result.gpr()); + jsValueResult(result.gpr(), node); + break; + } + + case FFI::Type::Int64: + case FFI::Type::Uint64: + case FFI::Type::Int64Fast: + case FFI::Type::Uint64Fast: + case FFI::Type::Pointer: + case FFI::Type::CString: + case FFI::Type::Function: + case FFI::Type::Buffer: { + GPRTemporary slotValue(this); + GPRTemporary result(this); + GPRReg slotValueGPR = slotValue.gpr(); + GPRReg resultGPR = result.gpr(); + load64(returnSlot, slotValueGPR); + callOperation(operationFFIBoxSlot, resultGPR, TrustedImmPtr(frozenGlobalObject), TrustedImm32(static_cast(static_cast(signature.returnType()))), slotValueGPR, TrustedImm32(needsArenaBracket && signature.returnType() == FFI::Type::CString ? 1 : 0)); + jsValueResult(resultGPR, node); + break; + } + + case FFI::Type::RESERVED_WasNapiEnv: + case FFI::Type::BufferLength: + DFG_CRASH(m_graph, node, "CallFFI: the reserved tag / buffer_length is never a return type"); + break; + } + + FFI::g_ffiCompileCounts.dfgCallFFI++; +} + +} } // namespace JSC::DFG + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +#endif // ENABLE(DFG_JIT) && USE(JSVALUE64) + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIICStub.cpp b/Source/JavaScriptCore/ffi/FFIICStub.cpp new file mode 100644 index 0000000000000..b446cb1394419 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIICStub.cpp @@ -0,0 +1,434 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFIICStub.h" + +#if USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) + +#include "CCallHelpers.h" +#include "CallFrame.h" +#include "FFICallHost.h" +#include "FFIContext.h" +#include "FFIConversions.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "FPRInfo.h" +#include "GPRInfo.h" +#include "JITCode.h" +#include "JITOperations.h" +#include "JSArrayBufferView.h" +#include "JSBigInt.h" +#include "JSCJSValueInlines.h" +#include "JSCPtrTag.h" +#include "JSGlobalObject.h" +#include "JSType.h" +#include "LinkBuffer.h" +#include "Options.h" +#include "StackAlignment.h" +#include "ThunkGenerator.h" +#include "VirtualRegister.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace JSC { namespace FFI { + +#if USE(JSVALUE64) && !ENABLE(JIT_CAGE) + +namespace { + +constexpr GPRReg valueGPR = GPRInfo::regT0; // rax / x0: the JSValue being converted; the boxed return value. +constexpr GPRReg scratchGPR = GPRInfo::regT2; // rdx / x2 +constexpr GPRReg scratch2GPR = GPRInfo::regT3; // rcx / x3 +constexpr GPRReg scratch3GPR = GPRInfo::regT4; // r8 / x4 +constexpr GPRReg callTargetGPR = GPRInfo::nonArgGPR0; // r10 / x8: volatile, not an argument register. +constexpr FPRReg valueFPR = FPRInfo::fpRegT0; // xmm0 / q0 + +static_assert(valueGPR != callTargetGPR); +static_assert(scratchGPR != callTargetGPR && scratch2GPR != callTargetGPR && scratch3GPR != callTargetGPR); +static_assert(valueGPR != scratchGPR && valueGPR != scratch2GPR && valueGPR != scratch3GPR); +static_assert(scratchGPR != scratch2GPR && scratchGPR != scratch3GPR && scratch2GPR != scratch3GPR); +static_assert(callTargetGPR != GPRInfo::argumentGPR0 && callTargetGPR != GPRInfo::argumentGPR1 && callTargetGPR != GPRInfo::argumentGPR2); +static_assert(callTargetGPR != GPRInfo::returnValueGPR); + +constexpr int32_t numberTagSaveOffset = -static_cast(sizeof(CPURegister)); +constexpr int32_t notCellMaskSaveOffset = -static_cast(2 * sizeof(CPURegister)); +constexpr size_t tagSaveAreaBytes = 2 * sizeof(CPURegister); // 16, keeps the frame 16-byte aligned. +static_assert(!(tagSaveAreaBytes % stackAlignmentBytes())); +static_assert(!stackAdjustmentForAlignment()); + +void emitSaveTagRegisters(CCallHelpers& jit) +{ + jit.store64(GPRInfo::numberTagRegister, CCallHelpers::Address(GPRInfo::callFrameRegister, numberTagSaveOffset)); + jit.store64(GPRInfo::notCellMaskRegister, CCallHelpers::Address(GPRInfo::callFrameRegister, notCellMaskSaveOffset)); + jit.emitMaterializeTagCheckRegisters(); +} + +void emitRestoreTagRegisters(CCallHelpers& jit) +{ + jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, numberTagSaveOffset), GPRInfo::numberTagRegister); + jit.load64(CCallHelpers::Address(GPRInfo::callFrameRegister, notCellMaskSaveOffset), GPRInfo::notCellMaskRegister); +} + +void emitConvertArgument(CCallHelpers& jit, Type type, CCallHelpers::Address argument, CCallHelpers::Address slot, CCallHelpers::JumpList& slowPath) +{ + switch (type) { + case Type::Char: + case Type::Int8: + jit.load64(argument, valueGPR); + slowPath.append(jit.branchIfNotInt32(valueGPR)); + jit.signExtend8To32(valueGPR, valueGPR); + jit.signExtend32ToPtr(valueGPR, valueGPR); + jit.store64(valueGPR, slot); + return; + + case Type::Uint8: + jit.load64(argument, valueGPR); + slowPath.append(jit.branchIfNotInt32(valueGPR)); + jit.zeroExtend8To32(valueGPR, valueGPR); // 32-bit result clears bits [63:32] on both targets. + jit.store64(valueGPR, slot); + return; + + case Type::Int16: + jit.load64(argument, valueGPR); + slowPath.append(jit.branchIfNotInt32(valueGPR)); + jit.signExtend16To32(valueGPR, valueGPR); + jit.signExtend32ToPtr(valueGPR, valueGPR); + jit.store64(valueGPR, slot); + return; + + case Type::Uint16: + jit.load64(argument, valueGPR); + slowPath.append(jit.branchIfNotInt32(valueGPR)); + jit.zeroExtend16To32(valueGPR, valueGPR); + jit.store64(valueGPR, slot); + return; + + case Type::Int32: + jit.load64(argument, valueGPR); + slowPath.append(jit.branchIfNotInt32(valueGPR)); + jit.signExtend32ToPtr(valueGPR, valueGPR); + jit.store64(valueGPR, slot); + return; + + case Type::Uint32: + jit.load64(argument, valueGPR); + slowPath.append(jit.branchIfNotInt32(valueGPR)); + jit.zeroExtend32ToWord(valueGPR, valueGPR); + jit.store64(valueGPR, slot); + return; + + case Type::Bool: { + jit.load64(argument, valueGPR); + auto notInt32 = jit.branchIfNotInt32(valueGPR); + jit.compare32(CCallHelpers::NotEqual, valueGPR, CCallHelpers::TrustedImm32(0), scratchGPR); + jit.store64(scratchGPR, slot); + auto done = jit.jump(); + notInt32.link(&jit); + slowPath.append(jit.branchIfNotBoolean(valueGPR, scratchGPR)); + jit.and32(CCallHelpers::TrustedImm32(1), valueGPR, scratchGPR); + jit.store64(scratchGPR, slot); + done.link(&jit); + return; + } + + case Type::Int64: + case Type::Uint64: + case Type::Int64Fast: + case Type::Uint64Fast: { + jit.load64(argument, valueGPR); + auto notInt32 = jit.branchIfNotInt32(valueGPR); + jit.signExtend32ToPtr(valueGPR, valueGPR); // int32 -> sign-extend (then reinterpret for the unsigned types). + jit.store64(valueGPR, slot); + auto done = jit.jump(); + notInt32.link(&jit); + slowPath.append(jit.branchIfNotCell(valueGPR)); + slowPath.append(jit.branchIfNotHeapBigInt(valueGPR)); + slowPath.append(jit.branch32(CCallHelpers::Above, CCallHelpers::Address(valueGPR, JSBigInt::offsetOfLength()), CCallHelpers::TrustedImm32(1))); + jit.toBigInt64(valueGPR, scratchGPR); + jit.store64(scratchGPR, slot); + done.link(&jit); + return; + } + + case Type::Double: + case Type::Float: { + jit.load64(argument, valueGPR); + slowPath.append(jit.branchIfNotNumber(valueGPR)); + auto isInt32 = jit.branchIfInt32(valueGPR); + jit.unboxDouble(valueGPR, scratchGPR, valueFPR); + auto haveDouble = jit.jump(); + isInt32.link(&jit); + jit.convertInt32ToDouble(valueGPR, valueFPR); + haveDouble.link(&jit); + if (type == Type::Float) { + jit.convertDoubleToFloat(valueFPR, valueFPR); + jit.store32(CCallHelpers::TrustedImm32(0), slot.withOffset(4)); + jit.storeFloat(valueFPR, slot); + } else + jit.storeDouble(valueFPR, slot); + return; + } + + case Type::Pointer: + case Type::CString: + case Type::Function: + case Type::Buffer: { + jit.load64(argument, valueGPR); + CCallHelpers::JumpList stored; + if (type != Type::Buffer) { + auto notInt32 = jit.branchIfNotInt32(valueGPR); + jit.signExtend32ToPtr(valueGPR, valueGPR); + jit.store64(valueGPR, slot); + stored.append(jit.jump()); + notInt32.link(&jit); + auto notNumber = jit.branchIfNotNumber(valueGPR); + jit.unboxDouble(valueGPR, scratchGPR, valueFPR); + jit.truncateDoubleToInt64(valueFPR, scratchGPR); // by definition identical to FFI::doubleToInt64. + jit.store64(scratchGPR, slot); + stored.append(jit.jump()); + notNumber.link(&jit); + } + slowPath.append(jit.branchIfNotCell(valueGPR)); + slowPath.append(jit.branchIfNotType(valueGPR, JSTypeRange { static_cast(FirstTypedArrayType), static_cast(LastTypedArrayType) })); + jit.loadPtr(CCallHelpers::Address(valueGPR, JSArrayBufferView::offsetOfVector()), scratchGPR); + slowPath.append(jit.branchTestPtr(CCallHelpers::Zero, scratchGPR)); // null / detached vector: keep those semantics in C++. + jit.load64(CCallHelpers::Address(valueGPR, JSArrayBufferView::offsetOfLength()), scratch2GPR); + jit.cageConditionally(Gigacage::Primitive, scratchGPR, scratch2GPR, scratch3GPR); + jit.store64(scratchGPR, slot); + stored.link(&jit); + return; + } + + case Type::BufferLength: + slowPath.append(jit.jump()); + return; + + case Type::JSValue: + jit.load64(argument, valueGPR); + jit.store64(valueGPR, slot); + return; + + case Type::RESERVED_WasNapiEnv: + case Type::Void: + RELEASE_ASSERT_NOT_REACHED(); + return; + } + RELEASE_ASSERT_NOT_REACHED(); +} + +void emitBoxReturnValue(CCallHelpers& jit, VM& vm, JSGlobalObject* globalObject, Type type, CCallHelpers::Address returnSlot, CCallHelpers::JumpList& exceptionChecks) +{ + JSValueRegs resultRegs { GPRInfo::returnValueGPR }; + switch (type) { + case Type::Void: + jit.moveTrustedValue(jsUndefined(), resultRegs); + return; + + case Type::Char: + case Type::Int8: + case Type::Uint8: + case Type::Int16: + case Type::Uint16: + case Type::Int32: + jit.load32(returnSlot, GPRInfo::returnValueGPR); + jit.boxInt32(GPRInfo::returnValueGPR, resultRegs); + return; + + case Type::Uint32: { + jit.load64(returnSlot, GPRInfo::returnValueGPR); + auto fitsInInt32 = jit.branch32(CCallHelpers::GreaterThanOrEqual, GPRInfo::returnValueGPR, CCallHelpers::TrustedImm32(0)); + jit.convertInt64ToDouble(GPRInfo::returnValueGPR, valueFPR); + jit.boxDouble(valueFPR, resultRegs); + auto done = jit.jump(); + fitsInInt32.link(&jit); + jit.boxInt32(GPRInfo::returnValueGPR, resultRegs); + done.link(&jit); + return; + } + + case Type::Bool: + jit.load32(returnSlot, GPRInfo::returnValueGPR); + jit.or32(CCallHelpers::TrustedImm32(JSValue::ValueFalse), GPRInfo::returnValueGPR); + return; + + case Type::Double: + jit.loadDouble(returnSlot, valueFPR); + jit.purifyNaN(valueFPR, valueFPR); // a native NaN payload must never forge a boxed JSValue. + jit.boxDouble(valueFPR, resultRegs); + return; + + case Type::Float: + jit.loadFloat(returnSlot, valueFPR); + jit.convertFloatToDouble(valueFPR, valueFPR); + jit.purifyNaN(valueFPR, valueFPR); + jit.boxDouble(valueFPR, resultRegs); + return; + + case Type::JSValue: + jit.load64(returnSlot, GPRInfo::returnValueGPR); + return; + + case Type::Int64: + case Type::Uint64: + case Type::Int64Fast: + case Type::Uint64Fast: + case Type::Pointer: + case Type::CString: + case Type::Function: + jit.load64(returnSlot, GPRInfo::argumentGPR2); + jit.move(CCallHelpers::TrustedImm32(static_cast(type)), GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImmPtr(globalObject), GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImm32(0), GPRInfo::argumentGPR3); + jit.move(CCallHelpers::TrustedImmPtr(tagCFunction(operationFFIBoxSlot)), callTargetGPR); + jit.call(callTargetGPR, OperationPtrTag); + exceptionChecks.append(jit.emitExceptionCheck(vm)); + return; + + case Type::Buffer: + case Type::BufferLength: + case Type::RESERVED_WasNapiEnv: + RELEASE_ASSERT_NOT_REACHED(); + return; + } + RELEASE_ASSERT_NOT_REACHED(); +} + +} // anonymous namespace + +RefPtr generateICStubCode(VM& vm, JSGlobalObject* globalObject, Signature& signature, void* target) +{ + ASSERT(!isCompilationThread()); + + if (!Options::useFFIICStub() || !Options::useJIT() || Options::forceICFailure()) + return nullptr; + + CodePtr invokeThunk = signature.invokeThunk(); + if (!invokeThunk) + return nullptr; + + const unsigned argumentCount = signature.argumentCount(); + const Type returnType = signature.returnType(); + + for (unsigned i = 0; i < argumentCount; ++i) { + if (signature.argumentType(i) == Type::BufferLength) + return nullptr; + } + + const size_t slotBufferBytes = signature.slotBufferBytes(); + const size_t slotAreaBytes = WTF::roundUpToMultipleOf(slotBufferBytes); + const size_t frameBytes = tagSaveAreaBytes + slotAreaBytes; + RELEASE_ASSERT(!(frameBytes % stackAlignmentBytes())); + RELEASE_ASSERT(frameBytes <= static_cast(std::numeric_limits::max())); + const int32_t slotsOffsetFromFP = -static_cast(frameBytes); + auto slotAddress = [&](unsigned slotIndex) { + return CCallHelpers::Address(GPRInfo::callFrameRegister, slotsOffsetFromFP + static_cast(argumentSlotOffset(slotIndex))); + }; + const CCallHelpers::Address returnSlotAddress = slotAddress(argumentCount); + + CCallHelpers jit; + JIT_COMMENT(jit, "FFI IC stub for ", signature.toString()); + + CCallHelpers::JumpList slowPath; + CCallHelpers::JumpList exceptionChecks; + + jit.emitFunctionPrologue(); + jit.subPtr(CCallHelpers::TrustedImm32(static_cast(frameBytes)), CCallHelpers::stackPointerRegister); + emitSaveTagRegisters(jit); + jit.storePtr(CCallHelpers::TrustedImmPtr(nullptr), CCallHelpers::addressFor(CallFrameSlot::codeBlock)); + + jit.storePtr(GPRInfo::callFrameRegister, &vm.topCallFrame); + + if (argumentCount) { + JIT_COMMENT(jit, "arity check"); + slowPath.append(jit.branch32(CCallHelpers::Below, CCallHelpers::payloadFor(CallFrameSlot::argumentCountIncludingThis), CCallHelpers::TrustedImm32(argumentCount + 1))); + } + + for (unsigned i = 0; i < argumentCount; ++i) { + Type type = signature.argumentType(i); + JIT_COMMENT(jit, "argument ", i, " : ", name(type)); + CCallHelpers::Address argument = CCallHelpers::addressFor(virtualRegisterForArgumentIncludingThis(static_cast(i) + 1)); + emitConvertArgument(jit, type, argument, slotAddress(i), slowPath); + } + + JIT_COMMENT(jit, "call invoke thunk"); + jit.addPtr(CCallHelpers::TrustedImm32(slotsOffsetFromFP), GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImmPtr(target), GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImmPtr(invokeThunk.taggedPtr()), callTargetGPR); + jit.call(callTargetGPR, OperationPtrTag); + + exceptionChecks.append(jit.emitExceptionCheck(vm)); + + JIT_COMMENT(jit, "box return value : ", name(returnType)); + emitBoxReturnValue(jit, vm, globalObject, returnType, returnSlotAddress, exceptionChecks); + + emitRestoreTagRegisters(jit); + jit.emitFunctionEpilogue(); + jit.ret(); + + slowPath.link(&jit); + JIT_COMMENT(jit, "slow path"); + jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); + jit.move(CCallHelpers::TrustedImmPtr(globalObject), GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImmPtr(tagCFunction(operationFFICallSlowPath)), callTargetGPR); + jit.call(callTargetGPR, OperationPtrTag); + exceptionChecks.append(jit.emitExceptionCheck(vm)); + emitRestoreTagRegisters(jit); + jit.emitFunctionEpilogue(); + jit.ret(); + + exceptionChecks.link(&jit); + JIT_COMMENT(jit, "exception handler"); + emitRestoreTagRegisters(jit); + jit.copyCalleeSavesToEntryFrameCalleeSavesBuffer(vm.topEntryFrame, GPRInfo::argumentGPR0); + jit.storePtr(GPRInfo::callFrameRegister, &vm.topCallFrame); + jit.move(CCallHelpers::TrustedImmPtr(&vm), GPRInfo::argumentGPR0); + jit.move(CCallHelpers::TrustedImmPtr(tagCFunction(operationVMHandleException)), GPRInfo::regT3); + jit.call(GPRInfo::regT3, OperationPtrTag); + jit.jumpToExceptionHandler(vm); + + LinkBuffer linkBuffer(jit, GLOBAL_THUNK_ID, LinkBuffer::Profile::Thunk, JITCompilationCanFail); + if (linkBuffer.didFailToAllocate()) [[unlikely]] + return nullptr; + linkBuffer.setIsThunk(); + auto codeRef = FINALIZE_CODE_IF(Options::dumpDisassembly() || Options::dumpFFIDisassembly(), linkBuffer, JSEntryPtrTag, "FFI ic"_s, "FFI ic %s", signature.toString().utf8().data()); + + g_ffiCompileCounts.icStub++; + dataLogLnIf(Options::verboseFFI(), "FFI: generated IC stub ", signature.toString(), " target ", RawPointer(target), " code ", RawPointer(codeRef.code().taggedPtr())); + + return adoptRef(new DirectJITCode(codeRef, codeRef.code(), JITType::HostCallThunk, NoIntrinsic)); +} + +#endif // USE(JSVALUE64) && !ENABLE(JIT_CAGE) + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) diff --git a/Source/JavaScriptCore/ffi/FFIICStub.h b/Source/JavaScriptCore/ffi/FFIICStub.h new file mode 100644 index 0000000000000..ad3194c1465f5 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIICStub.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) + +#include +#include + +namespace JSC { + +class JITCode; +class JSGlobalObject; +class VM; + +namespace FFI { + +class Signature; + +RefPtr generateICStubCode(VM&, JSGlobalObject*, Signature&, void* target); + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) diff --git a/Source/JavaScriptCore/ffi/FFIInvokeThunk.cpp b/Source/JavaScriptCore/ffi/FFIInvokeThunk.cpp new file mode 100644 index 0000000000000..b64627b26c184 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIInvokeThunk.cpp @@ -0,0 +1,267 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFIInvokeThunk.h" + +#if USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) + +#include "CCallHelpers.h" +#include "FFICallingConvention.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "FPRInfo.h" +#include "GPRInfo.h" +#include "LinkBuffer.h" +#include "Options.h" +#include "ThunkGenerator.h" +#include + +namespace JSC { namespace FFI { + +#if !ENABLE(JIT_CAGE) && (CPU(X86_64) || CPU(ARM64)) + +namespace { + +#if CPU(X86_64) +static constexpr GPRReg shuffleGPR = X86Registers::eax; +#elif CPU(ARM64) +static constexpr GPRReg shuffleGPR = ARM64Registers::x8; +#endif + +inline CCallHelpers::Address slotAddress(GPRReg slotsReg, unsigned slotIndex) +{ + return CCallHelpers::Address(slotsReg, static_cast(slotIndex * slotSize)); +} + +void emitStackArgument(CCallHelpers& jit, const CallLayout& layout, const ArgLocation& loc, GPRReg slotsReg, unsigned slotIndex) +{ + ASSERT(loc.kind == ArgLocation::Kind::Stack); + CCallHelpers::Address src = slotAddress(slotsReg, slotIndex); + CCallHelpers::Address dst(MacroAssembler::stackPointerRegister, static_cast(loc.stackOffset)); + + jit.load64(src, shuffleGPR); + if (layout.packing == StackPacking::Natural) { + switch (nativeSizeInBytes(loc.type)) { + case 1: + jit.store8(shuffleGPR, dst); + break; + case 2: + jit.store16(shuffleGPR, dst); + break; + case 4: + jit.store32(shuffleGPR, dst); + break; + case 8: + jit.store64(shuffleGPR, dst); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + } + return; + } + + ASSERT(!(loc.stackOffset % 8)); + ASSERT(loc.stackOffset + 8 <= layout.stackBytes); + jit.store64(shuffleGPR, dst); +} + +void emitReturnNormalization(CCallHelpers& jit, const Signature& signature, GPRReg slotsReg) +{ + Type returnType = signature.returnType(); + unsigned returnSlotIndex = signature.argumentCount(); + CCallHelpers::Address returnSlot = slotAddress(slotsReg, returnSlotIndex); + + switch (argClass(returnType)) { + case ArgClass::Void: + break; + + case ArgClass::Float: { + jit.storeFloat(FPRInfo::returnValueFPR, returnSlot); + jit.store32(CCallHelpers::TrustedImm32(0), returnSlot.withOffset(4)); + break; + } + + case ArgClass::Double: + jit.storeDouble(FPRInfo::returnValueFPR, returnSlot); + break; + + case ArgClass::Int: { + GPRReg returnGPR = GPRInfo::returnValueGPR; + if (returnType == Type::Bool) { + jit.and32(CCallHelpers::TrustedImm32(0xff), returnGPR); + jit.compare32(CCallHelpers::NotEqual, returnGPR, CCallHelpers::TrustedImm32(0), returnGPR); + } else { + switch (nativeSizeInBytes(returnType)) { + case 1: + if (isSigned(returnType)) { + jit.signExtend8To32(returnGPR, returnGPR); + jit.signExtend32To64(returnGPR, returnGPR); + } else { + jit.zeroExtend8To32(returnGPR, returnGPR); + jit.zeroExtend32ToWord(returnGPR, returnGPR); + } + break; + case 2: + if (isSigned(returnType)) { + jit.signExtend16To32(returnGPR, returnGPR); + jit.signExtend32To64(returnGPR, returnGPR); + } else { + jit.zeroExtend16To32(returnGPR, returnGPR); + jit.zeroExtend32ToWord(returnGPR, returnGPR); + } + break; + case 4: + if (isSigned(returnType)) + jit.signExtend32To64(returnGPR, returnGPR); + else + jit.zeroExtend32ToWord(returnGPR, returnGPR); + break; + case 8: + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + } + } + jit.store64(returnGPR, returnSlot); + break; + } + } +} + +} // anonymous namespace + +MacroAssemblerCodeRef generateInvokeThunk(const Signature& signature) +{ + const NativeCC cc = hostNativeCC(); + CallLayout layout = computeCallLayout(cc, signature, Direction::Outgoing); + ASSERT(layout.arguments.size() == signature.argumentCount()); + RELEASE_ASSERT(!(layout.stackBytes % 16)); + + auto [slotsReg, targetReg] = scratchGPRsForInvoke(cc); + ASSERT(slotsReg != shuffleGPR && targetReg != shuffleGPR); + ASSERT(slotsReg != GPRInfo::returnValueGPR && targetReg != GPRInfo::returnValueGPR); + + std::span integerArgumentRegs = integerArgumentRegisters(cc); + std::span floatArgumentRegs = floatArgumentRegisters(cc); + + CCallHelpers jit(nullptr); + + jit.emitFunctionPrologue(); +#if CPU(X86_64) + ASSERT(slotsReg == X86Registers::ebx); + jit.pushToSave(X86Registers::ebx); + jit.subPtr(CCallHelpers::TrustedImm32(8), MacroAssembler::stackPointerRegister); +#elif CPU(ARM64) + ASSERT(slotsReg == ARM64Registers::x19); + jit.pushPair(ARM64Registers::x19, ARM64Registers::x20); +#endif + + if (layout.stackBytes) + jit.subPtr(CCallHelpers::TrustedImm32(layout.stackBytes), MacroAssembler::stackPointerRegister); + + jit.move(GPRInfo::argumentGPR1, slotsReg); + jit.move(GPRInfo::argumentGPR0, targetReg); + + for (unsigned i = 0; i < layout.arguments.size(); ++i) { + const ArgLocation& loc = layout.arguments[i]; + if (loc.kind == ArgLocation::Kind::Stack) + emitStackArgument(jit, layout, loc, slotsReg, i); + } + for (unsigned i = 0; i < layout.arguments.size(); ++i) { + const ArgLocation& loc = layout.arguments[i]; + if (loc.kind != ArgLocation::Kind::FPR) + continue; + FPRReg fpr = floatArgumentRegs[loc.regIndex]; + switch (argClass(loc.type)) { + case ArgClass::Float: + jit.loadFloat(slotAddress(slotsReg, i), fpr); + break; + case ArgClass::Double: + jit.loadDouble(slotAddress(slotsReg, i), fpr); + break; + default: + RELEASE_ASSERT_NOT_REACHED(); + } + } + for (unsigned i = 0; i < layout.arguments.size(); ++i) { + const ArgLocation& loc = layout.arguments[i]; + if (loc.kind != ArgLocation::Kind::GPR) + continue; + ASSERT(argClass(loc.type) == ArgClass::Int); + GPRReg gpr = integerArgumentRegs[loc.regIndex]; + ASSERT(gpr != slotsReg && gpr != targetReg); + jit.load64(slotAddress(slotsReg, i), gpr); + } + +#if CPU(X86_64) + if (cc == NativeCC::SysV64) { + unsigned floatArgumentRegisterUseCount = 0; + for (const ArgLocation& loc : layout.arguments) { + if (loc.kind == ArgLocation::Kind::FPR) + ++floatArgumentRegisterUseCount; + } + ASSERT(floatArgumentRegisterUseCount <= floatArgumentRegs.size()); + jit.move(CCallHelpers::TrustedImm32(floatArgumentRegisterUseCount), X86Registers::eax); + } +#endif + + jit.call(targetReg, CFunctionPtrTag); + + emitReturnNormalization(jit, signature, slotsReg); + + if (layout.stackBytes) + jit.addPtr(CCallHelpers::TrustedImm32(layout.stackBytes), MacroAssembler::stackPointerRegister); +#if CPU(X86_64) + jit.addPtr(CCallHelpers::TrustedImm32(8), MacroAssembler::stackPointerRegister); + jit.popToRestore(X86Registers::ebx); +#elif CPU(ARM64) + jit.popPair(ARM64Registers::x19, ARM64Registers::x20); +#endif + jit.emitFunctionEpilogue(); + jit.ret(); + + LinkBuffer linkBuffer(jit, GLOBAL_THUNK_ID, LinkBuffer::Profile::Thunk, JITCompilationCanFail); + if (linkBuffer.didFailToAllocate()) [[unlikely]] + return { }; + + linkBuffer.setIsThunk(); + auto code = FINALIZE_CODE_IF(Options::dumpDisassembly() || Options::dumpFFIDisassembly(), linkBuffer, JITThunkPtrTag, "FFI invoke"_s, "FFI invoke %s", signature.toString().utf8().data()); + dataLogLnIf(Options::verboseFFI(), "[FFI] generated invoke thunk for ", signature.toString()); + return code; +} + +#else // ENABLE(JIT_CAGE) || !(CPU(X86_64) || CPU(ARM64)) + +MacroAssemblerCodeRef generateInvokeThunk(const Signature&) +{ + return { }; +} + +#endif + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) diff --git a/Source/JavaScriptCore/ffi/FFIInvokeThunk.h b/Source/JavaScriptCore/ffi/FFIInvokeThunk.h new file mode 100644 index 0000000000000..20456942b370e --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIInvokeThunk.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) + +#include "JSCPtrTag.h" +#include "JSExportMacros.h" +#include "MacroAssemblerCodeRef.h" + +namespace JSC { namespace FFI { + +class Signature; + +#if ENABLE(JIT) + +JS_EXPORT_PRIVATE MacroAssemblerCodeRef generateInvokeThunk(const Signature&); + +#endif // ENABLE(JIT) + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFISignature.cpp b/Source/JavaScriptCore/ffi/FFISignature.cpp new file mode 100644 index 0000000000000..02c49428aa43d --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFISignature.cpp @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFISignature.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "Options.h" +#include +#include +#include +#include +#include +#include +#include + +#if ENABLE(JIT) && USE(JSVALUE64) && !ENABLE(JIT_CAGE) +#include "FFIInvokeThunk.h" +#endif + +namespace JSC { namespace FFI { + +WTF_MAKE_TZONE_ALLOCATED_IMPL(Signature); +WTF_MAKE_TZONE_ALLOCATED_IMPL(SignatureRegistry); + +CompileCounts g_ffiCompileCounts; + +Signature::Signature(std::span arguments, Type returnType) + : m_arguments(arguments) + , m_returnType(returnType) +{ + m_hash = computeHash(arguments, returnType); +} + +RefPtr Signature::tryCreate(std::span arguments, Type returnType) +{ + if (arguments.size() > maxArguments) + return nullptr; + if (!isValidReturnType(returnType)) + return nullptr; + for (Type type : arguments) { + if (!isValidArgumentType(type)) + return nullptr; + } + return SignatureRegistry::singleton().intern(arguments, returnType); +} + +unsigned Signature::computeHash(std::span arguments, Type returnType) +{ + unsigned hash = WTF::IntHash::hash(static_cast(returnType)); + hash = WTF::pairIntHash(hash, static_cast(arguments.size())); + for (Type type : arguments) + hash = WTF::pairIntHash(hash, static_cast(type)); + return hash; +} + +bool Signature::structurallyEquals(std::span arguments, Type returnType, const Signature& signature) +{ + if (returnType != signature.returnType()) + return false; + if (arguments.size() != signature.argumentCount()) + return false; + for (unsigned i = 0; i < arguments.size(); ++i) { + if (arguments[i] != signature.argumentType(i)) + return false; + } + return true; +} + +bool Signature::operator==(const Signature& other) const +{ + if (this == &other) + return true; + return structurallyEquals(arguments(), returnType(), other); +} + +String Signature::toString() const +{ + StringBuilder builder; + builder.append(name(m_returnType)); + builder.append('('); + for (unsigned i = 0; i < argumentCount(); ++i) { + if (i) + builder.append(','); + builder.append(name(argumentType(i))); + } + builder.append(')'); + return builder.toString(); +} + +CodePtr Signature::invokeThunk() +{ + if (auto* published = m_publishedInvokeThunk.load(std::memory_order_acquire)) [[likely]] + return CodePtr::fromTaggedPtr(published); + + Locker locker { m_codeLock }; +#if ENABLE(JIT) && USE(JSVALUE64) && !ENABLE(JIT_CAGE) + if (!m_invokeThunkCode) { + m_invokeThunkCode = generateInvokeThunk(*this); + if (m_invokeThunkCode) { + dataLogLnIf(Options::verboseFFI(), "FFI: generated invoke thunk for ", toString(), " at ", RawPointer(m_invokeThunkCode.code().taggedPtr())); + m_publishedInvokeThunk.store(m_invokeThunkCode.code().taggedPtr(), std::memory_order_release); + } + } +#endif + return m_invokeThunkCode.code(); +} + +SignatureRegistry& SignatureRegistry::singleton() +{ + static LazyNeverDestroyed registry; + static std::once_flag onceKey; + std::call_once(onceKey, [&] { + registry.construct(); + }); + return registry; +} + +struct SignatureShape { + std::span arguments; + Type returnType; + unsigned hash; +}; + +struct SignatureShapeTranslator { + static unsigned hash(const SignatureShape& shape) { return shape.hash; } + static bool equal(const Ref& entry, const SignatureShape& shape) + { + return Signature::structurallyEquals(shape.arguments, shape.returnType, entry.get()); + } +}; + +Ref SignatureRegistry::intern(std::span arguments, Type returnType) +{ + Locker locker { m_lock }; + + SignatureShape shape { arguments, returnType, Signature::computeHash(arguments, returnType) }; + auto addResult = m_signatures.ensure(shape, [&] { + return adoptRef(*new Signature(arguments, returnType)); + }); + if (addResult.isNewEntry) + dataLogLnIf(Options::verboseFFI(), "FFI: interned signature ", (*addResult.iterator)->toString()); + return addResult.iterator->copyRef(); +} + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFISignature.h b/Source/JavaScriptCore/ffi/FFISignature.h new file mode 100644 index 0000000000000..ac69a3b21193c --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFISignature.h @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) + +#include "FFIType.h" +#include "JSCPtrTag.h" +#include "JSExportMacros.h" +#include "MacroAssemblerCodeRef.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace JSC { namespace FFI { + +using InvokeThunkFunction = void (JIT_OPERATION_ATTRIBUTES *)(void* target, uint64_t* slots); + +constexpr size_t slotSize = 8; + +constexpr size_t argumentSlotOffset(unsigned index) { return static_cast(index) * slotSize; } + +class Signature final : public ThreadSafeRefCounted { + WTF_MAKE_TZONE_ALLOCATED(Signature); + WTF_MAKE_NONCOPYABLE(Signature); +public: + static constexpr unsigned maxArguments = 32; + + JS_EXPORT_PRIVATE static RefPtr tryCreate(std::span arguments, Type returnType); + + unsigned argumentCount() const { return static_cast(m_arguments.size()); } + Type argumentType(unsigned index) const { return m_arguments[index]; } + std::span arguments() const LIFETIME_BOUND { return m_arguments.span(); } + Type returnType() const { return m_returnType; } + + unsigned slotCount() const { return argumentCount() + 1; } + size_t slotBufferBytes() const { return static_cast(slotCount()) * slotSize; } + + unsigned hash() const { return m_hash; } + JS_EXPORT_PRIVATE String toString() const; + + JS_EXPORT_PRIVATE bool operator==(const Signature&) const; + + JS_EXPORT_PRIVATE static bool structurallyEquals(std::span arguments, Type returnType, const Signature&); + JS_EXPORT_PRIVATE static unsigned computeHash(std::span arguments, Type returnType); + + JS_EXPORT_PRIVATE CodePtr invokeThunk(); + +private: + friend class SignatureRegistry; + Signature(std::span arguments, Type returnType); + + FixedVector m_arguments; + Type m_returnType; + unsigned m_hash { 0 }; + + Lock m_codeLock; + MacroAssemblerCodeRef m_invokeThunkCode WTF_GUARDED_BY_LOCK(m_codeLock); + std::atomic m_publishedInvokeThunk { nullptr }; +}; + +struct SignatureHash { + static unsigned hash(const Ref& signature) { return signature->hash(); } + static bool equal(const Ref& a, const Ref& b) { return a.ptr() == b.ptr() || Signature::structurallyEquals(a->arguments(), a->returnType(), b.get()); } + static constexpr bool safeToCompareToEmptyOrDeleted = false; +}; + +class SignatureRegistry { + WTF_MAKE_TZONE_ALLOCATED(SignatureRegistry); + WTF_MAKE_NONCOPYABLE(SignatureRegistry); +public: + SignatureRegistry() = default; + + JS_EXPORT_PRIVATE static SignatureRegistry& singleton(); + + JS_EXPORT_PRIVATE Ref intern(std::span arguments, Type returnType); + +private: + Lock m_lock; + UncheckedKeyHashSet, SignatureHash> m_signatures WTF_GUARDED_BY_LOCK(m_lock); +}; + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/FFIType.h b/Source/JavaScriptCore/ffi/FFIType.h new file mode 100644 index 0000000000000..af2a6e7f23504 --- /dev/null +++ b/Source/JavaScriptCore/ffi/FFIType.h @@ -0,0 +1,311 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) + +#include "JSExportMacros.h" +#include +#include +#include +#include + +namespace JSC { namespace FFI { + +enum class Type : uint8_t { + Char = 0, + Int8 = 1, + Uint8 = 2, + Int16 = 3, + Uint16 = 4, + Int32 = 5, + Uint32 = 6, + Int64 = 7, + Uint64 = 8, + Double = 9, + Float = 10, + Bool = 11, + Pointer = 12, + Void = 13, + CString = 14, + Int64Fast = 15, + Uint64Fast = 16, + Function = 17, + RESERVED_WasNapiEnv = 18, + JSValue = 19, + Buffer = 20, + BufferLength = 21, +}; + +constexpr unsigned numberOfTypes = 22; + +constexpr ASCIILiteral name(Type type) +{ + switch (type) { + case Type::Char: + return "char"_s; + case Type::Int8: + return "i8"_s; + case Type::Uint8: + return "u8"_s; + case Type::Int16: + return "i16"_s; + case Type::Uint16: + return "u16"_s; + case Type::Int32: + return "i32"_s; + case Type::Uint32: + return "u32"_s; + case Type::Int64: + return "i64"_s; + case Type::Uint64: + return "u64"_s; + case Type::Double: + return "f64"_s; + case Type::Float: + return "f32"_s; + case Type::Bool: + return "bool"_s; + case Type::Pointer: + return "ptr"_s; + case Type::Void: + return "void"_s; + case Type::CString: + return "cstring"_s; + case Type::Int64Fast: + return "i64_fast"_s; + case Type::Uint64Fast: + return "u64_fast"_s; + case Type::Function: + return "function"_s; + case Type::RESERVED_WasNapiEnv: + return "invalid"_s; + case Type::JSValue: + return "jsvalue"_s; + case Type::Buffer: + return "buffer"_s; + case Type::BufferLength: + return "buffer_length"_s; + } + return "invalid"_s; +} + +inline std::optional parseType(StringView string) +{ + if (string == "char"_s) + return Type::Char; + if (string == "i8"_s) + return Type::Int8; + if (string == "u8"_s) + return Type::Uint8; + if (string == "i16"_s) + return Type::Int16; + if (string == "u16"_s) + return Type::Uint16; + if (string == "i32"_s) + return Type::Int32; + if (string == "u32"_s) + return Type::Uint32; + if (string == "i64"_s) + return Type::Int64; + if (string == "u64"_s) + return Type::Uint64; + if (string == "f64"_s) + return Type::Double; + if (string == "f32"_s) + return Type::Float; + if (string == "bool"_s) + return Type::Bool; + if (string == "ptr"_s) + return Type::Pointer; + if (string == "void"_s) + return Type::Void; + if (string == "cstring"_s) + return Type::CString; + if (string == "i64_fast"_s) + return Type::Int64Fast; + if (string == "u64_fast"_s) + return Type::Uint64Fast; + if (string == "function"_s) + return Type::Function; + if (string == "jsvalue"_s || string == "napi_value"_s) // napi_value: legacy spelling of the same raw-JSValue type + return Type::JSValue; + if (string == "buffer"_s) + return Type::Buffer; + if (string == "buffer_length"_s || string == "buffer_bytelength"_s) // buffer_bytelength: alias of the same length twin + return Type::BufferLength; + + if (string == "int8_t"_s) + return Type::Int8; + if (string == "uint8_t"_s) + return Type::Uint8; + if (string == "int16_t"_s) + return Type::Int16; + if (string == "uint16_t"_s) + return Type::Uint16; + if (string == "int32_t"_s || string == "int"_s || string == "c_int"_s) + return Type::Int32; + if (string == "uint32_t"_s || string == "c_uint"_s) + return Type::Uint32; + if (string == "int64_t"_s || string == "isize"_s) + return Type::Int64; + if (string == "uint64_t"_s || string == "usize"_s || string == "size_t"_s) + return Type::Uint64; + if (string == "double"_s) + return Type::Double; + if (string == "float"_s) + return Type::Float; + if (string == "void*"_s || string == "pointer"_s || string == "char*"_s) + return Type::Pointer; + if (string == "callback"_s || string == "fn"_s) + return Type::Function; + + return std::nullopt; +} + +enum class ArgClass : uint8_t { Void, Int, Float, Double }; + +constexpr ArgClass argClass(Type type) +{ + switch (type) { + case Type::Void: + return ArgClass::Void; + case Type::Float: + return ArgClass::Float; + case Type::Double: + return ArgClass::Double; + case Type::Char: + case Type::Int8: + case Type::Uint8: + case Type::Int16: + case Type::Uint16: + case Type::Int32: + case Type::Uint32: + case Type::Int64: + case Type::Uint64: + case Type::Bool: + case Type::Pointer: + case Type::CString: + case Type::Int64Fast: + case Type::Uint64Fast: + case Type::Function: + case Type::RESERVED_WasNapiEnv: + case Type::JSValue: + case Type::Buffer: + case Type::BufferLength: + return ArgClass::Int; + } + return ArgClass::Int; +} + +constexpr bool isValidReturnType(Type type) +{ + return type != Type::RESERVED_WasNapiEnv && type != Type::Buffer && type != Type::BufferLength; +} + +constexpr bool isValidArgumentType(Type type) +{ + return type != Type::Void && type != Type::RESERVED_WasNapiEnv; +} + +constexpr unsigned nativeSizeInBytes(Type type) +{ + switch (type) { + case Type::Void: + return 0; + case Type::Char: + case Type::Int8: + case Type::Uint8: + case Type::Bool: + return 1; + case Type::Int16: + case Type::Uint16: + return 2; + case Type::Int32: + case Type::Uint32: + case Type::Float: + return 4; + case Type::Int64: + case Type::Uint64: + case Type::Double: + case Type::Pointer: + case Type::CString: + case Type::Int64Fast: + case Type::Uint64Fast: + case Type::Function: + case Type::RESERVED_WasNapiEnv: + case Type::JSValue: + case Type::Buffer: + case Type::BufferLength: // uint64_t / size_t, exactly like Uint64 + return 8; + } + return 0; +} + +constexpr bool isSigned(Type type) +{ + switch (type) { + case Type::Char: + case Type::Int8: + case Type::Int16: + case Type::Int32: + case Type::Int64: + case Type::Int64Fast: + return true; + case Type::Uint8: + case Type::Uint16: + case Type::Uint32: + case Type::Uint64: + case Type::Uint64Fast: + case Type::Double: + case Type::Float: + case Type::Bool: + case Type::Pointer: + case Type::Void: + case Type::CString: + case Type::Function: + case Type::RESERVED_WasNapiEnv: + case Type::JSValue: + case Type::Buffer: + case Type::BufferLength: // unsigned 64-bit, exactly like Uint64 + return false; + } + return false; +} + +struct CompileCounts { + std::atomic icStub { 0 }; + std::atomic dfgCallFFI { 0 }; + std::atomic ftlCallFFI { 0 }; +}; + +extern JS_EXPORT_PRIVATE CompileCounts g_ffiCompileCounts; + +} } // namespace JSC::FFI + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/JSFFICallback.cpp b/Source/JavaScriptCore/ffi/JSFFICallback.cpp new file mode 100644 index 0000000000000..260c3d3664987 --- /dev/null +++ b/Source/JavaScriptCore/ffi/JSFFICallback.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "JSFFICallback.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "Error.h" +#include "ExceptionHelpers.h" +#include "FFICallbackThunk.h" +#include "FFIConversions.h" +#include "JSCInlines.h" +#include "Options.h" +#include +#include + +namespace JSC { + +const ClassInfo JSFFICallback::s_info = { "FFICallback"_s, &Base::s_info, nullptr, nullptr, CREATE_METHOD_TABLE(JSFFICallback) }; + +static JSC_DECLARE_HOST_FUNCTION(ffiCallbackProtoFuncClose); +JSC_DEFINE_HOST_FUNCTION(ffiCallbackProtoFuncClose, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + auto* callback = dynamicDowncast(callFrame->thisValue()); + if (!callback) [[unlikely]] + return throwVMTypeError(globalObject, scope, "FFICallback.prototype.close called on an incompatible receiver"_s); + callback->close(); + return JSValue::encode(jsUndefined()); +} + +JSFFICallback::JSFFICallback(VM& vm, Structure* structure, Ref&& signature) + : Base(vm, structure) + , m_signature(WTF::move(signature)) +{ +} + +JSFFICallback::~JSFFICallback() = default; + +void JSFFICallback::destroy(JSCell* cell) +{ + static_cast(cell)->JSFFICallback::~JSFFICallback(); +} + +JSObject* JSFFICallback::createPrototype(VM& vm, JSGlobalObject* globalObject) +{ + JSObject* prototype = JSFinalObject::create(vm, JSFinalObject::createStructure(vm, globalObject, globalObject->objectPrototype(), 1)); + constexpr unsigned attributes = static_cast(PropertyAttribute::DontEnum); + prototype->putDirectNativeFunction(vm, globalObject, Identifier::fromString(vm, "close"_s), 0, ffiCallbackProtoFuncClose, ImplementationVisibility::Public, NoIntrinsic, attributes); + return prototype; +} + +Structure* JSFFICallback::createStructure(VM& vm, JSGlobalObject* globalObject, JSValue prototype) +{ + return Structure::create(vm, globalObject, prototype, TypeInfo(ObjectType, StructureFlags), info()); +} + +JSFFICallback* JSFFICallback::create(VM& vm, JSGlobalObject* globalObject, Structure* structure, JSObject* callable, Ref&& signature, bool threadsafe, void* embedderContext) +{ + globalObject->ffiContext(); + + auto scope = DECLARE_THROW_SCOPE(vm); + + if (!Options::useJIT()) [[unlikely]] { + throwTypeError(globalObject, scope, "bun:ffi requires the JIT"_s); + return nullptr; + } + +#if !FFI_CALLBACK_THUNK_SUPPORTED + UNUSED_PARAM(structure); + UNUSED_PARAM(callable); + UNUSED_PARAM(signature); + UNUSED_PARAM(threadsafe); + UNUSED_PARAM(embedderContext); + throwTypeError(globalObject, scope, "bun:ffi is not supported on this architecture"_s); + return nullptr; +#else + ASSERT(callable); + if (threadsafe && !FFI::FFIContext::threadsafeDispatch()) [[unlikely]] { + throwTypeError(globalObject, scope, "bun:ffi: no threadsafe dispatch registered (FFIContext::setThreadsafeDispatch)"_s); + return nullptr; + } + JSFFICallback* callback = new (NotNull, allocateCell(vm)) JSFFICallback(vm, structure, WTF::move(signature)); + callback->m_threadsafe = threadsafe; + callback->m_embedderContext = embedderContext; + callback->finishCreation(vm, callable); + RETURN_IF_EXCEPTION(scope, nullptr); + if (!callback->m_entryCode) [[unlikely]] { + throwOutOfMemoryError(globalObject, scope); + return nullptr; + } + globalObject->ffiContext().addLiveCallback(vm, *globalObject, callback); + return callback; +#endif +} + +void JSFFICallback::finishCreation(VM& vm, JSObject* callable) +{ + Base::finishCreation(vm); + ASSERT(inherits(info())); + + m_callable.set(vm, this, callable); + +#if FFI_CALLBACK_THUNK_SUPPORTED + m_entryCode = FFI::generateCallbackThunk(vm, *this); + if (!m_entryCode) [[unlikely]] + return; // create() throws the OutOfMemoryError. + + if (Options::verboseFFI()) [[unlikely]] + dataLogLn("FFI: created JSFFICallback ", RawPointer(this), " signature ", m_signature->toString(), " entrypoint ", RawPointer(nativeEntrypoint())); + + constexpr unsigned attributes = static_cast(PropertyAttribute::ReadOnly | PropertyAttribute::DontEnum | PropertyAttribute::DontDelete); + putDirect(vm, Identifier::fromString(vm, "ptr"_s), FFI::pointerToJSValue(globalObject(), reinterpret_cast(nativeEntrypoint())), attributes); + putDirect(vm, Identifier::fromString(vm, "threadsafe"_s), jsBoolean(m_threadsafe), attributes); +#endif +} + +void* JSFFICallback::nativeEntrypoint() const +{ + return untagCFunctionPtr(m_entryCode.code().taggedPtr()); +} + +const char* JSFFICallback::setReturnCString(const CString& string) +{ + m_returnCString.clear(); + m_returnCString.append(string.span()); + m_returnCString.append('\0'); + return m_returnCString.begin(); +} + +void JSFFICallback::close() +{ + if (m_closed) + return; + m_closed = true; + VM& vm = this->vm(); + if (!markClosedAndReportPending()) + unroot(); + constexpr unsigned attributes = static_cast(PropertyAttribute::ReadOnly | PropertyAttribute::DontEnum | PropertyAttribute::DontDelete); + putDirect(vm, Identifier::fromString(vm, "ptr"_s), jsNull(), attributes); +} + +void JSFFICallback::unroot() +{ + if (auto* globalObject = this->globalObject()) + globalObject->ffiContext().removeLiveCallback(*globalObject, this); +} + +template +void JSFFICallback::visitChildrenImpl(JSCell* cell, Visitor& visitor) +{ + JSFFICallback* thisObject = uncheckedDowncast(cell); + ASSERT_GC_OBJECT_INHERITS(thisObject, info()); + + Base::visitChildren(thisObject, visitor); + + visitor.append(thisObject->m_callable); +} + +DEFINE_VISIT_CHILDREN(JSFFICallback); + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/JSFFICallback.h b/Source/JavaScriptCore/ffi/JSFFICallback.h new file mode 100644 index 0000000000000..e9e5a92e4a1ce --- /dev/null +++ b/Source/JavaScriptCore/ffi/JSFFICallback.h @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#include "FFISignature.h" +#include "JSObject.h" +#include "MacroAssemblerCodeRef.h" +#include + +namespace JSC { + +class JSFFICallback final : public JSNonFinalObject { +public: + using Base = JSNonFinalObject; + + static constexpr unsigned StructureFlags = Base::StructureFlags; + + static constexpr DestructionMode needsDestruction = NeedsDestruction; + static void destroy(JSCell*); + + template + static GCClient::IsoSubspace* subspaceFor(VM& vm) + { + return vm.ffiCallbackSpace(); + } + + DECLARE_EXPORT_INFO; + + DECLARE_VISIT_CHILDREN; + + JS_EXPORT_PRIVATE static JSFFICallback* create(VM&, JSGlobalObject*, Structure*, JSObject* callable, Ref&&, bool threadsafe = false, void* embedderContext = nullptr); + static Structure* createStructure(VM&, JSGlobalObject*, JSValue prototype); + static JSObject* createPrototype(VM&, JSGlobalObject*); + + JS_EXPORT_PRIVATE void* nativeEntrypoint() const; + + JS_EXPORT_PRIVATE void close(); + bool isClosed() const { return m_closed; } + + JSObject* callable() const { return m_callable.get(); } + const char* setReturnCString(const CString&); + FFI::Signature& signature() const { return m_signature.get(); } + bool isThreadsafe() const { return m_threadsafe; } + void* embedderContext() const { return m_embedderContext; } + static constexpr unsigned closedBit = 0x80000000u; + static constexpr unsigned countMask = 0x7fffffffu; + bool tryBeginThreadsafeInvocation() + { + unsigned state = m_threadsafeState.load(std::memory_order_acquire); + do { + if (state & closedBit) + return false; + } while (!m_threadsafeState.compare_exchange_weak(state, state + 1, std::memory_order_acq_rel, std::memory_order_acquire)); + return true; + } + bool endThreadsafeInvocation() + { + unsigned state = m_threadsafeState.fetch_sub(1, std::memory_order_acq_rel) - 1; + return (state & closedBit) && !(state & countMask); + } + bool markClosedAndReportPending() { return m_threadsafeState.fetch_or(closedBit, std::memory_order_acq_rel) & countMask; } + void unroot(); + +private: + JSFFICallback(VM&, Structure*, Ref&&); + ~JSFFICallback(); + + void finishCreation(VM&, JSObject* callable); + + WriteBarrier m_callable; + Ref m_signature; + Vector m_returnCString; + MacroAssemblerCodeRef m_entryCode; + void* m_embedderContext { nullptr }; // opaque, embedder-owned; passed to threadsafe dispatch + std::atomic m_threadsafeState { 0 }; // closedBit | pending-invocation count (see above) + bool m_threadsafe { false }; + bool m_closed { false }; +}; + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/JSFFIFunction.cpp b/Source/JavaScriptCore/ffi/JSFFIFunction.cpp new file mode 100644 index 0000000000000..1af46200db80a --- /dev/null +++ b/Source/JavaScriptCore/ffi/JSFFIFunction.cpp @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "JSFFIFunction.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include "Error.h" +#include "FFICallHost.h" +#include "FFIConversions.h" +#include "FFISignature.h" +#include "JITCode.h" +#include "JSCInlines.h" +#include "JSObjectInlines.h" +#include "NativeExecutable.h" +#include "SlotVisitorInlines.h" +#include "StructureInlines.h" +#include +#include + +#if ENABLE(JIT) +#include "FFIICStub.h" +#endif + +namespace JSC { + +const ClassInfo JSFFIFunction::s_info = { "Function"_s, &Base::s_info, nullptr, nullptr, CREATE_METHOD_TABLE(JSFFIFunction) }; +CLASSINFO_KEEP_ADDRESS_UNIQUE(JSFFIFunction); + +JSFFIFunction::JSFFIFunction(VM& vm, NativeExecutable* executable, JSGlobalObject* globalObject, Structure* structure, Ref&& signature, void* target, RefPtr&& icCode, const FFI::CallHooks* hooks) + : Base(vm, executable, globalObject, structure) + , m_signature(WTF::move(signature)) + , m_target(target) + , m_icCode(WTF::move(icCode)) + , m_hooks(hooks) +{ + ASSERT(!m_hooks || !m_icCode); +} + +JSFFIFunction::~JSFFIFunction() = default; + +template +void JSFFIFunction::visitChildrenImpl(JSCell* cell, Visitor& visitor) +{ + auto* thisObject = uncheckedDowncast(cell); + ASSERT_GC_OBJECT_INHERITS(thisObject, info()); + Base::visitChildren(thisObject, visitor); + visitor.append(thisObject->m_owner); // keeps the owner (e.g. library handle) alive +} + +DEFINE_VISIT_CHILDREN(JSFFIFunction); + +void JSFFIFunction::destroy(JSCell* cell) +{ + static_cast(cell)->JSFFIFunction::~JSFFIFunction(); +} + +static constexpr unsigned ffiIntrinsicAttributes = static_cast(PropertyAttribute::ReadOnly | PropertyAttribute::DontEnum | PropertyAttribute::DontDelete); +static constexpr PropertyOffset ptrOffset = firstOutOfLineOffset; +static constexpr PropertyOffset nativeOffset = firstOutOfLineOffset + 1; + +Structure* JSFFIFunction::createStructure(VM& vm, JSGlobalObject* globalObject, JSValue prototype) +{ + ASSERT(globalObject); + Structure* structure = Structure::create(vm, globalObject, prototype, TypeInfo(JSFunctionType, StructureFlags), info()); + PropertyOffset offset; + structure = Structure::addPropertyTransition(vm, structure, Identifier::fromString(vm, "ptr"_s), ffiIntrinsicAttributes, offset); + ASSERT_UNUSED(offset, offset == ptrOffset); + structure = Structure::addPropertyTransition(vm, structure, Identifier::fromString(vm, "native"_s), ffiIntrinsicAttributes, offset); + ASSERT(offset == nativeOffset); + return structure; +} + +JSFFIFunction* JSFFIFunction::create(VM& vm, JSGlobalObject* globalObject, Structure* structure, Ref&& signatureRef, void* target, const String& name, JSObject* owner, const FFI::CallHooks* hooks) +{ + auto scope = DECLARE_THROW_SCOPE(vm); + + if (!Options::useJIT()) [[unlikely]] { + throwTypeError(globalObject, scope, "bun:ffi requires the JIT"_s); + return nullptr; + } + +#if !USE(JSVALUE64) || ENABLE(JIT_CAGE) || !(CPU(X86_64) || CPU(ARM64)) + UNUSED_PARAM(structure); + UNUSED_PARAM(signatureRef); + UNUSED_PARAM(target); + UNUSED_PARAM(name); + UNUSED_PARAM(owner); + UNUSED_PARAM(hooks); + throwTypeError(globalObject, scope, "bun:ffi is not supported on this architecture"_s); + return nullptr; +#else + Ref signature = WTF::move(signatureRef); + unsigned length = signature->argumentCount(); + + globalObject->ffiContext(); + + NativeExecutable* base = vm.getHostFunction(FFI::ffiHostCall, ImplementationVisibility::Public, NoIntrinsic, callHostFunctionAsConstructor, nullptr, length, name); + + RefPtr stub; +#if ENABLE(JIT) + if (Options::useFFIICStub() && !hooks) + stub = FFI::generateICStubCode(vm, globalObject, signature.get(), target); +#endif + + NativeExecutable* executable = base; + if (stub) + executable = NativeExecutable::create(vm, Ref(*stub), FFI::ffiHostCall, base->generatedJITCodeForConstruct(), callHostFunctionAsConstructor, ImplementationVisibility::Public, length, name); + + JSFFIFunction* function = new (NotNull, allocateCell(vm)) JSFFIFunction(vm, executable, globalObject, structure, WTF::move(signature), target, WTF::move(stub), hooks); + function->finishCreation(vm); + if (owner) + function->m_owner.set(vm, function, owner); // write-barriered: the owner outlives this function + + function->setButterfly(vm, Butterfly::create(vm, function, 0, structure->outOfLineCapacity(), false, IndexingHeader(), 0)); + function->putDirectOffset(vm, ptrOffset, FFI::pointerToJSValue(globalObject, reinterpret_cast(target))); + function->putDirectOffset(vm, nativeOffset, function); + + dataLogLnIf(Options::verboseFFI(), "FFI: created JSFFIFunction '", name, "' ", function->signature().toString(), " target=", RawPointer(target), " icStub=", !!function->icCode()); + + RELEASE_AND_RETURN(scope, function); +#endif +} + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/JSFFIFunction.h b/Source/JavaScriptCore/ffi/JSFFIFunction.h new file mode 100644 index 0000000000000..0e2464052d54f --- /dev/null +++ b/Source/JavaScriptCore/ffi/JSFFIFunction.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#if USE(BUN_JSC_ADDITIONS) + +#include "FFISignature.h" +#include "JSFunction.h" +#include +#include + +namespace JSC { + +class JITCode; +class JSGlobalObject; + +namespace FFI { + +struct CallHooks { + void* (*before)(JSGlobalObject*, CallFrame*); + void (*after)(JSGlobalObject*, CallFrame*, void* token); +}; + +} // namespace FFI + +class JSFFIFunction final : public JSFunction { +public: + using Base = JSFunction; + + static constexpr unsigned StructureFlags = Base::StructureFlags; + + static constexpr DestructionMode needsDestruction = NeedsDestruction; // Holds Ref + RefPtr. + static void destroy(JSCell*); + ~JSFFIFunction(); // Out-of-line: RefPtr needs the complete type only in the .cpp. + + template + static GCClient::IsoSubspace* subspaceFor(VM& vm) + { + return vm.ffiFunctionSpace(); + } + + DECLARE_EXPORT_INFO; + + static Structure* createStructure(VM&, JSGlobalObject*, JSValue prototype); + + JS_EXPORT_PRIVATE static JSFFIFunction* create(VM&, JSGlobalObject*, Structure*, Ref&&, void* target, const String& name, JSObject* owner = nullptr, const FFI::CallHooks* hooks = nullptr); + + DECLARE_VISIT_CHILDREN; + + FFI::Signature& signature() const { return m_signature.get(); } + void* target() const { return m_target; } + JSObject* owner() const { return m_owner.get(); } + const FFI::CallHooks* hooks() const { return m_hooks; } + bool isHostPathOnly() const { return !!m_hooks; } + JITCode* icCode() const { return m_icCode.get(); } + +private: + JSFFIFunction(VM&, NativeExecutable*, JSGlobalObject*, Structure*, Ref&&, void* target, RefPtr&& icCode, const FFI::CallHooks* hooks); + + Ref m_signature; + void* m_target; + RefPtr m_icCode; // Keeps the IC entry stub alive; null when no stub. + WriteBarrier m_owner; // Optional; keeps the owner (e.g. library handle object) alive. + const FFI::CallHooks* m_hooks; // Optional, static lifetime; non-null => host path only. +}; + +} // namespace JSC + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/tests/FFITestFixtures.cpp b/Source/JavaScriptCore/ffi/tests/FFITestFixtures.cpp new file mode 100644 index 0000000000000..2d541c3ffedbb --- /dev/null +++ b/Source/JavaScriptCore/ffi/tests/FFITestFixtures.cpp @@ -0,0 +1,803 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFITestFixtures.h" + +#if USE(BUN_JSC_ADDITIONS) + +#include +#include +#include +#include +#include +#include + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +extern "C" { + +signed char ffi_echo_char(signed char x) { return x; } +int8_t ffi_echo_i8(int8_t x) { return x; } +uint8_t ffi_echo_u8(uint8_t x) { return x; } +int16_t ffi_echo_i16(int16_t x) { return x; } +uint16_t ffi_echo_u16(uint16_t x) { return x; } +int32_t ffi_echo_i32(int32_t x) { return x; } +uint32_t ffi_echo_u32(uint32_t x) { return x; } +int64_t ffi_echo_i64(int64_t x) { return x; } +uint64_t ffi_echo_u64(uint64_t x) { return x; } +float ffi_echo_f32(float x) { return x; } +double ffi_echo_f64(double x) { return x; } +bool ffi_echo_bool(bool x) { return x; } +void* ffi_echo_ptr(void* x) { return x; } +const char* ffi_echo_cstring(const char* x) { return x; } +jsvalue_slot ffi_echo_jsvalue(jsvalue_slot x) { return x; } + +int64_t ffi_widen_char(signed char x) { return x; } +int64_t ffi_widen_i8(int8_t x) { return x; } +int64_t ffi_widen_u8(uint8_t x) { return x; } +int64_t ffi_widen_i16(int16_t x) { return x; } +int64_t ffi_widen_u16(uint16_t x) { return x; } +void* ffi_ret_null_ptr(void) { return nullptr; } +int8_t ffi_ret_two_as_bool(void) { return 2; } +int8_t ffi_ret_neg_one_i8(void) { return -1; } +int16_t ffi_ret_neg_one_i16(void) { return -1; } +int32_t ffi_ret_neg_one_i32(void) { return -1; } +int64_t ffi_ret_neg_one_i64(void) { return -1; } +uint8_t ffi_ret_neg_one_u8(void) { return static_cast(-1); } +uint16_t ffi_ret_neg_one_u16(void) { return static_cast(-1); } +uint32_t ffi_ret_neg_one_u32(void) { return static_cast(-1); } +uint64_t ffi_ret_neg_one_u64(void) { return static_cast(-1); } + +float ffi_ret_nan_f32(void) { return std::bit_cast(0x7fc00001u); } +double ffi_ret_impure_nan_f64(void) { return std::bit_cast(0x7ff0000000000001ull); } +double ffi_ret_neg_zero_f64(void) { return -0.0; } +float ffi_ret_denormal_f32(void) { return std::bit_cast(0x00000001u); } +double ffi_ret_inf_f64(void) { return std::bit_cast(0x7ff0000000000000ull); } + +int32_t ffi_add_i32(int32_t a, int32_t b) { return static_cast(static_cast(a) + static_cast(b)); } +double ffi_add_f64(double a, double b) { return a + b; } +int64_t ffi_add_i64(int64_t a, int64_t b) { return static_cast(static_cast(a) + static_cast(b)); } +uint64_t ffi_add_u64(uint64_t a, uint64_t b) { return a + b; } +float ffi_add_f32(float a, float b) { return a + b; } + +int64_t ffi_sum_i32_0(void) { return 0; } +int64_t ffi_sum_i32_1(int32_t a0) { return int64_t(a0); } +int64_t ffi_sum_i32_2(int32_t a0, int32_t a1) { return int64_t(a0) + int64_t(a1); } +int64_t ffi_sum_i32_4(int32_t a0, int32_t a1, int32_t a2, int32_t a3) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3); +} +int64_t ffi_sum_i32_6(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5); +} +int64_t ffi_sum_i32_7(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6); +} +int64_t ffi_sum_i32_8(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7); +} +int64_t ffi_sum_i32_9(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7, int32_t a8) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + int64_t(a8); +} +int64_t ffi_sum_i32_12(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7, int32_t a8, int32_t a9, int32_t a10, int32_t a11) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + + int64_t(a8) + int64_t(a9) + int64_t(a10) + int64_t(a11); +} +int64_t ffi_sum_i32_16(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7, int32_t a8, int32_t a9, int32_t a10, int32_t a11, int32_t a12, int32_t a13, int32_t a14, int32_t a15) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + + int64_t(a8) + int64_t(a9) + int64_t(a10) + int64_t(a11) + int64_t(a12) + int64_t(a13) + int64_t(a14) + int64_t(a15); +} + +double ffi_sum_f64_1(double a0) { return a0; } +double ffi_sum_f64_2(double a0, double a1) { return a0 + a1; } +double ffi_sum_f64_7(double a0, double a1, double a2, double a3, double a4, double a5, double a6) +{ + return a0 + a1 + a2 + a3 + a4 + a5 + a6; +} +double ffi_sum_f64_8(double a0, double a1, double a2, double a3, double a4, double a5, double a6, double a7) +{ + return a0 + a1 + a2 + a3 + a4 + a5 + a6 + a7; +} +double ffi_sum_f64_9(double a0, double a1, double a2, double a3, double a4, double a5, double a6, double a7, double a8) +{ + return a0 + a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8; +} +double ffi_sum_f64_12(double a0, double a1, double a2, double a3, double a4, double a5, double a6, double a7, double a8, double a9, double a10, double a11) +{ + return a0 + a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8 + a9 + a10 + a11; +} + +int64_t ffi_sum_u8_10(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + int64_t(a8) + int64_t(a9); +} +int64_t ffi_sum_u8_12(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + int64_t(a8) + int64_t(a9) + + int64_t(a10) + int64_t(a11); +} +int64_t ffi_sum_i16_10(int16_t a0, int16_t a1, int16_t a2, int16_t a3, int16_t a4, int16_t a5, int16_t a6, int16_t a7, int16_t a8, int16_t a9) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + int64_t(a8) + int64_t(a9); +} +int64_t ffi_sum_i16_12(int16_t a0, int16_t a1, int16_t a2, int16_t a3, int16_t a4, int16_t a5, int16_t a6, int16_t a7, int16_t a8, int16_t a9, int16_t a10, int16_t a11) +{ + return int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + int64_t(a8) + int64_t(a9) + + int64_t(a10) + int64_t(a11); +} + +static inline double ffiPtrToDouble(void* p) +{ + return static_cast(reinterpret_cast(p)); +} + +double ffi_mix_1(int32_t a0, double a1, int64_t a2, float a3, void* a4, uint8_t a5, double a6, int16_t a7, double a8, int32_t a9) +{ + return 1.0 * static_cast(a0) + + 2.0 * a1 + + 3.0 * static_cast(a2) + + 4.0 * static_cast(a3) + + 5.0 * ffiPtrToDouble(a4) + + 6.0 * static_cast(a5) + + 7.0 * a6 + + 8.0 * static_cast(a7) + + 9.0 * a8 + + 10.0 * static_cast(a9); +} + +double ffi_mix_2(float a0, int32_t a1, float a2, int32_t a3, float a4, int32_t a5, float a6, int32_t a7, float a8, int32_t a9) +{ + return 1.0 * static_cast(a0) + + 2.0 * static_cast(a1) + + 3.0 * static_cast(a2) + + 4.0 * static_cast(a3) + + 5.0 * static_cast(a4) + + 6.0 * static_cast(a5) + + 7.0 * static_cast(a6) + + 8.0 * static_cast(a7) + + 9.0 * static_cast(a8) + + 10.0 * static_cast(a9); +} + +double ffi_mix_3(double a0, double a1, double a2, double a3, double a4, double a5, double a6, double a7, int32_t a8) +{ + return 1.0 * a0 + 2.0 * a1 + 3.0 * a2 + 4.0 * a3 + 5.0 * a4 + 6.0 * a5 + 7.0 * a6 + 8.0 * a7 + 9.0 * static_cast(a8); +} + +double ffi_mix_4(int64_t a0, int64_t a1, int64_t a2, int64_t a3, int64_t a4, int64_t a5, double a6, int64_t a7, double a8) +{ + return 1.0 * static_cast(a0) + + 2.0 * static_cast(a1) + + 3.0 * static_cast(a2) + + 4.0 * static_cast(a3) + + 5.0 * static_cast(a4) + + 6.0 * static_cast(a5) + + 7.0 * a6 + + 8.0 * static_cast(a7) + + 9.0 * a8; +} + +double ffi_mix_5(uint8_t a0, int8_t a1, uint16_t a2, int16_t a3, uint32_t a4, int32_t a5, uint64_t a6, int64_t a7) +{ + return 1.0 * static_cast(a0) + + 2.0 * static_cast(a1) + + 3.0 * static_cast(a2) + + 4.0 * static_cast(a3) + + 5.0 * static_cast(a4) + + 6.0 * static_cast(a5) + + 7.0 * static_cast(a6) + + 8.0 * static_cast(a7); +} + +double ffi_mix_6(bool a0, bool a1, int32_t a2, bool a3, double a4, bool a5, float a6, bool a7, bool a8, bool a9, bool a10, bool a11, bool a12) +{ + return 1.0 * (a0 ? 1.0 : 0.0) + + 2.0 * (a1 ? 1.0 : 0.0) + + 3.0 * static_cast(a2) + + 4.0 * (a3 ? 1.0 : 0.0) + + 5.0 * a4 + + 6.0 * (a5 ? 1.0 : 0.0) + + 7.0 * static_cast(a6) + + 8.0 * (a7 ? 1.0 : 0.0) + + 9.0 * (a8 ? 1.0 : 0.0) + + 10.0 * (a9 ? 1.0 : 0.0) + + 11.0 * (a10 ? 1.0 : 0.0) + + 12.0 * (a11 ? 1.0 : 0.0) + + 13.0 * (a12 ? 1.0 : 0.0); +} + +double ffi_mix_7(void* a0, signed char a1, void* a2, signed char a3, void* a4, signed char a5, void* a6, signed char a7, void* a8, signed char a9) +{ + return 1.0 * ffiPtrToDouble(a0) + + 2.0 * static_cast(a1) + + 3.0 * ffiPtrToDouble(a2) + + 4.0 * static_cast(a3) + + 5.0 * ffiPtrToDouble(a4) + + 6.0 * static_cast(a5) + + 7.0 * ffiPtrToDouble(a6) + + 8.0 * static_cast(a7) + + 9.0 * ffiPtrToDouble(a8) + + 10.0 * static_cast(a9); +} + +double ffi_mix_8(float a0, double a1, float a2, double a3, float a4, double a5, float a6, double a7, float a8, double a9, float a10, double a11) +{ + return 1.0 * static_cast(a0) + + 2.0 * a1 + + 3.0 * static_cast(a2) + + 4.0 * a3 + + 5.0 * static_cast(a4) + + 6.0 * a5 + + 7.0 * static_cast(a6) + + 8.0 * a7 + + 9.0 * static_cast(a8) + + 10.0 * a9 + + 11.0 * static_cast(a10) + + 12.0 * a11; +} + +void ffi_ptr_write_u32(uint32_t* p, uint32_t v) { *p = v; } +uint32_t ffi_ptr_read_u32(uint32_t* p) { return *p; } +uint64_t ffi_strlen(const char* s) { return static_cast(strlen(s)); } +void* ffi_ptr_identity(void* p) { return p; } +void* ffi_high_ptr(void) { return reinterpret_cast(static_cast(0x00007fffdeadbee0ull)); } +uint64_t ffi_view_byte_length(const uint8_t*, uint64_t length) { return length; } +int32_t ffi_view_last_byte(const uint8_t* ptr, uint64_t length) { return length ? static_cast(ptr[length - 1]) : -1; } + +void ffi_bench_noop(void) { } +const char* ffi_bench_string(void) { return "Hello, world!"; } +uint32_t ffi_bench_hash(const uint8_t* ptr, uint32_t length) +{ + uint32_t hash = 0; + for (uint32_t i = 0; i < length; ++i) + hash = hash * 0x10001000u + ptr[i]; + return hash; +} + +int32_t ffi_call_cb_i32(int32_t (*cb)(int32_t), int32_t x) { return cb(x); } + +double ffi_call_cb_f64_x8(double (*cb)(double, double, double, double, double, double, double, double), double a0, double a1, double a2, double a3, double a4, double a5, double a6, double a7) +{ + return cb(a0, a1, a2, a3, a4, a5, a6, a7); +} + +double ffi_call_cb_mix(double (*cb)(int32_t, double, int64_t, float, void*), int32_t a0, double a1, int64_t a2, float a3, void* a4) +{ + return cb(a0, a1, a2, a3, a4); +} + +void ffi_call_cb_void(void (*cb)(void)) { cb(); } + +int32_t ffi_call_cb_reentrant(int32_t (*cb)(int32_t), int32_t depth) +{ + int32_t sum = 0; + for (int32_t i = 0; i < depth; ++i) + sum += cb(i); + return sum; +} + +int64_t ffi_call_cb_i32_x9(int64_t (*cb)(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t), int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7, int32_t a8) +{ + return cb(a0, a1, a2, a3, a4, a5, a6, a7, a8); +} + +double ffi_call_cb_f64_x9(double (*cb)(double, double, double, double, double, double, double, double, double), double a0, double a1, double a2, double a3, double a4, double a5, double a6, double a7, double a8) +{ + return cb(a0, a1, a2, a3, a4, a5, a6, a7, a8); +} + +int64_t ffi_call_cb_u8_x10(int64_t (*cb)(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t), uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9) +{ + return cb(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9); +} + +int64_t ffi_sum_i32_x10(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7, int32_t a8, int32_t a9) +{ + return static_cast(a0) * 1 + static_cast(a1) * 2 + static_cast(a2) * 3 + + static_cast(a3) * 4 + static_cast(a4) * 5 + static_cast(a5) * 6 + + static_cast(a6) * 7 + static_cast(a7) * 8 + static_cast(a8) * 9 + + static_cast(a9) * 10; +} + +void ffi_call_cb_from_thread(void (*cb)(int32_t, int64_t, uint64_t, double), int32_t a, int64_t b, uint64_t c, double d) +{ + auto thread = Thread::create("ffi-cb-from-thread"_s, [cb, a, b, c, d] { + cb(a, b, c, d); + }); + thread->waitForCompletion(); +} + +int64_t ffi_call_cb_ret_i8(int8_t (*cb)(void)) { return static_cast(cb()); } +int64_t ffi_call_cb_ret_u8(uint8_t (*cb)(void)) { return static_cast(cb()); } +int64_t ffi_call_cb_ret_i64(int64_t (*cb)(void)) { return cb(); } +uint64_t ffi_call_cb_ret_u64(uint64_t (*cb)(void)) { return cb(); } +int32_t ffi_call_cb_ret_bool(bool (*cb)(void)) { return cb() ? 10 : 20; } +float ffi_call_cb_ret_f32(float (*cb)(void)) { return cb(); } +double ffi_call_cb_ret_f64(double (*cb)(void)) { return cb(); } +const char* ffi_call_cb_ret_cstring(const char* (*cb)(void)) { return cb(); } +void* ffi_call_cb_ret_ptr(void* (*cb)(void)) { return cb(); } + +uint32_t ffi_call_cb_then_read_u32(uint32_t (*cb)(void), uint32_t* p) +{ + cb(); + return *p; +} + +#if CPU(X86_64) + +struct FFICanaryStateX64 { + uint64_t original[7]; // rbx, r12, r13, r14, r15, rsi, rdi + uint64_t sentinel[7]; + uint64_t observed[7]; + unsigned char originalXMM[10][16]; // xmm6 - xmm15 (Win64 nonvolatile only) + unsigned char sentinelXMM[10][16]; + unsigned char observedXMM[10][16]; + void (*callback)(void); +}; +static_assert(offsetof(FFICanaryStateX64, original) == 0); +static_assert(offsetof(FFICanaryStateX64, sentinel) == 56); +static_assert(offsetof(FFICanaryStateX64, observed) == 112); +static_assert(offsetof(FFICanaryStateX64, originalXMM) == 168); +static_assert(offsetof(FFICanaryStateX64, sentinelXMM) == 328); +static_assert(offsetof(FFICanaryStateX64, observedXMM) == 488); +static_assert(offsetof(FFICanaryStateX64, callback) == 648); + +int32_t ffi_canary_call(void (*cb)(void)) +{ + FFICanaryStateX64 state; + memset(&state, 0, sizeof(state)); + state.callback = cb; + for (unsigned i = 0; i < 7; ++i) + state.sentinel[i] = 0xC0FFEE00CA11EE00ull + (uint64_t(i + 1) << 8) + (i + 1); + for (unsigned i = 0; i < 10; ++i) { + for (unsigned j = 0; j < 16; ++j) + state.sentinelXMM[i][j] = static_cast(0xA0 + i * 16 + j); + } + +#if OS(WINDOWS) + asm volatile( + "movq %[state], %%rax\n\t" + "movq %%rbx, 0(%%rax)\n\t" + "movq %%r12, 8(%%rax)\n\t" + "movq %%r13, 16(%%rax)\n\t" + "movq %%r14, 24(%%rax)\n\t" + "movq %%r15, 32(%%rax)\n\t" + "movq %%rsi, 40(%%rax)\n\t" + "movq %%rdi, 48(%%rax)\n\t" + "movdqu %%xmm6, 168(%%rax)\n\t" + "movdqu %%xmm7, 184(%%rax)\n\t" + "movdqu %%xmm8, 200(%%rax)\n\t" + "movdqu %%xmm9, 216(%%rax)\n\t" + "movdqu %%xmm10, 232(%%rax)\n\t" + "movdqu %%xmm11, 248(%%rax)\n\t" + "movdqu %%xmm12, 264(%%rax)\n\t" + "movdqu %%xmm13, 280(%%rax)\n\t" + "movdqu %%xmm14, 296(%%rax)\n\t" + "movdqu %%xmm15, 312(%%rax)\n\t" + "movq 56(%%rax), %%rbx\n\t" + "movq 64(%%rax), %%r12\n\t" + "movq 72(%%rax), %%r13\n\t" + "movq 80(%%rax), %%r14\n\t" + "movq 88(%%rax), %%r15\n\t" + "movq 96(%%rax), %%rsi\n\t" + "movq 104(%%rax), %%rdi\n\t" + "movdqu 328(%%rax), %%xmm6\n\t" + "movdqu 344(%%rax), %%xmm7\n\t" + "movdqu 360(%%rax), %%xmm8\n\t" + "movdqu 376(%%rax), %%xmm9\n\t" + "movdqu 392(%%rax), %%xmm10\n\t" + "movdqu 408(%%rax), %%xmm11\n\t" + "movdqu 424(%%rax), %%xmm12\n\t" + "movdqu 440(%%rax), %%xmm13\n\t" + "movdqu 456(%%rax), %%xmm14\n\t" + "movdqu 472(%%rax), %%xmm15\n\t" + "movq %%rsp, %%r11\n\t" + "subq $272, %%rsp\n\t" + "andq $-16, %%rsp\n\t" + "pushq %%r11\n\t" + "pushq %%rax\n\t" + "movq 648(%%rax), %%r10\n\t" + "subq $32, %%rsp\n\t" + "callq *%%r10\n\t" + "addq $32, %%rsp\n\t" + "popq %%rax\n\t" + "popq %%r11\n\t" + "movq %%r11, %%rsp\n\t" + "movq %%rbx, 112(%%rax)\n\t" + "movq %%r12, 120(%%rax)\n\t" + "movq %%r13, 128(%%rax)\n\t" + "movq %%r14, 136(%%rax)\n\t" + "movq %%r15, 144(%%rax)\n\t" + "movq %%rsi, 152(%%rax)\n\t" + "movq %%rdi, 160(%%rax)\n\t" + "movdqu %%xmm6, 488(%%rax)\n\t" + "movdqu %%xmm7, 504(%%rax)\n\t" + "movdqu %%xmm8, 520(%%rax)\n\t" + "movdqu %%xmm9, 536(%%rax)\n\t" + "movdqu %%xmm10, 552(%%rax)\n\t" + "movdqu %%xmm11, 568(%%rax)\n\t" + "movdqu %%xmm12, 584(%%rax)\n\t" + "movdqu %%xmm13, 600(%%rax)\n\t" + "movdqu %%xmm14, 616(%%rax)\n\t" + "movdqu %%xmm15, 632(%%rax)\n\t" + "movq 0(%%rax), %%rbx\n\t" + "movq 8(%%rax), %%r12\n\t" + "movq 16(%%rax), %%r13\n\t" + "movq 24(%%rax), %%r14\n\t" + "movq 32(%%rax), %%r15\n\t" + "movq 40(%%rax), %%rsi\n\t" + "movq 48(%%rax), %%rdi\n\t" + "movdqu 168(%%rax), %%xmm6\n\t" + "movdqu 184(%%rax), %%xmm7\n\t" + "movdqu 200(%%rax), %%xmm8\n\t" + "movdqu 216(%%rax), %%xmm9\n\t" + "movdqu 232(%%rax), %%xmm10\n\t" + "movdqu 248(%%rax), %%xmm11\n\t" + "movdqu 264(%%rax), %%xmm12\n\t" + "movdqu 280(%%rax), %%xmm13\n\t" + "movdqu 296(%%rax), %%xmm14\n\t" + "movdqu 312(%%rax), %%xmm15\n\t" + : + : [state] "r"(&state) + : "memory", "cc", "rax", "rcx", "rdx", "r8", "r9", "r10", "r11", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#else // !OS(WINDOWS): System V AMD64 + asm volatile( + "movq %[state], %%rax\n\t" + "movq %%rbx, 0(%%rax)\n\t" + "movq %%r12, 8(%%rax)\n\t" + "movq %%r13, 16(%%rax)\n\t" + "movq %%r14, 24(%%rax)\n\t" + "movq %%r15, 32(%%rax)\n\t" + "movq 56(%%rax), %%rbx\n\t" + "movq 64(%%rax), %%r12\n\t" + "movq 72(%%rax), %%r13\n\t" + "movq 80(%%rax), %%r14\n\t" + "movq 88(%%rax), %%r15\n\t" + "movq %%rsp, %%r11\n\t" + "subq $256, %%rsp\n\t" + "andq $-16, %%rsp\n\t" + "pushq %%r11\n\t" + "pushq %%rax\n\t" + "movq 648(%%rax), %%r10\n\t" + "callq *%%r10\n\t" + "popq %%rax\n\t" + "popq %%r11\n\t" + "movq %%r11, %%rsp\n\t" + "movq %%rbx, 112(%%rax)\n\t" + "movq %%r12, 120(%%rax)\n\t" + "movq %%r13, 128(%%rax)\n\t" + "movq %%r14, 136(%%rax)\n\t" + "movq %%r15, 144(%%rax)\n\t" + "movq 0(%%rax), %%rbx\n\t" + "movq 8(%%rax), %%r12\n\t" + "movq 16(%%rax), %%r13\n\t" + "movq 24(%%rax), %%r14\n\t" + "movq 32(%%rax), %%r15\n\t" + : + : [state] "r"(&state) + : "memory", "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", "r11", + "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", + "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); +#endif + + uint32_t mask = 0; +#if OS(WINDOWS) + for (unsigned i = 0; i < 7; ++i) { + if (state.observed[i] != state.sentinel[i]) + mask |= 1u << i; + } + for (unsigned i = 0; i < 10; ++i) { + if (memcmp(state.observedXMM[i], state.sentinelXMM[i], 16)) + mask |= 1u << (7 + i); + } +#else + for (unsigned i = 0; i < 5; ++i) { + if (state.observed[i] != state.sentinel[i]) + mask |= 1u << i; + } +#endif + return static_cast(mask); +} + +#elif CPU(ARM64) + +#if OS(LINUX) || OS(FREEBSD) +#define FFI_CANARY_CLOBBER_X18 "x18", +#else +#define FFI_CANARY_CLOBBER_X18 +#endif + +struct FFICanaryStateARM64 { + uint64_t originalX[10]; // x19 - x28 + uint64_t sentinelX[10]; + uint64_t observedX[10]; + unsigned char originalQ[8][16]; // q8 - q15 + unsigned char sentinelQ[8][16]; + unsigned char observedQ[8][16]; + void (*callback)(void); +}; +static_assert(offsetof(FFICanaryStateARM64, originalX) == 0); +static_assert(offsetof(FFICanaryStateARM64, sentinelX) == 80); +static_assert(offsetof(FFICanaryStateARM64, observedX) == 160); +static_assert(offsetof(FFICanaryStateARM64, originalQ) == 240); +static_assert(offsetof(FFICanaryStateARM64, sentinelQ) == 368); +static_assert(offsetof(FFICanaryStateARM64, observedQ) == 496); +static_assert(offsetof(FFICanaryStateARM64, callback) == 624); + +int32_t ffi_canary_call(void (*cb)(void)) +{ + FFICanaryStateARM64 state; + memset(&state, 0, sizeof(state)); + state.callback = cb; + for (unsigned i = 0; i < 10; ++i) + state.sentinelX[i] = 0xC0FFEE00CA11EE00ull + (uint64_t(i + 1) << 8) + (i + 1); + for (unsigned i = 0; i < 8; ++i) { + for (unsigned j = 0; j < 16; ++j) + state.sentinelQ[i][j] = static_cast(0x60 + i * 16 + j); + } + + asm volatile( + "mov x9, %[state]\n\t" + "stp x19, x20, [x9, #0]\n\t" + "stp x21, x22, [x9, #16]\n\t" + "stp x23, x24, [x9, #32]\n\t" + "stp x25, x26, [x9, #48]\n\t" + "stp x27, x28, [x9, #64]\n\t" + "stp q8, q9, [x9, #240]\n\t" + "stp q10, q11, [x9, #272]\n\t" + "stp q12, q13, [x9, #304]\n\t" + "stp q14, q15, [x9, #336]\n\t" + "ldp x19, x20, [x9, #80]\n\t" + "ldp x21, x22, [x9, #96]\n\t" + "ldp x23, x24, [x9, #112]\n\t" + "ldp x25, x26, [x9, #128]\n\t" + "ldp x27, x28, [x9, #144]\n\t" + "ldp q8, q9, [x9, #368]\n\t" + "ldp q10, q11, [x9, #400]\n\t" + "ldp q12, q13, [x9, #432]\n\t" + "ldp q14, q15, [x9, #464]\n\t" + "str x9, [sp, #-16]!\n\t" + "ldr x10, [x9, #624]\n\t" + "blr x10\n\t" + "ldr x9, [sp], #16\n\t" + "stp x19, x20, [x9, #160]\n\t" + "stp x21, x22, [x9, #176]\n\t" + "stp x23, x24, [x9, #192]\n\t" + "stp x25, x26, [x9, #208]\n\t" + "stp x27, x28, [x9, #224]\n\t" + "stp q8, q9, [x9, #496]\n\t" + "stp q10, q11, [x9, #528]\n\t" + "stp q12, q13, [x9, #560]\n\t" + "stp q14, q15, [x9, #592]\n\t" + "ldp x19, x20, [x9, #0]\n\t" + "ldp x21, x22, [x9, #16]\n\t" + "ldp x23, x24, [x9, #32]\n\t" + "ldp x25, x26, [x9, #48]\n\t" + "ldp x27, x28, [x9, #64]\n\t" + "ldp q8, q9, [x9, #240]\n\t" + "ldp q10, q11, [x9, #272]\n\t" + "ldp q12, q13, [x9, #304]\n\t" + "ldp q14, q15, [x9, #336]\n\t" + : + : [state] "r"(&state) + : "memory", "cc", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", + "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", FFI_CANARY_CLOBBER_X18 "x30", + "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + + uint32_t mask = 0; + for (unsigned i = 0; i < 10; ++i) { + if (state.observedX[i] != state.sentinelX[i]) + mask |= 1u << i; + } + for (unsigned i = 0; i < 8; ++i) { + if (memcmp(state.observedQ[i], state.sentinelQ[i], 8)) + mask |= 1u << (10 + i); + } + return static_cast(mask); +} + +#else + +int32_t ffi_canary_call(void (*cb)(void)) +{ + cb(); + return 0; +} + +#endif + +static inline double ffiAlignedVectorProbe() +{ + alignas(16) double buffer[2]; + buffer[0] = 1.0; + buffer[1] = 0.0; +#if CPU(X86_64) + asm volatile( + "movaps (%[buf]), %%xmm0\n\t" + "movaps %%xmm0, (%[buf])\n\t" + : + : [buf] "r"(&buffer[0]) + : "xmm0", "memory"); +#elif CPU(ARM64) + asm volatile( + "str q0, [sp, #-16]!\n\t" + "ld1 { v0.4s }, [%[buf]]\n\t" + "st1 { v0.4s }, [%[buf]]\n\t" + "ldr q0, [sp], #16\n\t" + : + : [buf] "r"(&buffer[0]) + : "v0", "memory"); +#endif + return buffer[0]; +} + +double ffi_align_probe_0(void) +{ + return ffiAlignedVectorProbe(); +} + +double ffi_align_probe_9(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7, int32_t a8) +{ + double folded = ffiAlignedVectorProbe(); + int64_t junk = int64_t(a0) + int64_t(a1) + int64_t(a2) + int64_t(a3) + int64_t(a4) + int64_t(a5) + int64_t(a6) + int64_t(a7) + int64_t(a8); + return folded + (static_cast(junk) * 0.0); +} + +} // extern "C" + +#define FFI_FIXTURE(name) { #name, reinterpret_cast(&name) } + +std::span ffiTestFixtures() +{ + static const FFIFixtureEntry fixtures[] = { + FFI_FIXTURE(ffi_echo_char), + FFI_FIXTURE(ffi_echo_i8), + FFI_FIXTURE(ffi_echo_u8), + FFI_FIXTURE(ffi_echo_i16), + FFI_FIXTURE(ffi_echo_u16), + FFI_FIXTURE(ffi_echo_i32), + FFI_FIXTURE(ffi_echo_u32), + FFI_FIXTURE(ffi_echo_i64), + FFI_FIXTURE(ffi_echo_u64), + FFI_FIXTURE(ffi_echo_f32), + FFI_FIXTURE(ffi_echo_f64), + FFI_FIXTURE(ffi_echo_bool), + FFI_FIXTURE(ffi_echo_ptr), + FFI_FIXTURE(ffi_echo_cstring), + FFI_FIXTURE(ffi_echo_jsvalue), + FFI_FIXTURE(ffi_widen_char), + FFI_FIXTURE(ffi_widen_i8), + FFI_FIXTURE(ffi_widen_u8), + FFI_FIXTURE(ffi_widen_i16), + FFI_FIXTURE(ffi_widen_u16), + FFI_FIXTURE(ffi_ret_null_ptr), + FFI_FIXTURE(ffi_ret_two_as_bool), + FFI_FIXTURE(ffi_ret_neg_one_i8), + FFI_FIXTURE(ffi_ret_neg_one_i16), + FFI_FIXTURE(ffi_ret_neg_one_i32), + FFI_FIXTURE(ffi_ret_neg_one_i64), + FFI_FIXTURE(ffi_ret_neg_one_u8), + FFI_FIXTURE(ffi_ret_neg_one_u16), + FFI_FIXTURE(ffi_ret_neg_one_u32), + FFI_FIXTURE(ffi_ret_neg_one_u64), + FFI_FIXTURE(ffi_ret_nan_f32), + FFI_FIXTURE(ffi_ret_impure_nan_f64), + FFI_FIXTURE(ffi_ret_neg_zero_f64), + FFI_FIXTURE(ffi_ret_denormal_f32), + FFI_FIXTURE(ffi_ret_inf_f64), + FFI_FIXTURE(ffi_add_i32), + FFI_FIXTURE(ffi_add_f64), + FFI_FIXTURE(ffi_add_i64), + FFI_FIXTURE(ffi_add_u64), + FFI_FIXTURE(ffi_add_f32), + FFI_FIXTURE(ffi_sum_i32_0), + FFI_FIXTURE(ffi_sum_i32_1), + FFI_FIXTURE(ffi_sum_i32_2), + FFI_FIXTURE(ffi_sum_i32_4), + FFI_FIXTURE(ffi_sum_i32_6), + FFI_FIXTURE(ffi_sum_i32_7), + FFI_FIXTURE(ffi_sum_i32_8), + FFI_FIXTURE(ffi_sum_i32_9), + FFI_FIXTURE(ffi_sum_i32_12), + FFI_FIXTURE(ffi_sum_i32_16), + FFI_FIXTURE(ffi_sum_f64_1), + FFI_FIXTURE(ffi_sum_f64_2), + FFI_FIXTURE(ffi_sum_f64_7), + FFI_FIXTURE(ffi_sum_f64_8), + FFI_FIXTURE(ffi_sum_f64_9), + FFI_FIXTURE(ffi_sum_f64_12), + FFI_FIXTURE(ffi_sum_u8_10), + FFI_FIXTURE(ffi_sum_u8_12), + FFI_FIXTURE(ffi_sum_i16_10), + FFI_FIXTURE(ffi_sum_i16_12), + FFI_FIXTURE(ffi_mix_1), + FFI_FIXTURE(ffi_mix_2), + FFI_FIXTURE(ffi_mix_3), + FFI_FIXTURE(ffi_mix_4), + FFI_FIXTURE(ffi_mix_5), + FFI_FIXTURE(ffi_mix_6), + FFI_FIXTURE(ffi_mix_7), + FFI_FIXTURE(ffi_mix_8), + FFI_FIXTURE(ffi_ptr_write_u32), + FFI_FIXTURE(ffi_ptr_read_u32), + FFI_FIXTURE(ffi_strlen), + FFI_FIXTURE(ffi_ptr_identity), + FFI_FIXTURE(ffi_high_ptr), + FFI_FIXTURE(ffi_view_byte_length), + FFI_FIXTURE(ffi_view_last_byte), + FFI_FIXTURE(ffi_bench_noop), + FFI_FIXTURE(ffi_bench_string), + FFI_FIXTURE(ffi_bench_hash), + FFI_FIXTURE(ffi_call_cb_i32), + FFI_FIXTURE(ffi_call_cb_f64_x8), + FFI_FIXTURE(ffi_call_cb_mix), + FFI_FIXTURE(ffi_call_cb_void), + FFI_FIXTURE(ffi_call_cb_reentrant), + FFI_FIXTURE(ffi_call_cb_i32_x9), + FFI_FIXTURE(ffi_call_cb_f64_x9), + FFI_FIXTURE(ffi_call_cb_u8_x10), + FFI_FIXTURE(ffi_call_cb_ret_i8), + FFI_FIXTURE(ffi_call_cb_ret_u8), + FFI_FIXTURE(ffi_call_cb_ret_i64), + FFI_FIXTURE(ffi_call_cb_ret_u64), + FFI_FIXTURE(ffi_call_cb_ret_bool), + FFI_FIXTURE(ffi_call_cb_ret_f32), + FFI_FIXTURE(ffi_call_cb_ret_f64), + FFI_FIXTURE(ffi_call_cb_ret_cstring), + FFI_FIXTURE(ffi_call_cb_ret_ptr), + FFI_FIXTURE(ffi_call_cb_then_read_u32), + FFI_FIXTURE(ffi_canary_call), + FFI_FIXTURE(ffi_sum_i32_x10), + FFI_FIXTURE(ffi_call_cb_from_thread), + FFI_FIXTURE(ffi_align_probe_0), + FFI_FIXTURE(ffi_align_probe_9), + }; + return std::span(fixtures); +} + +#undef FFI_FIXTURE + +const FFIFixtureEntry* ffiTestFixtureNamed(const char* name) +{ + for (const auto& entry : ffiTestFixtures()) { + if (!strcmp(entry.name, name)) + return &entry; + } + return nullptr; +} + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/tests/FFITestFixtures.h b/Source/JavaScriptCore/ffi/tests/FFITestFixtures.h new file mode 100644 index 0000000000000..fdb640bafad53 --- /dev/null +++ b/Source/JavaScriptCore/ffi/tests/FFITestFixtures.h @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include + +#if USE(BUN_JSC_ADDITIONS) + +#include "JSExportMacros.h" +#include +#include + +typedef int64_t jsvalue_slot; // an encoded JSValue crossing the boundary as an int64 + +extern "C" { + +JS_EXPORT_PRIVATE signed char ffi_echo_char(signed char); +JS_EXPORT_PRIVATE int8_t ffi_echo_i8(int8_t); +JS_EXPORT_PRIVATE uint8_t ffi_echo_u8(uint8_t); +JS_EXPORT_PRIVATE int16_t ffi_echo_i16(int16_t); +JS_EXPORT_PRIVATE uint16_t ffi_echo_u16(uint16_t); +JS_EXPORT_PRIVATE int32_t ffi_echo_i32(int32_t); +JS_EXPORT_PRIVATE uint32_t ffi_echo_u32(uint32_t); +JS_EXPORT_PRIVATE int64_t ffi_echo_i64(int64_t); +JS_EXPORT_PRIVATE uint64_t ffi_echo_u64(uint64_t); +JS_EXPORT_PRIVATE float ffi_echo_f32(float); +JS_EXPORT_PRIVATE double ffi_echo_f64(double); +JS_EXPORT_PRIVATE bool ffi_echo_bool(bool); +JS_EXPORT_PRIVATE void* ffi_echo_ptr(void*); +JS_EXPORT_PRIVATE const char* ffi_echo_cstring(const char*); +JS_EXPORT_PRIVATE jsvalue_slot ffi_echo_jsvalue(jsvalue_slot); + +JS_EXPORT_PRIVATE int64_t ffi_widen_char(signed char); +JS_EXPORT_PRIVATE int64_t ffi_widen_i8(int8_t); +JS_EXPORT_PRIVATE int64_t ffi_widen_u8(uint8_t); +JS_EXPORT_PRIVATE int64_t ffi_widen_i16(int16_t); +JS_EXPORT_PRIVATE int64_t ffi_widen_u16(uint16_t); +JS_EXPORT_PRIVATE void* ffi_ret_null_ptr(void); +JS_EXPORT_PRIVATE int8_t ffi_ret_two_as_bool(void); +JS_EXPORT_PRIVATE int8_t ffi_ret_neg_one_i8(void); +JS_EXPORT_PRIVATE int16_t ffi_ret_neg_one_i16(void); +JS_EXPORT_PRIVATE int32_t ffi_ret_neg_one_i32(void); +JS_EXPORT_PRIVATE int64_t ffi_ret_neg_one_i64(void); +JS_EXPORT_PRIVATE uint8_t ffi_ret_neg_one_u8(void); +JS_EXPORT_PRIVATE uint16_t ffi_ret_neg_one_u16(void); +JS_EXPORT_PRIVATE uint32_t ffi_ret_neg_one_u32(void); +JS_EXPORT_PRIVATE uint64_t ffi_ret_neg_one_u64(void); + +JS_EXPORT_PRIVATE float ffi_ret_nan_f32(void); // NaN with a non-canonical payload; must be purified before boxing. +JS_EXPORT_PRIVATE double ffi_ret_impure_nan_f64(void); // 0x7ff0000000000001; must be purified before boxing. +JS_EXPORT_PRIVATE double ffi_ret_neg_zero_f64(void); +JS_EXPORT_PRIVATE float ffi_ret_denormal_f32(void); // smallest positive denormal (bits 0x00000001). +JS_EXPORT_PRIVATE double ffi_ret_inf_f64(void); + +JS_EXPORT_PRIVATE int32_t ffi_add_i32(int32_t, int32_t); +JS_EXPORT_PRIVATE double ffi_add_f64(double, double); +JS_EXPORT_PRIVATE int64_t ffi_add_i64(int64_t, int64_t); +JS_EXPORT_PRIVATE uint64_t ffi_add_u64(uint64_t, uint64_t); +JS_EXPORT_PRIVATE float ffi_add_f32(float, float); + +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_0(void); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_1(int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_2(int32_t, int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_4(int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_6(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_7(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_8(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_9(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_12(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_16(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE double ffi_sum_f64_1(double); +JS_EXPORT_PRIVATE double ffi_sum_f64_2(double, double); +JS_EXPORT_PRIVATE double ffi_sum_f64_7(double, double, double, double, double, double, double); +JS_EXPORT_PRIVATE double ffi_sum_f64_8(double, double, double, double, double, double, double, double); +JS_EXPORT_PRIVATE double ffi_sum_f64_9(double, double, double, double, double, double, double, double, double); +JS_EXPORT_PRIVATE double ffi_sum_f64_12(double, double, double, double, double, double, double, double, double, double, double, double); +JS_EXPORT_PRIVATE int64_t ffi_sum_u8_10(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_u8_12(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i16_10(int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i16_12(int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t, int16_t); + +JS_EXPORT_PRIVATE double ffi_mix_1(int32_t, double, int64_t, float, void*, uint8_t, double, int16_t, double, int32_t); +JS_EXPORT_PRIVATE double ffi_mix_2(float, int32_t, float, int32_t, float, int32_t, float, int32_t, float, int32_t); +JS_EXPORT_PRIVATE double ffi_mix_3(double, double, double, double, double, double, double, double, int32_t); +JS_EXPORT_PRIVATE double ffi_mix_4(int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, double, int64_t, double); +JS_EXPORT_PRIVATE double ffi_mix_5(uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t); +JS_EXPORT_PRIVATE double ffi_mix_6(bool, bool, int32_t, bool, double, bool, float, bool, bool, bool, bool, bool, bool); +JS_EXPORT_PRIVATE double ffi_mix_7(void*, signed char, void*, signed char, void*, signed char, void*, signed char, void*, signed char); +JS_EXPORT_PRIVATE double ffi_mix_8(float, double, float, double, float, double, float, double, float, double, float, double); + +JS_EXPORT_PRIVATE void ffi_ptr_write_u32(uint32_t*, uint32_t); +JS_EXPORT_PRIVATE uint32_t ffi_ptr_read_u32(uint32_t*); +JS_EXPORT_PRIVATE uint64_t ffi_strlen(const char*); +JS_EXPORT_PRIVATE void* ffi_ptr_identity(void*); +JS_EXPORT_PRIVATE void* ffi_high_ptr(void); // returns (void*)0x00007fffdeadbee0 +JS_EXPORT_PRIVATE uint64_t ffi_view_byte_length(const uint8_t* ptr, uint64_t length); +JS_EXPORT_PRIVATE int32_t ffi_view_last_byte(const uint8_t* ptr, uint64_t length); +JS_EXPORT_PRIVATE void ffi_bench_noop(void); +JS_EXPORT_PRIVATE const char* ffi_bench_string(void); +JS_EXPORT_PRIVATE uint32_t ffi_bench_hash(const uint8_t* ptr, uint32_t length); + +JS_EXPORT_PRIVATE int32_t ffi_call_cb_i32(int32_t (*cb)(int32_t), int32_t x); +JS_EXPORT_PRIVATE double ffi_call_cb_f64_x8(double (*cb)(double, double, double, double, double, double, double, double), double, double, double, double, double, double, double, double); +JS_EXPORT_PRIVATE double ffi_call_cb_mix(double (*cb)(int32_t, double, int64_t, float, void*), int32_t, double, int64_t, float, void*); +JS_EXPORT_PRIVATE void ffi_call_cb_void(void (*cb)(void)); +JS_EXPORT_PRIVATE int32_t ffi_call_cb_reentrant(int32_t (*cb)(int32_t), int32_t depth); // returns sum of cb(i) for i in [0, depth) +JS_EXPORT_PRIVATE int64_t ffi_call_cb_i32_x9(int64_t (*cb)(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t), int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE double ffi_call_cb_f64_x9(double (*cb)(double, double, double, double, double, double, double, double, double), double, double, double, double, double, double, double, double, double); +JS_EXPORT_PRIVATE int64_t ffi_call_cb_u8_x10(int64_t (*cb)(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t), uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t); +JS_EXPORT_PRIVATE int64_t ffi_sum_i32_x10(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); +JS_EXPORT_PRIVATE void ffi_call_cb_from_thread(void (*cb)(int32_t, int64_t, uint64_t, double), int32_t a, int64_t b, uint64_t c, double d); +JS_EXPORT_PRIVATE int64_t ffi_call_cb_ret_i8(int8_t (*cb)(void)); // returns (int64_t) of the callback result +JS_EXPORT_PRIVATE int64_t ffi_call_cb_ret_u8(uint8_t (*cb)(void)); +JS_EXPORT_PRIVATE int64_t ffi_call_cb_ret_i64(int64_t (*cb)(void)); +JS_EXPORT_PRIVATE uint64_t ffi_call_cb_ret_u64(uint64_t (*cb)(void)); +JS_EXPORT_PRIVATE int32_t ffi_call_cb_ret_bool(bool (*cb)(void)); // returns 10 if the callback returned true, else 20 +JS_EXPORT_PRIVATE float ffi_call_cb_ret_f32(float (*cb)(void)); +JS_EXPORT_PRIVATE double ffi_call_cb_ret_f64(double (*cb)(void)); +JS_EXPORT_PRIVATE const char* ffi_call_cb_ret_cstring(const char* (*cb)(void)); +JS_EXPORT_PRIVATE void* ffi_call_cb_ret_ptr(void* (*cb)(void)); +JS_EXPORT_PRIVATE uint32_t ffi_call_cb_then_read_u32(uint32_t (*cb)(void), uint32_t* p); // calls cb() (which may GC), then returns *p + +JS_EXPORT_PRIVATE int32_t ffi_canary_call(void (*cb)(void)); + +JS_EXPORT_PRIVATE double ffi_align_probe_0(void); +JS_EXPORT_PRIVATE double ffi_align_probe_9(int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t); + +} // extern "C" + +struct FFIFixtureEntry { + const char* name; + void* address; +}; + +JS_EXPORT_PRIVATE std::span ffiTestFixtures(); + +JS_EXPORT_PRIVATE const FFIFixtureEntry* ffiTestFixtureNamed(const char* name); + +#endif // USE(BUN_JSC_ADDITIONS) diff --git a/Source/JavaScriptCore/ffi/tests/testFFI.cpp b/Source/JavaScriptCore/ffi/tests/testFFI.cpp new file mode 100644 index 0000000000000..5724198e1470c --- /dev/null +++ b/Source/JavaScriptCore/ffi/tests/testFFI.cpp @@ -0,0 +1,2358 @@ +/* + * Copyright (C) 2026 Anthropic PBC. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" +#include "FFITestFixtures.h" + +#include "InitializeThreading.h" +#include "Options.h" +#include +#include +#include +#include +#include +#include +#include + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +#if USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) && USE(JSVALUE64) && (CPU(X86_64) || CPU(ARM64)) + +#include "CallData.h" +#include "CallFrameInlines.h" +#include "ConstructData.h" +#include "ErrorInstance.h" +#include "FFICallingConvention.h" +#include "FFIContext.h" +#include "FFIConversions.h" +#include "FFIInvokeThunk.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "FPRInfo.h" +#include "GPRInfo.h" +#include "JSArrayBufferView.h" +#include "JSBigInt.h" +#include "JSBigIntInlines.h" +#include "JSCInlines.h" +#include "JSCJSValueInlines.h" +#include "JSFFICallback.h" +#include "JSFFIFunction.h" +#include "JSFunction.h" +#include "JSGlobalObject.h" +#include "JSGlobalObjectInlines.h" +#include "JSLock.h" +#include "JSString.h" +#include "JSTypedArrays.h" +#include "MacroAssemblerCodeRef.h" +#include "ObjectConstructor.h" +#include "Protect.h" +#include "PureNaN.h" +#include "Symbol.h" +#include "TopExceptionScope.h" +#include "TypedArrayInlines.h" +#include "VM.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace JSC; + +namespace { + +static unsigned s_failureCount; +static unsigned s_checkCount; + +#define FFI_CHECK(condition) do { \ + s_checkCount++; \ + if (!(condition)) { \ + s_failureCount++; \ + dataLogLn(" FAIL: ", __FILE__, ":", __LINE__, ": ", #condition); \ + } \ + } while (false) + +#define FFI_CHECK_EQ(actualExpression, expectedExpression) do { \ + s_checkCount++; \ + auto _ffiActual = (actualExpression); \ + auto _ffiExpected = (expectedExpression); \ + if (!(_ffiActual == static_cast(_ffiExpected))) { \ + s_failureCount++; \ + dataLogLn(" FAIL: ", __FILE__, ":", __LINE__, ": ", #actualExpression, " == ", #expectedExpression, " (actual ", _ffiActual, ", expected ", _ffiExpected, ")"); \ + } \ + } while (false) + +#define FFI_CHECK_EQ_HEX(actualExpression, expectedExpression) do { \ + s_checkCount++; \ + uint64_t _ffiActual = static_cast(actualExpression); \ + uint64_t _ffiExpected = static_cast(expectedExpression); \ + if (_ffiActual != _ffiExpected) { \ + s_failureCount++; \ + dataLogLn(" FAIL: ", __FILE__, ":", __LINE__, ": ", #actualExpression, " == ", #expectedExpression, " (actual ", RawHex(_ffiActual), ", expected ", RawHex(_ffiExpected), ")"); \ + } \ + } while (false) + +static const char* s_filter; + +static bool shouldRun(const char* testName) +{ + return !s_filter || WTF::findIgnoringASCIICaseWithoutLength(testName, s_filter) != WTF::notFound; +} + +#define RUN(test) do { \ + if (!shouldRun(#test)) \ + break; \ + unsigned failuresBefore = s_failureCount; \ + dataLogLn(#test, "..."); \ + test; \ + dataLogLn(s_failureCount == failuresBefore ? "PASS: " : "FAIL: ", #test); \ + } while (false) + +static RefPtr s_vm; +static JSGlobalObject* s_globalObject; + +static FFI::FFIContext& ffiContext() +{ + return s_globalObject->ffiContext(); +} + +static FFI::StringArena* testStringArena() +{ + return &ffiContext().stringArena(); +} + +static const char* typeNameForLog(FFI::Type type) +{ + return FFI::name(type).characters(); +} + +static void testTypeTraits() +{ + using FFI::Type; + + FFI_CHECK_EQ(FFI::numberOfTypes, 22u); + + FFI_CHECK_EQ(static_cast(Type::Char), 0u); + FFI_CHECK_EQ(static_cast(Type::Int8), 1u); + FFI_CHECK_EQ(static_cast(Type::Uint8), 2u); + FFI_CHECK_EQ(static_cast(Type::Int16), 3u); + FFI_CHECK_EQ(static_cast(Type::Uint16), 4u); + FFI_CHECK_EQ(static_cast(Type::Int32), 5u); + FFI_CHECK_EQ(static_cast(Type::Uint32), 6u); + FFI_CHECK_EQ(static_cast(Type::Int64), 7u); + FFI_CHECK_EQ(static_cast(Type::Uint64), 8u); + FFI_CHECK_EQ(static_cast(Type::Double), 9u); + FFI_CHECK_EQ(static_cast(Type::Float), 10u); + FFI_CHECK_EQ(static_cast(Type::Bool), 11u); + FFI_CHECK_EQ(static_cast(Type::Pointer), 12u); + FFI_CHECK_EQ(static_cast(Type::Void), 13u); + FFI_CHECK_EQ(static_cast(Type::CString), 14u); + FFI_CHECK_EQ(static_cast(Type::Int64Fast), 15u); + FFI_CHECK_EQ(static_cast(Type::Uint64Fast), 16u); + FFI_CHECK_EQ(static_cast(Type::Function), 17u); + FFI_CHECK_EQ(static_cast(Type::RESERVED_WasNapiEnv), 18u); + FFI_CHECK_EQ(static_cast(Type::JSValue), 19u); + FFI_CHECK_EQ(static_cast(Type::Buffer), 20u); + FFI_CHECK_EQ(static_cast(Type::BufferLength), 21u); + + static const struct { Type type; ASCIILiteral name; } names[] = { + { Type::Char, "char"_s }, { Type::Int8, "i8"_s }, { Type::Uint8, "u8"_s }, { Type::Int16, "i16"_s }, + { Type::Uint16, "u16"_s }, { Type::Int32, "i32"_s }, { Type::Uint32, "u32"_s }, { Type::Int64, "i64"_s }, + { Type::Uint64, "u64"_s }, { Type::Double, "f64"_s }, { Type::Float, "f32"_s }, { Type::Bool, "bool"_s }, + { Type::Pointer, "ptr"_s }, { Type::Void, "void"_s }, { Type::CString, "cstring"_s }, { Type::Int64Fast, "i64_fast"_s }, + { Type::Uint64Fast, "u64_fast"_s }, { Type::Function, "function"_s }, { Type::JSValue, "jsvalue"_s }, { Type::Buffer, "buffer"_s }, + { Type::BufferLength, "buffer_length"_s }, + }; + for (auto& entry : names) { + FFI_CHECK(!strcmp(FFI::name(entry.type).characters(), entry.name.characters())); + std::optional parsed = FFI::parseType(StringView(entry.name)); + FFI_CHECK(parsed.has_value()); + if (parsed) + FFI_CHECK(*parsed == entry.type); + } + + static const struct { ASCIILiteral alias; Type type; } aliases[] = { + { "int8_t"_s, Type::Int8 }, { "uint8_t"_s, Type::Uint8 }, { "int16_t"_s, Type::Int16 }, { "uint16_t"_s, Type::Uint16 }, + { "int32_t"_s, Type::Int32 }, { "int"_s, Type::Int32 }, { "c_int"_s, Type::Int32 }, { "uint32_t"_s, Type::Uint32 }, + { "c_uint"_s, Type::Uint32 }, { "int64_t"_s, Type::Int64 }, { "isize"_s, Type::Int64 }, { "uint64_t"_s, Type::Uint64 }, + { "usize"_s, Type::Uint64 }, { "size_t"_s, Type::Uint64 }, { "double"_s, Type::Double }, { "float"_s, Type::Float }, + { "void*"_s, Type::Pointer }, { "pointer"_s, Type::Pointer }, { "char*"_s, Type::Pointer }, { "callback"_s, Type::Function }, + { "fn"_s, Type::Function }, { "napi_value"_s, Type::JSValue }, + { "buffer_bytelength"_s, Type::BufferLength }, + }; + for (auto& entry : aliases) { + std::optional parsed = FFI::parseType(StringView(entry.alias)); + FFI_CHECK(parsed.has_value()); + if (parsed) + FFI_CHECK(*parsed == entry.type); + } + FFI_CHECK(!FFI::parseType(StringView("int128"_s)).has_value()); + FFI_CHECK(!FFI::parseType(StringView(""_s)).has_value()); + FFI_CHECK(!FFI::parseType(StringView("I32"_s)).has_value()); + + for (unsigned tag = 0; tag < FFI::numberOfTypes; ++tag) { + Type type = static_cast(tag); + FFI::ArgClass expected = FFI::ArgClass::Int; + if (type == Type::Void) + expected = FFI::ArgClass::Void; + else if (type == Type::Float) + expected = FFI::ArgClass::Float; + else if (type == Type::Double) + expected = FFI::ArgClass::Double; + FFI_CHECK(FFI::argClass(type) == expected); + FFI_CHECK_EQ(FFI::isValidReturnType(type), type != Type::RESERVED_WasNapiEnv && type != Type::Buffer && type != Type::BufferLength); + FFI_CHECK_EQ(FFI::isValidArgumentType(type), type != Type::Void && type != Type::RESERVED_WasNapiEnv); + } + + FFI_CHECK(FFI::isSigned(Type::Char)); + FFI_CHECK(FFI::isSigned(Type::Int8)); + FFI_CHECK(FFI::isSigned(Type::Int16)); + FFI_CHECK(FFI::isSigned(Type::Int32)); + FFI_CHECK(FFI::isSigned(Type::Int64)); + FFI_CHECK(FFI::isSigned(Type::Int64Fast)); + FFI_CHECK(!FFI::isSigned(Type::Uint8)); + FFI_CHECK(!FFI::isSigned(Type::Uint16)); + FFI_CHECK(!FFI::isSigned(Type::Uint32)); + FFI_CHECK(!FFI::isSigned(Type::Uint64)); + FFI_CHECK(!FFI::isSigned(Type::Uint64Fast)); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Char), 1u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Int8), 1u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Uint8), 1u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Bool), 1u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Int16), 2u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Uint16), 2u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Int32), 4u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Uint32), 4u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Float), 4u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Int64), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Uint64), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Int64Fast), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Uint64Fast), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Double), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Pointer), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::CString), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Function), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::Buffer), 8u); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::BufferLength), 8u); + FFI_CHECK(!FFI::isSigned(Type::BufferLength)); + FFI_CHECK_EQ(FFI::nativeSizeInBytes(Type::JSValue), 8u); + FFI_CHECK_EQ(FFI::slotSize, 8u); +} + +static void testSignatures() +{ + using FFI::Type; + + Vector args { Type::Int32, Type::Double }; + RefPtr a = FFI::Signature::tryCreate(args.span(), Type::Double); + RefPtr b = FFI::Signature::tryCreate(args.span(), Type::Double); + FFI_CHECK(!!a); + FFI_CHECK(!!b); + if (!a || !b) + return; + + FFI_CHECK(a.get() == b.get()); + FFI_CHECK(*a == *b); + FFI_CHECK_EQ(a->hash(), b->hash()); + Ref viaRegistry = FFI::SignatureRegistry::singleton().intern(args.span(), Type::Double); + FFI_CHECK(viaRegistry.ptr() == a.get()); + FFI_CHECK(a->toString() == "f64(i32,f64)"_s); + FFI_CHECK_EQ(a->argumentCount(), 2u); + FFI_CHECK_EQ(a->slotCount(), 3u); + FFI_CHECK_EQ(a->slotBufferBytes(), 24u); + FFI_CHECK(a->argumentType(0) == Type::Int32); + FFI_CHECK(a->argumentType(1) == Type::Double); + FFI_CHECK(a->returnType() == Type::Double); + + RefPtr c = FFI::Signature::tryCreate(args.span(), Type::Float); + FFI_CHECK(!!c); + if (c) { + FFI_CHECK(c.get() != a.get()); + FFI_CHECK(!(*c == *a)); + FFI_CHECK(c->toString() == "f32(i32,f64)"_s); + } + + Vector reversed { Type::Double, Type::Int32 }; + RefPtr d = FFI::Signature::tryCreate(reversed.span(), Type::Double); + FFI_CHECK(!!d); + if (d) { + FFI_CHECK(d.get() != a.get()); + FFI_CHECK(d->toString() == "f64(f64,i32)"_s); + } + + Vector withJSValue { Type::Int32, Type::JSValue }; + RefPtr jsvalueSignature = FFI::Signature::tryCreate(withJSValue.span(), Type::JSValue); + FFI_CHECK(!!jsvalueSignature); + if (jsvalueSignature) { + FFI_CHECK_EQ(jsvalueSignature->argumentCount(), 2u); + FFI_CHECK_EQ(jsvalueSignature->slotCount(), 3u); + FFI_CHECK(jsvalueSignature->toString() == "jsvalue(i32,jsvalue)"_s); + } + + RefPtr empty = FFI::Signature::tryCreate({ }, Type::Void); + FFI_CHECK(!!empty); + if (empty) { + FFI_CHECK_EQ(empty->argumentCount(), 0u); + FFI_CHECK_EQ(empty->slotCount(), 1u); + FFI_CHECK(empty->toString() == "void()"_s); + } + + Vector thirtyTwo(FillWith { }, 32, Type::Int32); + Vector thirtyThree(FillWith { }, 33, Type::Int32); + FFI_CHECK(!!FFI::Signature::tryCreate(thirtyTwo.span(), Type::Int64)); + FFI_CHECK(!FFI::Signature::tryCreate(thirtyThree.span(), Type::Int64)); + FFI_CHECK_EQ(FFI::Signature::maxArguments, 32u); + + Vector voidArg { Type::Int32, Type::Void }; + FFI_CHECK(!FFI::Signature::tryCreate(voidArg.span(), Type::Int32)); + Vector oneInt { Type::Int32 }; + Vector reservedArg { Type::RESERVED_WasNapiEnv }; + FFI_CHECK(!FFI::Signature::tryCreate(oneInt.span(), Type::RESERVED_WasNapiEnv)); + FFI_CHECK(!FFI::Signature::tryCreate(reservedArg.span(), Type::Int32)); + FFI_CHECK(!FFI::Signature::tryCreate(oneInt.span(), Type::Buffer)); + Vector bufferLengthArg { Type::Pointer, Type::BufferLength }; + FFI_CHECK(!FFI::Signature::tryCreate(oneInt.span(), Type::BufferLength)); + FFI_CHECK(!!FFI::Signature::tryCreate(bufferLengthArg.span(), Type::Uint64)); + FFI_CHECK(!!FFI::Signature::tryCreate(oneInt.span(), Type::JSValue)); + FFI_CHECK(!!FFI::Signature::tryCreate(oneInt.span(), Type::Void)); + + Vector everyArg { + Type::Char, Type::Int8, Type::Uint8, Type::Int16, Type::Uint16, Type::Int32, Type::Uint32, + Type::Int64, Type::Uint64, Type::Double, Type::Float, Type::Bool, Type::Pointer, Type::CString, + Type::Int64Fast, Type::Uint64Fast, Type::Function, Type::JSValue, Type::Buffer, Type::BufferLength, + }; + RefPtr every = FFI::Signature::tryCreate(everyArg.span(), Type::Char); + FFI_CHECK(!!every); + if (every) { + FFI_CHECK_EQ(every->argumentCount(), 20u); + FFI_CHECK(every->toString() == "char(char,i8,u8,i16,u16,i32,u32,i64,u64,f64,f32,bool,ptr,cstring,i64_fast,u64_fast,function,jsvalue,buffer,buffer_length)"_s); + } +} + +struct GoldenCase { + ASCIILiteral name; + Vector arguments; + FFI::Type returnType; + ASCIILiteral sysv64; + ASCIILiteral aapcs64EightByteSlots; + ASCIILiteral darwinARM64Packed; + ASCIILiteral win64; +}; + +struct ParsedGolden { + struct Location { + FFI::ArgLocation::Kind kind; + unsigned index; // register index or stack offset + }; + Vector locations; + unsigned stackBytes { 0 }; + FFI::ArgClass returnClass { FFI::ArgClass::Void }; +}; + +static ParsedGolden parseGolden(ASCIILiteral literal) +{ + ParsedGolden result; + String string(literal); + for (StringView token : StringView(string).split(' ')) { + if (token.isEmpty()) + continue; + char16_t head = token[0]; + StringView rest = token.substring(1); + if (head == 'g' || head == 'f' || head == 's') { + std::optional value = parseInteger(rest); + RELEASE_ASSERT(value); + ParsedGolden::Location location; + if (head == 'g') + location.kind = FFI::ArgLocation::Kind::GPR; + else if (head == 'f') + location.kind = FFI::ArgLocation::Kind::FPR; + else + location.kind = FFI::ArgLocation::Kind::Stack; + location.index = *value; + result.locations.append(location); + continue; + } + if (head == '#') { + std::optional value = parseInteger(rest); + RELEASE_ASSERT(value); + result.stackBytes = *value; + continue; + } + if (head == 'r') { + RELEASE_ASSERT(rest.length() == 1); + switch (rest[0]) { + case 'v': result.returnClass = FFI::ArgClass::Void; break; + case 'i': result.returnClass = FFI::ArgClass::Int; break; + case 'f': result.returnClass = FFI::ArgClass::Float; break; + case 'd': result.returnClass = FFI::ArgClass::Double; break; + default: RELEASE_ASSERT_NOT_REACHED(); + } + continue; + } + RELEASE_ASSERT_NOT_REACHED(); + } + return result; +} + +static const char* nativeCCName(FFI::NativeCC cc, FFI::StackPacking packing) +{ + switch (cc) { + case FFI::NativeCC::SysV64: return "SysV64"; + case FFI::NativeCC::AAPCS64: + return packing == FFI::StackPacking::Natural ? "AAPCS64(Darwin natural packing)" : "AAPCS64(8-byte slots)"; + case FFI::NativeCC::Win64: return "Win64"; + } + return "?"; +} + +static void checkLayoutAgainstGolden(const GoldenCase& golden, FFI::NativeCC cc, FFI::StackPacking packing, ASCIILiteral expectedLiteral) +{ + ParsedGolden expected = parseGolden(expectedLiteral); + RefPtr signature = FFI::Signature::tryCreate(golden.arguments.span(), golden.returnType); + FFI_CHECK(!!signature); + if (!signature) + return; + + FFI::CallLayout layout = FFI::computeCallLayout(cc, packing, *signature, FFI::Direction::Outgoing); + bool ok = true; + ok &= layout.cc == cc; + ok &= layout.packing == packing; + if (FFI::stackPackingForNativeCC(cc) == packing) { + FFI::CallLayout hostSelected = FFI::computeCallLayout(cc, *signature, FFI::Direction::Outgoing); + ok &= hostSelected.stackBytes == layout.stackBytes; + ok &= hostSelected.packing == packing; + ok &= hostSelected.arguments.size() == layout.arguments.size(); + for (unsigned i = 0; ok && i < layout.arguments.size(); ++i) { + ok &= hostSelected.arguments[i].kind == layout.arguments[i].kind; + ok &= hostSelected.arguments[i].regIndex == layout.arguments[i].regIndex; + ok &= hostSelected.arguments[i].stackOffset == layout.arguments[i].stackOffset; + } + } + ok &= layout.arguments.size() == expected.locations.size(); + ok &= layout.arguments.size() == golden.arguments.size(); + ok &= layout.stackBytes == expected.stackBytes; + ok &= !(layout.stackBytes % 16); + ok &= layout.returnClass == expected.returnClass; + ok &= layout.returnClass == FFI::argClass(golden.returnType); + if (ok) { + for (unsigned i = 0; i < layout.arguments.size(); ++i) { + const FFI::ArgLocation& location = layout.arguments[i]; + const ParsedGolden::Location& want = expected.locations[i]; + ok &= location.type == golden.arguments[i]; + ok &= location.kind == want.kind; + if (!ok) + break; + switch (location.kind) { + case FFI::ArgLocation::Kind::GPR: + case FFI::ArgLocation::Kind::FPR: + ok &= location.regIndex == want.index; + break; + case FFI::ArgLocation::Kind::Stack: + ok &= location.stackOffset == want.index; + ok &= location.stackOffset + FFI::nativeSizeInBytes(location.type) <= layout.stackBytes; + break; + } + if (!ok) + break; + } + } + s_checkCount++; + if (!ok) { + s_failureCount++; + dataLogLn(" FAIL: layout of ", golden.name.characters(), " for ", nativeCCName(cc, packing), " -- expected \"", expectedLiteral.characters(), "\""); + dataLog(" got:"); + for (const FFI::ArgLocation& location : layout.arguments) { + switch (location.kind) { + case FFI::ArgLocation::Kind::GPR: dataLog(" g", static_cast(location.regIndex)); break; + case FFI::ArgLocation::Kind::FPR: dataLog(" f", static_cast(location.regIndex)); break; + case FFI::ArgLocation::Kind::Stack: dataLog(" s", location.stackOffset); break; + } + } + dataLogLn(" #", layout.stackBytes, " returnClass=", static_cast(layout.returnClass), " cc=", static_cast(layout.cc)); + } + + FFI::CallLayout incoming = FFI::computeCallLayout(cc, packing, *signature, FFI::Direction::Incoming); + FFI_CHECK_EQ(incoming.arguments.size(), layout.arguments.size()); + for (unsigned i = 0; i < layout.arguments.size() && i < incoming.arguments.size(); ++i) { + FFI_CHECK(incoming.arguments[i].kind == layout.arguments[i].kind); + if (layout.arguments[i].kind == FFI::ArgLocation::Kind::Stack) { + FFI_CHECK_EQ(unsigned(incoming.arguments[i].stackOffset), unsigned(layout.arguments[i].stackOffset)); + FFI_CHECK_EQ(FFI::incomingStackOffset(incoming, i), layout.arguments[i].stackOffset + 16u); + } else + FFI_CHECK_EQ(unsigned(incoming.arguments[i].regIndex), unsigned(layout.arguments[i].regIndex)); + } +} + +static Vector buildGoldenCorpus() +{ + using T = FFI::Type; + Vector corpus; + auto add = [&](ASCIILiteral name, Vector&& arguments, FFI::Type returnType, ASCIILiteral sysv, ASCIILiteral aapcs, ASCIILiteral darwin, ASCIILiteral win) { + corpus.append(GoldenCase { name, WTF::move(arguments), returnType, sysv, aapcs, darwin, win }); + }; + + add("void()"_s, { }, T::Void, + "#0 rv"_s, "#0 rv"_s, "#0 rv"_s, "#32 rv"_s); + add("i32(i32)"_s, { T::Int32 }, T::Int32, + "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #32 ri"_s); + add("f64(f64)"_s, { T::Double }, T::Double, + "f0 #0 rd"_s, "f0 #0 rd"_s, "f0 #0 rd"_s, "f0 #32 rd"_s); + add("f32(f32)"_s, { T::Float }, T::Float, + "f0 #0 rf"_s, "f0 #0 rf"_s, "f0 #0 rf"_s, "f0 #32 rf"_s); + add("i64(i64)"_s, { T::Int64 }, T::Int64, + "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #32 ri"_s); + add("ptr(ptr)"_s, { T::Pointer }, T::Pointer, + "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #32 ri"_s); + add("bool(bool)"_s, { T::Bool }, T::Bool, + "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #32 ri"_s); + add("char(char)"_s, { T::Char }, T::Char, + "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #0 ri"_s, "g0 #32 ri"_s); + add("i64(i32,i32)"_s, { T::Int32, T::Int32 }, T::Int64, + "g0 g1 #0 ri"_s, "g0 g1 #0 ri"_s, "g0 g1 #0 ri"_s, "g0 g1 #32 ri"_s); + add("f64(f64,f64)"_s, { T::Double, T::Double }, T::Double, + "f0 f1 #0 rd"_s, "f0 f1 #0 rd"_s, "f0 f1 #0 rd"_s, "f0 f1 #32 rd"_s); + add("f64(i32,f64)"_s, { T::Int32, T::Double }, T::Double, + "g0 f0 #0 rd"_s, "g0 f0 #0 rd"_s, "g0 f0 #0 rd"_s, "g0 f1 #32 rd"_s); + add("f64(f64,i32)"_s, { T::Double, T::Int32 }, T::Double, + "f0 g0 #0 rd"_s, "f0 g0 #0 rd"_s, "f0 g0 #0 rd"_s, "f0 g1 #32 rd"_s); + add("f64(i32,f64,i64,f32,ptr)"_s, { T::Int32, T::Double, T::Int64, T::Float, T::Pointer }, T::Double, + "g0 f0 g1 f1 g2 #0 rd"_s, "g0 f0 g1 f1 g2 #0 rd"_s, "g0 f0 g1 f1 g2 #0 rd"_s, "g0 f1 g2 f3 s32 #48 rd"_s); + add("i64(i32x6)"_s, { T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32 }, T::Int64, + "g0 g1 g2 g3 g4 g5 #0 ri"_s, "g0 g1 g2 g3 g4 g5 #0 ri"_s, "g0 g1 g2 g3 g4 g5 #0 ri"_s, "g0 g1 g2 g3 s32 s40 #48 ri"_s); + add("i64(i32x7)"_s, Vector(FillWith { }, 7, T::Int32), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 #16 ri"_s, "g0 g1 g2 g3 g4 g5 g6 #0 ri"_s, "g0 g1 g2 g3 g4 g5 g6 #0 ri"_s, "g0 g1 g2 g3 s32 s40 s48 #64 ri"_s); + add("i64(i32x8)"_s, Vector(FillWith { }, 8, T::Int32), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 #16 ri"_s, "g0 g1 g2 g3 g4 g5 g6 g7 #0 ri"_s, "g0 g1 g2 g3 g4 g5 g6 g7 #0 ri"_s, "g0 g1 g2 g3 s32 s40 s48 s56 #64 ri"_s); + add("i64(i32x9)"_s, Vector(FillWith { }, 9, T::Int32), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 #32 ri"_s, "g0 g1 g2 g3 g4 g5 g6 g7 s0 #16 ri"_s, "g0 g1 g2 g3 g4 g5 g6 g7 s0 #16 ri"_s, "g0 g1 g2 g3 s32 s40 s48 s56 s64 #80 ri"_s); + add("i64(i32x12)"_s, Vector(FillWith { }, 12, T::Int32), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 s32 s40 #48 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 s16 s24 #32 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s4 s8 s12 #16 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 s80 s88 #96 ri"_s); + add("i64(i32x16)"_s, Vector(FillWith { }, 16, T::Int32), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 s32 s40 s48 s56 s64 s72 #80 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 s16 s24 s32 s40 s48 s56 #64 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s4 s8 s12 s16 s20 s24 s28 #32 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 s80 s88 s96 s104 s112 s120 #128 ri"_s); + add("f64(f64x8)"_s, Vector(FillWith { }, 8, T::Double), T::Double, + "f0 f1 f2 f3 f4 f5 f6 f7 #0 rd"_s, "f0 f1 f2 f3 f4 f5 f6 f7 #0 rd"_s, "f0 f1 f2 f3 f4 f5 f6 f7 #0 rd"_s, "f0 f1 f2 f3 s32 s40 s48 s56 #64 rd"_s); + add("f64(f64x9)"_s, Vector(FillWith { }, 9, T::Double), T::Double, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 #16 rd"_s, "f0 f1 f2 f3 f4 f5 f6 f7 s0 #16 rd"_s, "f0 f1 f2 f3 f4 f5 f6 f7 s0 #16 rd"_s, "f0 f1 f2 f3 s32 s40 s48 s56 s64 #80 rd"_s); + add("f64(f64x12)"_s, Vector(FillWith { }, 12, T::Double), T::Double, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 s24 #32 rd"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 s24 #32 rd"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 s24 #32 rd"_s, + "f0 f1 f2 f3 s32 s40 s48 s56 s64 s72 s80 s88 #96 rd"_s); + add("i32(i32x4)"_s, Vector(FillWith { }, 4, T::Int32), T::Int32, + "g0 g1 g2 g3 #0 ri"_s, "g0 g1 g2 g3 #0 ri"_s, "g0 g1 g2 g3 #0 ri"_s, "g0 g1 g2 g3 #32 ri"_s); + add("i32(i32x5)"_s, Vector(FillWith { }, 5, T::Int32), T::Int32, + "g0 g1 g2 g3 g4 #0 ri"_s, "g0 g1 g2 g3 g4 #0 ri"_s, "g0 g1 g2 g3 g4 #0 ri"_s, "g0 g1 g2 g3 s32 #48 ri"_s); + add("f64(f64x4)"_s, Vector(FillWith { }, 4, T::Double), T::Double, + "f0 f1 f2 f3 #0 rd"_s, "f0 f1 f2 f3 #0 rd"_s, "f0 f1 f2 f3 #0 rd"_s, "f0 f1 f2 f3 #32 rd"_s); + add("f64(f64x5)"_s, Vector(FillWith { }, 5, T::Double), T::Double, + "f0 f1 f2 f3 f4 #0 rd"_s, "f0 f1 f2 f3 f4 #0 rd"_s, "f0 f1 f2 f3 f4 #0 rd"_s, "f0 f1 f2 f3 s32 #48 rd"_s); + add("mix_1"_s, { T::Int32, T::Double, T::Int64, T::Float, T::Pointer, T::Uint8, T::Double, T::Int16, T::Double, T::Int32 }, T::Double, + "g0 f0 g1 f1 g2 g3 f2 g4 f3 g5 #0 rd"_s, + "g0 f0 g1 f1 g2 g3 f2 g4 f3 g5 #0 rd"_s, + "g0 f0 g1 f1 g2 g3 f2 g4 f3 g5 #0 rd"_s, + "g0 f1 g2 f3 s32 s40 s48 s56 s64 s72 #80 rd"_s); + add("mix_2"_s, { T::Float, T::Int32, T::Float, T::Int32, T::Float, T::Int32, T::Float, T::Int32, T::Float, T::Int32 }, T::Double, + "f0 g0 f1 g1 f2 g2 f3 g3 f4 g4 #0 rd"_s, + "f0 g0 f1 g1 f2 g2 f3 g3 f4 g4 #0 rd"_s, + "f0 g0 f1 g1 f2 g2 f3 g3 f4 g4 #0 rd"_s, + "f0 g1 f2 g3 s32 s40 s48 s56 s64 s72 #80 rd"_s); + add("mix_3"_s, { T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Int32 }, T::Double, + "f0 f1 f2 f3 f4 f5 f6 f7 g0 #0 rd"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 g0 #0 rd"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 g0 #0 rd"_s, + "f0 f1 f2 f3 s32 s40 s48 s56 s64 #80 rd"_s); + add("mix_4"_s, { T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Double, T::Int64, T::Double }, T::Double, + "g0 g1 g2 g3 g4 g5 f0 s0 f1 #16 rd"_s, + "g0 g1 g2 g3 g4 g5 f0 g6 f1 #0 rd"_s, + "g0 g1 g2 g3 g4 g5 f0 g6 f1 #0 rd"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 #80 rd"_s); + add("mix_5"_s, { T::Uint8, T::Int8, T::Uint16, T::Int16, T::Uint32, T::Int32, T::Uint64, T::Int64 }, T::Double, + "g0 g1 g2 g3 g4 g5 s0 s8 #16 rd"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 #0 rd"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 #0 rd"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 #64 rd"_s); + add("mix_6"_s, { T::Bool, T::Bool, T::Int32, T::Bool, T::Double, T::Bool, T::Float, T::Bool, T::Bool, T::Bool, T::Bool, T::Bool, T::Bool }, T::Double, + "g0 g1 g2 g3 f0 g4 f1 g5 s0 s8 s16 s24 s32 #48 rd"_s, + "g0 g1 g2 g3 f0 g4 f1 g5 g6 g7 s0 s8 s16 #32 rd"_s, + "g0 g1 g2 g3 f0 g4 f1 g5 g6 g7 s0 s1 s2 #16 rd"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 s80 s88 s96 #112 rd"_s); + add("mix_7"_s, { T::Pointer, T::Char, T::Pointer, T::Char, T::Pointer, T::Char, T::Pointer, T::Char, T::Pointer, T::Char }, T::Double, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 #32 rd"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 #16 rd"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 #16 rd"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 #80 rd"_s); + add("mix_8"_s, { T::Float, T::Double, T::Float, T::Double, T::Float, T::Double, T::Float, T::Double, T::Float, T::Double, T::Float, T::Double }, T::Double, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 s24 #32 rd"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 s24 #32 rd"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 s24 #32 rd"_s, + "f0 f1 f2 f3 s32 s40 s48 s56 s64 s72 s80 s88 #96 rd"_s); + add("i64(u8x10)"_s, Vector(FillWith { }, 10, T::Uint8), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 #32 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 #16 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s1 #16 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 #80 ri"_s); + add("i64(i16x10)"_s, Vector(FillWith { }, 10, T::Int16), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 #32 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 #16 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s2 #16 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 #80 ri"_s); + add("i64(u8x12)"_s, Vector(FillWith { }, 12, T::Uint8), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 s32 s40 #48 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 s16 s24 #32 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s1 s2 s3 #16 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 s80 s88 #96 ri"_s); + add("i64(i16x12)"_s, Vector(FillWith { }, 12, T::Int16), T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 s32 s40 #48 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 s16 s24 #32 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s2 s4 s6 #16 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 s80 s88 #96 ri"_s); + add("f32(f32x10)"_s, Vector(FillWith { }, 10, T::Float), T::Float, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 #16 rf"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 #16 rf"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s4 #16 rf"_s, + "f0 f1 f2 f3 s32 s40 s48 s56 s64 s72 #80 rf"_s); + add("u64(cstring,u64,i64_fast,u64_fast,function,buffer,jsvalue)"_s, + { T::CString, T::Uint64, T::Int64Fast, T::Uint64Fast, T::Function, T::Buffer, T::JSValue }, T::Uint64, + "g0 g1 g2 g3 g4 g5 s0 #16 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 #0 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 #0 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 #64 ri"_s); + add("i64(i64x8,i8,i8,i8)"_s, + { T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Int8, T::Int8, T::Int8 }, T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 s32 #48 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 s16 #32 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s1 s2 #16 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 s80 #96 ri"_s); + add("f32(f64x8,f32,f32,f32)"_s, + { T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Float, T::Float, T::Float }, T::Float, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 #32 rf"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s8 s16 #32 rf"_s, + "f0 f1 f2 f3 f4 f5 f6 f7 s0 s4 s8 #16 rf"_s, + "f0 f1 f2 f3 s32 s40 s48 s56 s64 s72 s80 #96 rf"_s); + add("i64(i32x8,i8,i32,i8,i64)"_s, + { T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int8, T::Int32, T::Int8, T::Int64 }, T::Int64, + "g0 g1 g2 g3 g4 g5 s0 s8 s16 s24 s32 s40 #48 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s8 s16 s24 #32 ri"_s, + "g0 g1 g2 g3 g4 g5 g6 g7 s0 s4 s8 s16 #32 ri"_s, + "g0 g1 g2 g3 s32 s40 s48 s56 s64 s72 s80 s88 #96 ri"_s); + return corpus; +} + +static void testCallLayoutGoldens() +{ + Vector corpus = buildGoldenCorpus(); + FFI_CHECK(corpus.size() >= 40); + + for (const GoldenCase& golden : corpus) { + checkLayoutAgainstGolden(golden, FFI::NativeCC::SysV64, FFI::StackPacking::EightByteSlots, golden.sysv64); + checkLayoutAgainstGolden(golden, FFI::NativeCC::Win64, FFI::StackPacking::EightByteSlots, golden.win64); + checkLayoutAgainstGolden(golden, FFI::NativeCC::AAPCS64, FFI::StackPacking::EightByteSlots, golden.aapcs64EightByteSlots); + checkLayoutAgainstGolden(golden, FFI::NativeCC::AAPCS64, FFI::StackPacking::Natural, golden.darwinARM64Packed); + } +#if OS(DARWIN) && CPU(ARM64) + FFI_CHECK(FFI::stackPackingForNativeCC(FFI::NativeCC::AAPCS64) == FFI::StackPacking::Natural); +#else + FFI_CHECK(FFI::stackPackingForNativeCC(FFI::NativeCC::AAPCS64) == FFI::StackPacking::EightByteSlots); +#endif + FFI_CHECK(FFI::stackPackingForNativeCC(FFI::NativeCC::SysV64) == FFI::StackPacking::EightByteSlots); + FFI_CHECK(FFI::stackPackingForNativeCC(FFI::NativeCC::Win64) == FFI::StackPacking::EightByteSlots); + + FFI_CHECK_EQ(FFI::shadowStackBytes(FFI::NativeCC::SysV64), 0u); + FFI_CHECK_EQ(FFI::shadowStackBytes(FFI::NativeCC::AAPCS64), 0u); + FFI_CHECK_EQ(FFI::shadowStackBytes(FFI::NativeCC::Win64), 32u); +#if OS(WINDOWS) && CPU(X86_64) + FFI_CHECK(FFI::hostNativeCC() == FFI::NativeCC::Win64); +#elif CPU(ARM64) + FFI_CHECK(FFI::hostNativeCC() == FFI::NativeCC::AAPCS64); +#else + FFI_CHECK(FFI::hostNativeCC() == FFI::NativeCC::SysV64); +#endif + +#if CPU(X86_64) + { + auto ints = FFI::integerArgumentRegisters(FFI::NativeCC::SysV64); + auto floats = FFI::floatArgumentRegisters(FFI::NativeCC::SysV64); + FFI_CHECK_EQ(ints.size(), 6u); + FFI_CHECK_EQ(floats.size(), 8u); + if (ints.size() == 6) { + FFI_CHECK(ints[0] == X86Registers::edi); + FFI_CHECK(ints[1] == X86Registers::esi); + FFI_CHECK(ints[2] == X86Registers::edx); + FFI_CHECK(ints[3] == X86Registers::ecx); + FFI_CHECK(ints[4] == X86Registers::r8); + FFI_CHECK(ints[5] == X86Registers::r9); + } + for (unsigned i = 0; i < floats.size() && i < 8; ++i) + FFI_CHECK(floats[i] == static_cast(X86Registers::xmm0 + i)); + + auto winInts = FFI::integerArgumentRegisters(FFI::NativeCC::Win64); + auto winFloats = FFI::floatArgumentRegisters(FFI::NativeCC::Win64); + FFI_CHECK_EQ(winInts.size(), 4u); + FFI_CHECK_EQ(winFloats.size(), 4u); + if (winInts.size() == 4) { + FFI_CHECK(winInts[0] == X86Registers::ecx); + FFI_CHECK(winInts[1] == X86Registers::edx); + FFI_CHECK(winInts[2] == X86Registers::r8); + FFI_CHECK(winInts[3] == X86Registers::r9); + } + for (unsigned i = 0; i < winFloats.size() && i < 4; ++i) + FFI_CHECK(winFloats[i] == static_cast(X86Registers::xmm0 + i)); + + auto scratch = FFI::scratchGPRsForInvoke(FFI::hostNativeCC()); + FFI_CHECK(scratch[0] == X86Registers::ebx); + FFI_CHECK(scratch[1] == X86Registers::r10); + FFI_CHECK(scratch[0] != X86Registers::r11 && scratch[1] != X86Registers::r11); + FFI_CHECK(scratch[0] != GPRInfo::returnValueGPR && scratch[1] != GPRInfo::returnValueGPR); + for (GPRReg reg : FFI::integerArgumentRegisters(FFI::hostNativeCC())) + FFI_CHECK(reg != scratch[0] && reg != scratch[1]); + } +#elif CPU(ARM64) + { + auto ints = FFI::integerArgumentRegisters(FFI::NativeCC::AAPCS64); + auto floats = FFI::floatArgumentRegisters(FFI::NativeCC::AAPCS64); + FFI_CHECK_EQ(ints.size(), 8u); + FFI_CHECK_EQ(floats.size(), 8u); + for (unsigned i = 0; i < ints.size() && i < 8; ++i) + FFI_CHECK(ints[i] == static_cast(ARM64Registers::x0 + i)); + for (unsigned i = 0; i < floats.size() && i < 8; ++i) + FFI_CHECK(floats[i] == static_cast(ARM64Registers::q0 + i)); + + auto scratch = FFI::scratchGPRsForInvoke(FFI::hostNativeCC()); + FFI_CHECK(scratch[0] == ARM64Registers::x19); + FFI_CHECK(scratch[1] == ARM64Registers::x9); + FFI_CHECK(scratch[0] != ARM64Registers::x16 && scratch[0] != ARM64Registers::x17); + FFI_CHECK(scratch[1] != ARM64Registers::x16 && scratch[1] != ARM64Registers::x17); + FFI_CHECK(scratch[0] != GPRInfo::returnValueGPR && scratch[1] != GPRInfo::returnValueGPR); + for (GPRReg reg : FFI::integerArgumentRegisters(FFI::hostNativeCC())) + FFI_CHECK(reg != scratch[0] && reg != scratch[1]); + } +#endif +} + +struct ReferenceLocation { + FFI::ArgLocation::Kind kind; + unsigned index { 0 }; + unsigned offset { 0 }; +}; + +struct ReferenceLayout { + Vector locations; + unsigned stackBytes { 0 }; +}; + +static ReferenceLayout referenceLayout(FFI::NativeCC cc, FFI::StackPacking packing, std::span arguments) +{ + ReferenceLayout result; + bool darwinPacking = cc == FFI::NativeCC::AAPCS64 && packing == FFI::StackPacking::Natural; + unsigned gprLimit = cc == FFI::NativeCC::SysV64 ? 6 : 8; + unsigned fprLimit = 8; + unsigned gprIndex = 0; + unsigned fprIndex = 0; + unsigned nextStackAddress = 0; + for (unsigned i = 0; i < arguments.size(); ++i) { + FFI::Type type = arguments[i]; + FFI::ArgClass klass = FFI::argClass(type); + bool isFloatClass = klass == FFI::ArgClass::Float || klass == FFI::ArgClass::Double; + ReferenceLocation location; + if (cc == FFI::NativeCC::Win64) { + if (i < 4) { + location.kind = isFloatClass ? FFI::ArgLocation::Kind::FPR : FFI::ArgLocation::Kind::GPR; + location.index = i; + } else { + location.kind = FFI::ArgLocation::Kind::Stack; + location.offset = 32 + (i - 4) * 8; + } + } else if (isFloatClass && fprIndex < fprLimit) { + location.kind = FFI::ArgLocation::Kind::FPR; + location.index = fprIndex++; + } else if (!isFloatClass && gprIndex < gprLimit) { + location.kind = FFI::ArgLocation::Kind::GPR; + location.index = gprIndex++; + } else { + unsigned size = darwinPacking ? FFI::nativeSizeInBytes(type) : 8; + unsigned offset = roundUpToMultipleOf(size, nextStackAddress); + location.kind = FFI::ArgLocation::Kind::Stack; + location.offset = offset; + nextStackAddress = offset + size; + } + result.locations.append(location); + } + if (cc == FFI::NativeCC::Win64) { + unsigned stackArgumentCount = arguments.size() > 4 ? arguments.size() - 4 : 0; + result.stackBytes = roundUpToMultipleOf(16u, 32 + stackArgumentCount * 8); + } else + result.stackBytes = roundUpToMultipleOf(16u, nextStackAddress); + return result; +} + +static void testCallLayoutAgainstReferenceModel() +{ + static constexpr FFI::Type argumentTypes[] = { + FFI::Type::Char, FFI::Type::Int8, FFI::Type::Uint8, FFI::Type::Int16, FFI::Type::Uint16, + FFI::Type::Int32, FFI::Type::Uint32, FFI::Type::Int64, FFI::Type::Uint64, FFI::Type::Double, + FFI::Type::Float, FFI::Type::Bool, FFI::Type::Pointer, FFI::Type::CString, FFI::Type::Int64Fast, + FFI::Type::Uint64Fast, FFI::Type::Function, FFI::Type::JSValue, FFI::Type::Buffer, FFI::Type::BufferLength, + }; + static constexpr FFI::Type returnTypes[] = { + FFI::Type::Void, FFI::Type::Int32, FFI::Type::Uint32, FFI::Type::Int64, FFI::Type::Double, FFI::Type::Float, FFI::Type::Bool, FFI::Type::Pointer, + }; + static constexpr struct { FFI::NativeCC cc; FFI::StackPacking packing; } modes[] = { + { FFI::NativeCC::SysV64, FFI::StackPacking::EightByteSlots }, + { FFI::NativeCC::AAPCS64, FFI::StackPacking::EightByteSlots }, + { FFI::NativeCC::AAPCS64, FFI::StackPacking::Natural }, + { FFI::NativeCC::Win64, FFI::StackPacking::EightByteSlots }, + }; + + WeakRandom random(0x5EEDCA11u); + unsigned mismatches = 0; + for (unsigned iteration = 0; iteration < 400; ++iteration) { + unsigned count = random.getUint32(FFI::Signature::maxArguments + 1); + Vector arguments; + for (unsigned i = 0; i < count; ++i) + arguments.append(argumentTypes[random.getUint32(std::size(argumentTypes))]); + FFI::Type returnType = returnTypes[random.getUint32(std::size(returnTypes))]; + RefPtr signature = FFI::Signature::tryCreate(arguments.span(), returnType); + FFI_CHECK(!!signature); + if (!signature) + continue; + for (const auto& mode : modes) { + FFI::NativeCC cc = mode.cc; + FFI::CallLayout layout = FFI::computeCallLayout(cc, mode.packing, *signature, FFI::Direction::Outgoing); + ReferenceLayout reference = referenceLayout(cc, mode.packing, arguments.span()); + bool ok = layout.arguments.size() == reference.locations.size() + && layout.stackBytes == reference.stackBytes + && !(layout.stackBytes % 16) + && layout.returnClass == FFI::argClass(returnType) + && layout.cc == cc + && layout.packing == mode.packing; + if (ok) { + for (unsigned i = 0; i < layout.arguments.size(); ++i) { + ok &= layout.arguments[i].kind == reference.locations[i].kind; + ok &= layout.arguments[i].type == arguments[i]; + if (!ok) + break; + if (layout.arguments[i].kind == FFI::ArgLocation::Kind::Stack) + ok &= layout.arguments[i].stackOffset == reference.locations[i].offset; + else + ok &= layout.arguments[i].regIndex == reference.locations[i].index; + if (!ok) + break; + } + } + s_checkCount++; + if (!ok) { + s_failureCount++; + if (++mismatches < 10) + dataLogLn(" FAIL: layout mismatch vs reference model for ", signature->toString(), " under ", nativeCCName(cc, mode.packing)); + } + } + } +} + +static void testDoubleToInt64() +{ + constexpr int64_t indefinite = std::numeric_limits::min(); // 0x8000000000000000 + UNUSED_PARAM(indefinite); // only referenced by the CPU-specific branch below + constexpr double nan = std::numeric_limits::quiet_NaN(); + constexpr double inf = std::numeric_limits::infinity(); + constexpr double twoTo63 = 9223372036854775808.0; + constexpr double twoTo64 = 18446744073709551616.0; + + FFI_CHECK_EQ(FFI::doubleToInt64(0.0), 0); + FFI_CHECK_EQ(FFI::doubleToInt64(-0.0), 0); + FFI_CHECK_EQ(FFI::doubleToInt64(-1.5), -1); + FFI_CHECK_EQ(FFI::doubleToInt64(1.5), 1); + FFI_CHECK_EQ(FFI::doubleToInt64(2147483648.5), 2147483648ll); + FFI_CHECK_EQ(FFI::doubleToInt64(-2147483649.75), -2147483649ll); + FFI_CHECK_EQ(FFI::doubleToInt64(-twoTo63), indefinite); // exact -2^63 + FFI_CHECK_EQ(FFI::doubleToInt64(9223372036854774784.0), 9223372036854774784ll); // largest double < 2^63 + FFI_CHECK_EQ(FFI::doubleToInt64(4294967296.0), 4294967296ll); + FFI_CHECK_EQ(FFI::doubleToInt64(0.9999999999999999), 0); + FFI_CHECK_EQ(FFI::doubleToInt64(-0.9999999999999999), 0); + +#if CPU(X86_64) + FFI_CHECK_EQ(FFI::doubleToInt64(nan), indefinite); + FFI_CHECK_EQ(FFI::doubleToInt64(inf), indefinite); + FFI_CHECK_EQ(FFI::doubleToInt64(-inf), indefinite); + FFI_CHECK_EQ(FFI::doubleToInt64(twoTo63), indefinite); + FFI_CHECK_EQ(FFI::doubleToInt64(twoTo64), indefinite); + FFI_CHECK_EQ(FFI::doubleToInt64(-twoTo64), indefinite); +#elif CPU(ARM64) + FFI_CHECK_EQ(FFI::doubleToInt64(nan), 0); + FFI_CHECK_EQ(FFI::doubleToInt64(inf), std::numeric_limits::max()); + FFI_CHECK_EQ(FFI::doubleToInt64(-inf), std::numeric_limits::min()); + FFI_CHECK_EQ(FFI::doubleToInt64(twoTo63), std::numeric_limits::max()); + FFI_CHECK_EQ(FFI::doubleToInt64(twoTo64), std::numeric_limits::max()); + FFI_CHECK_EQ(FFI::doubleToInt64(-twoTo64), std::numeric_limits::min()); +#endif + + static const double corpus[] = { 0.0, -0.0, 1.0, -1.0, -1.5, 1.5, 255.75, -255.75, twoTo63, -twoTo63, twoTo64, nan, inf, -inf, 4294967295.5, -4294967295.5, 1e300, -1e300, 4.9e-324 }; + for (double value : corpus) + FFI_CHECK_EQ_HEX(FFI::doubleToUInt64(value), std::bit_cast(FFI::doubleToInt64(value))); +} + +enum class ExpectThrow : bool { No, Yes }; + +static bool convertToSlot(FFI::Type type, JSValue value, uint64_t& slotOut, ExpectThrow expectThrow) +{ + VM& vm = *s_vm; + auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + FFI::StringArena::Scope arenaScope(ffiContext()); + slotOut = 0xAAAAAAAAAAAAAAAAull; + bool ok = FFI::writeSlotFromJSValue(s_globalObject, ffiContext(), type, value, slotOut, testStringArena()); + Exception* exception = scope.exception(); + if (exception) { + JSValue error = exception->value(); + scope.clearException(); + s_checkCount++; + if (expectThrow == ExpectThrow::No) { + s_failureCount++; + dataLogLn(" FAIL: writeSlotFromJSValue(", typeNameForLog(type), ") threw unexpectedly"); + } + ErrorInstance* errorInstance = dynamicDowncast(error); + FFI_CHECK(errorInstance); + if (errorInstance) + FFI_CHECK(errorInstance->errorType() == ErrorType::TypeError); + FFI_CHECK(!ok); + return false; + } + s_checkCount++; + if (expectThrow == ExpectThrow::Yes) { + s_failureCount++; + dataLogLn(" FAIL: writeSlotFromJSValue(", typeNameForLog(type), ") should have thrown"); + } + FFI_CHECK(ok); + return ok; +} + +static void expectSlot(FFI::Type type, JSValue value, uint64_t expected) +{ + uint64_t slot = 0; + if (!convertToSlot(type, value, slot, ExpectThrow::No)) + return; + s_checkCount++; + if (slot != expected) { + s_failureCount++; + dataLogLn(" FAIL: writeSlotFromJSValue(", typeNameForLog(type), ") slot ", RawHex(slot), " != expected ", RawHex(expected)); + } +} + +static void expectSlotThrows(FFI::Type type, JSValue value) +{ + uint64_t slot = 0; + convertToSlot(type, value, slot, ExpectThrow::Yes); +} + +static JSValue slotToJS(FFI::Type type, uint64_t slot) +{ + VM& vm = *s_vm; + auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + JSValue result = FFI::jsValueFromSlot(s_globalObject, ffiContext(), type, slot); + Exception* exception = scope.exception(); + if (exception) { + s_checkCount++; + s_failureCount++; + dataLogLn(" FAIL: jsValueFromSlot(", typeNameForLog(type), ") threw"); + scope.clearException(); + return jsUndefined(); + } + return result; +} + +static void expectSlotToNumber(FFI::Type type, uint64_t slot, double expected) +{ + JSValue value = slotToJS(type, slot); + s_checkCount++; + if (!value.isNumber()) { + s_failureCount++; + dataLogLn(" FAIL: jsValueFromSlot(", typeNameForLog(type), ", ", RawHex(slot), ") is not a number"); + return; + } + double actual = value.asNumber(); + bool equal = (std::isnan(expected) && std::isnan(actual)) + || (actual == expected && !!std::signbit(actual) == !!std::signbit(expected)); + if (!equal) { + s_failureCount++; + dataLogLn(" FAIL: jsValueFromSlot(", typeNameForLog(type), ", ", RawHex(slot), ") -> ", actual, ", expected ", expected); + } +} + +static void expectSlotToBigInt(FFI::Type type, uint64_t slot, uint64_t expectedBits) +{ + JSValue value = slotToJS(type, slot); + s_checkCount++; + if (!value.isBigInt()) { + s_failureCount++; + dataLogLn(" FAIL: jsValueFromSlot(", typeNameForLog(type), ", ", RawHex(slot), ") is not a BigInt"); + return; + } + uint64_t actualBits = FFI::isSigned(type) + ? static_cast(JSBigInt::toBigInt64(value)) + : JSBigInt::toBigUInt64(value); + if (actualBits != expectedBits) { + s_failureCount++; + dataLogLn(" FAIL: jsValueFromSlot(", typeNameForLog(type), ", ", RawHex(slot), ") bits ", RawHex(actualBits), " != ", RawHex(expectedBits)); + } +} + +static JSUint8Array* makeUint8Array(unsigned length) +{ + return JSUint8Array::create(s_globalObject, s_globalObject->typedArrayStructure(TypeUint8, false), length); +} + +static void testConversions() +{ + VM& vm = *s_vm; + JSGlobalObject* globalObject = s_globalObject; + using T = FFI::Type; + + FFI::StringArena::Scope arenaScope(ffiContext()); + + constexpr uint64_t allOnes = ~static_cast(0); + const double nan = PNaN; // JSC's canonical (pure) NaN, runtime/PureNaN.h + const double inf = std::numeric_limits::infinity(); + + expectSlot(T::Int32, jsNumber(2147483647), 0x000000007fffffffull); + expectSlot(T::Int32, jsNumber(-2147483647 - 1), 0xffffffff80000000ull); + expectSlot(T::Int32, jsNumber(0), 0); + expectSlot(T::Int32, jsNumber(-1), allOnes); + expectSlot(T::Int32, jsNumber(4294967301.0), 5); // 2^32 + 5 wraps + expectSlot(T::Int32, jsNumber(2147483648.0), 0xffffffff80000000ull); // wraps to INT32_MIN + expectSlot(T::Int32, jsNumber(-2147483649.0), 0x000000007fffffffull); + expectSlot(T::Int32, jsNumber(0.9), 0); + expectSlot(T::Int32, jsNumber(-0.9), 0); + expectSlot(T::Int32, jsNumber(-0.0), 0); + expectSlot(T::Int32, jsNumber(nan), 0); + expectSlot(T::Int32, jsNumber(inf), 0); + expectSlot(T::Int32, jsNumber(-inf), 0); + expectSlot(T::Int32, jsUndefined(), 0); + expectSlot(T::Int32, jsNull(), 0); + expectSlot(T::Int32, jsBoolean(true), 1); + expectSlot(T::Int32, jsBoolean(false), 0); + expectSlot(T::Uint32, jsNumber(-1), 0x00000000ffffffffull); + expectSlot(T::Uint32, jsNumber(4294967295.0), 0x00000000ffffffffull); + expectSlot(T::Uint32, jsNumber(4294967296.0), 0); + expectSlot(T::Uint32, jsNumber(2147483648.0), 0x0000000080000000ull); + expectSlot(T::Uint32, jsNumber(-0.5), 0); + expectSlot(T::Int8, jsNumber(511), allOnes); // 0x1ff -> low byte 0xff sign-extended + expectSlot(T::Int8, jsNumber(128), 0xffffffffffffff80ull); + expectSlot(T::Int8, jsNumber(127), 127); + expectSlot(T::Int8, jsNumber(-129), 127); + expectSlot(T::Int8, jsNumber(-1), allOnes); + expectSlot(T::Char, jsNumber(-1), allOnes); // char is signed on every target + expectSlot(T::Char, jsNumber(255), allOnes); + expectSlot(T::Char, jsNumber(200), 0xffffffffffffffc8ull); // 200 -> -56 + expectSlot(T::Uint8, jsNumber(-1), 0xff); + expectSlot(T::Uint8, jsNumber(256), 0); + expectSlot(T::Uint8, jsNumber(511), 0xff); + expectSlot(T::Uint8, jsNumber(255.9), 0xff); + expectSlot(T::Int16, jsNumber(0x12345), 0x2345); + expectSlot(T::Int16, jsNumber(32768), 0xffffffffffff8000ull); + expectSlot(T::Int16, jsNumber(-32769), 0x7fff); + expectSlot(T::Uint16, jsNumber(-1), 0xffff); + expectSlot(T::Uint16, jsNumber(65536), 0); + expectSlot(T::Uint16, jsNumber(70000), 70000 - 65536); + + expectSlot(T::Bool, jsNumber(0), 0); + expectSlot(T::Bool, jsNumber(2), 1); + expectSlot(T::Bool, jsNumber(-1), 1); + expectSlot(T::Bool, jsNumber(0.5), 1); + expectSlot(T::Bool, jsNumber(-0.0), 0); + expectSlot(T::Bool, jsNumber(nan), 0); + expectSlot(T::Bool, jsNumber(inf), 1); + expectSlot(T::Bool, jsBoolean(true), 1); + expectSlot(T::Bool, jsBoolean(false), 0); + expectSlot(T::Bool, jsUndefined(), 0); + expectSlot(T::Bool, jsNull(), 0); + + expectSlot(T::Int64, jsNumber(1), 1); + expectSlot(T::Int64, jsNumber(-1), allOnes); + expectSlot(T::Int64, jsNumber(-2147483647 - 1), 0xffffffff80000000ull); + expectSlot(T::Int64, jsNumber(4294967296.0), 4294967296ull); + expectSlot(T::Int64, jsNumber(1e18), 1000000000000000000ull); + expectSlot(T::Int64, jsNumber(-1.5), std::bit_cast(FFI::doubleToInt64(-1.5))); + expectSlot(T::Int64, jsNumber(9223372036854775808.0), std::bit_cast(FFI::doubleToInt64(9223372036854775808.0))); + expectSlot(T::Int64, jsNumber(nan), std::bit_cast(FFI::doubleToInt64(nan))); + expectSlot(T::Int64, JSBigInt::createFrom(globalObject, static_cast(std::numeric_limits::max())), 0x7fffffffffffffffull); + expectSlot(T::Int64, JSBigInt::createFrom(globalObject, static_cast(std::numeric_limits::min())), 0x8000000000000000ull); + expectSlot(T::Int64, JSBigInt::createFrom(globalObject, static_cast(-1)), allOnes); + expectSlot(T::Int64, JSBigInt::createFrom(globalObject, static_cast(0xffffffffffffffffull)), allOnes); // 2^64-1 mod 2^64 + expectSlot(T::Uint64, jsNumber(-1), allOnes); + expectSlot(T::Uint64, jsNumber(2.5), std::bit_cast(FFI::doubleToInt64(2.5))); + expectSlot(T::Uint64, JSBigInt::createFrom(globalObject, static_cast(0xdeadbeefcafebabeull)), 0xdeadbeefcafebabeull); + expectSlot(T::Uint64, JSBigInt::createFrom(globalObject, static_cast(-2)), allOnes - 1); // BigInt(-2) mod 2^64 + expectSlot(T::Int64Fast, jsNumber(-42), std::bit_cast(static_cast(-42))); + expectSlot(T::Int64Fast, JSBigInt::createFrom(globalObject, static_cast(1) << 60), static_cast(1) << 60); + expectSlot(T::Uint64Fast, jsNumber(9007199254740992.0), 9007199254740992ull); + expectSlot(T::Uint64Fast, JSBigInt::createFrom(globalObject, static_cast(1) << 63), static_cast(1) << 63); + + expectSlot(T::Double, jsNumber(1.5), std::bit_cast(1.5)); + expectSlot(T::Double, jsNumber(-0.0), 0x8000000000000000ull); + expectSlot(T::Double, jsNumber(inf), std::bit_cast(inf)); + expectSlot(T::Double, jsNumber(-inf), std::bit_cast(-inf)); + expectSlot(T::Double, jsNumber(2147483647), std::bit_cast(2147483647.0)); + expectSlot(T::Double, jsNumber(4.9e-324), std::bit_cast(4.9e-324)); + { + uint64_t slot = 0; + if (convertToSlot(T::Double, jsNumber(nan), slot, ExpectThrow::No)) + FFI_CHECK(std::isnan(std::bit_cast(slot))); + if (convertToSlot(T::Double, jsUndefined(), slot, ExpectThrow::No)) + FFI_CHECK(std::isnan(std::bit_cast(slot))); // Number(undefined) is NaN + } + expectSlot(T::Float, jsNumber(1.5), static_cast(std::bit_cast(1.5f))); + expectSlot(T::Float, jsNumber(1.1), static_cast(std::bit_cast(1.1f))); + expectSlot(T::Float, jsNumber(-0.0), 0x0000000080000000ull); + expectSlot(T::Float, jsNumber(1e40), static_cast(std::bit_cast(std::numeric_limits::infinity()))); + expectSlot(T::Float, jsNumber(1e-46), static_cast(std::bit_cast(static_cast(1e-46)))); + { + uint64_t slot = 0; + if (convertToSlot(T::Float, jsNumber(nan), slot, ExpectThrow::No)) { + FFI_CHECK_EQ_HEX(slot >> 32, 0); // upper half zero even for NaN + FFI_CHECK(std::isnan(std::bit_cast(static_cast(slot)))); + } + } + + static uint32_t nativeScratch[16]; + JSUint8Array* uint8Array = makeUint8Array(32); + void* uint8ArrayVector = uint8Array ? uint8Array->vector() : nullptr; + expectSlot(T::Pointer, jsNumber(0), 0); + expectSlot(T::Pointer, jsNull(), 0); + expectSlot(T::Pointer, jsUndefined(), 0); + expectSlot(T::Pointer, jsNumber(4096), 4096); + expectSlot(T::Pointer, jsNumber(-1), allOnes); // int32 pointers are sign-extended + expectSlot(T::Pointer, jsNumber(1099511627776.0), 1099511627776ull); // 2^40 via the double path + expectSlot(T::Pointer, jsNumber(static_cast(reinterpret_cast(&nativeScratch[0]))), reinterpret_cast(&nativeScratch[0])); + if (uint8Array) { + FFI_CHECK(uint8ArrayVector); + expectSlot(T::Pointer, uint8Array, reinterpret_cast(uint8ArrayVector)); + expectSlot(T::CString, uint8Array, reinterpret_cast(uint8ArrayVector)); + expectSlot(T::Buffer, uint8Array, reinterpret_cast(uint8ArrayVector)); + expectSlot(T::Function, uint8Array, reinterpret_cast(uint8ArrayVector)); + } + expectSlot(T::CString, jsNumber(0), 0); + expectSlot(T::Function, jsNumber(0), 0); + expectSlot(T::Function, jsNumber(65536), 65536); + expectSlotThrows(T::Pointer, jsString(vm, String("hello"_s))); + expectSlotThrows(T::Buffer, jsString(vm, String("hello"_s))); + expectSlotThrows(T::Function, jsString(vm, String("hello"_s))); + expectSlotThrows(T::Pointer, Symbol::create(vm, SymbolImpl::createNullSymbol().get())); + expectSlotThrows(T::Int32, Symbol::create(vm, SymbolImpl::createNullSymbol().get())); + expectSlotThrows(T::Double, Symbol::create(vm, SymbolImpl::createNullSymbol().get())); + expectSlotThrows(T::Pointer, constructEmptyObject(globalObject)); + expectSlotThrows(T::Buffer, jsNumber(5)); // buffer requires a view + expectSlotThrows(T::Buffer, jsNull()); + expectSlotThrows(T::Buffer, constructEmptyObject(globalObject)); + + { + uint64_t slot = 0; + if (convertToSlot(T::CString, jsString(vm, String("hello"_s)), slot, ExpectThrow::No)) { + FFI_CHECK(slot); + if (slot) + FFI_CHECK(!strcmp(reinterpret_cast(slot), "hello")); + } + if (convertToSlot(T::CString, jsString(vm, emptyString()), slot, ExpectThrow::No)) { + FFI_CHECK(slot); + if (slot) + FFI_CHECK_EQ(strlen(reinterpret_cast(slot)), 0u); + } + static const char utf8Sample[] = "h\xC3\xA9llo \xE2\x86\x92 \xF0\x9D\x84\x9E"; // "héllo → 𝄞" + String nonASCII = String::fromUTF8(utf8Sample); + if (convertToSlot(T::CString, jsString(vm, nonASCII), slot, ExpectThrow::No)) { + FFI_CHECK(slot); + if (slot) + FFI_CHECK(!strcmp(reinterpret_cast(slot), utf8Sample)); + } + JSString* concatenated = jsString(vm, makeString("ro"_s, "pe "_s, 12345)); + if (convertToSlot(T::CString, concatenated, slot, ExpectThrow::No)) { + FFI_CHECK(slot); + if (slot) + FFI_CHECK(!strcmp(reinterpret_cast(slot), "rope 12345")); + } + } + + { + JSObject* object = constructEmptyObject(globalObject); + JSString* string = jsString(vm, String("napi"_s)); + expectSlot(T::JSValue, jsNumber(3.5), static_cast(JSValue::encode(jsNumber(3.5)))); + expectSlot(T::JSValue, jsBoolean(true), static_cast(JSValue::encode(jsBoolean(true)))); + expectSlot(T::JSValue, jsUndefined(), static_cast(JSValue::encode(jsUndefined()))); + expectSlot(T::JSValue, jsNull(), static_cast(JSValue::encode(jsNull()))); + expectSlot(T::JSValue, object, static_cast(JSValue::encode(object))); + expectSlot(T::JSValue, string, static_cast(JSValue::encode(string))); + JSValue backObject = slotToJS(T::JSValue, static_cast(JSValue::encode(object))); + FFI_CHECK(backObject == JSValue(object)); + JSValue backNumber = slotToJS(T::JSValue, static_cast(JSValue::encode(jsNumber(-7)))); + FFI_CHECK(backNumber == jsNumber(-7)); + } + + expectSlotToNumber(T::Int8, 0xff, -1); + expectSlotToNumber(T::Int8, 0xffffffffffffff80ull, -128); + expectSlotToNumber(T::Int8, 0x7f, 127); + expectSlotToNumber(T::Char, 0xffffffffffffffffull, -1); + expectSlotToNumber(T::Uint8, 0xff, 255); + expectSlotToNumber(T::Uint8, 0, 0); + expectSlotToNumber(T::Int16, 0xffff, -1); + expectSlotToNumber(T::Int16, 0x8000, -32768); + expectSlotToNumber(T::Uint16, 0xffff, 65535); + expectSlotToNumber(T::Int32, 0xffffffff80000000ull, -2147483648.0); + expectSlotToNumber(T::Int32, 0x7fffffff, 2147483647); + expectSlotToNumber(T::Uint32, 0xffffffff, 4294967295.0); + expectSlotToNumber(T::Uint32, 0x80000000, 2147483648.0); + expectSlotToNumber(T::Uint32, 0x7fffffff, 2147483647); + FFI_CHECK(slotToJS(T::Bool, 1) == jsBoolean(true)); + FFI_CHECK(slotToJS(T::Bool, 0) == jsBoolean(false)); + FFI_CHECK(slotToJS(T::Void, 0xdeadbeef) == jsUndefined()); + + expectSlotToBigInt(T::Int64, 42, 42); + expectSlotToBigInt(T::Int64, 0x8000000000000000ull, 0x8000000000000000ull); + expectSlotToBigInt(T::Int64, allOnes, allOnes); + expectSlotToBigInt(T::Uint64, allOnes, allOnes); + expectSlotToBigInt(T::Uint64, 0, 0); + expectSlotToNumber(T::Int64Fast, static_cast(9007199254740991ll), 9007199254740991.0); // 2^53-1 stays a Number + expectSlotToBigInt(T::Int64Fast, static_cast(9007199254740992ll), 9007199254740992ull); // 2^53 becomes a BigInt + expectSlotToNumber(T::Int64Fast, std::bit_cast(static_cast(-9007199254740991ll)), -9007199254740991.0); + expectSlotToBigInt(T::Int64Fast, std::bit_cast(static_cast(-9007199254740992ll)), std::bit_cast(static_cast(-9007199254740992ll))); + expectSlotToNumber(T::Int64Fast, allOnes, -1.0); + expectSlotToNumber(T::Uint64Fast, 9007199254740990ull, 9007199254740990.0); // < 2^53-1 stays a Number + expectSlotToBigInt(T::Uint64Fast, 9007199254740991ull, 9007199254740991ull); // == 2^53-1 becomes a BigInt (strict <, Bun quirk) + expectSlotToBigInt(T::Uint64Fast, allOnes, allOnes); + + expectSlotToNumber(T::Double, std::bit_cast(-0.0), -0.0); + expectSlotToNumber(T::Double, std::bit_cast(1.5), 1.5); + expectSlotToNumber(T::Double, std::bit_cast(inf), inf); + expectSlotToNumber(T::Double, std::bit_cast(4.9e-324), 4.9e-324); + expectSlotToNumber(T::Double, 0x7ff8000000000001ull, nan); // impure NaN must be purified (debug jsNumber ASSERT) + expectSlotToNumber(T::Double, 0x7ff0000000000001ull, nan); // signaling NaN payload + expectSlotToNumber(T::Double, 0xfff123456789abcdull, nan); + expectSlotToNumber(T::Float, 0x7fc00001, nan); + expectSlotToNumber(T::Float, 0x00000001, static_cast(std::bit_cast(0x00000001u))); // denormal + expectSlotToNumber(T::Float, static_cast(std::bit_cast(-1.5f)), -1.5); + expectSlotToNumber(T::Float, 0x80000000, -0.0); + + FFI_CHECK(slotToJS(T::Pointer, 0).isNull()); + FFI_CHECK(slotToJS(T::CString, 0).isNull()); + FFI_CHECK(slotToJS(T::Function, 0).isNull()); + FFI_CHECK(slotToJS(T::Buffer, 0).isNull()); + expectSlotToNumber(T::Pointer, 0x00007fffdeadbee0ull, static_cast(0x00007fffdeadbee0ull)); + { + static const char kHello[] = "hello"; + JSValue decoded = slotToJS(T::CString, reinterpret_cast(kHello)); + FFI_CHECK(decoded.isString()); + FFI_CHECK(asString(decoded)->value(s_globalObject) == String::fromUTF8("hello")); + } + expectSlotToNumber(T::Function, 1, 1); + expectSlotToNumber(T::Buffer, 65536, 65536); + + static const int64_t roundTripValues[] = { 0, 1, -1, 127, -128, 255, 32767, -32768, 65535, 2147483647, -2147483647 - 1, static_cast(4294967295u) }; + for (int64_t value : roundTripValues) { + static const struct { T type; int64_t min; int64_t max; } integerTypes[] = { + { T::Int8, -128, 127 }, { T::Char, -128, 127 }, { T::Uint8, 0, 255 }, { T::Int16, -32768, 32767 }, + { T::Uint16, 0, 65535 }, { T::Int32, -2147483647ll - 1, 2147483647 }, { T::Uint32, 0, 4294967295ll }, + }; + for (auto& entry : integerTypes) { + if (value < entry.min || value > entry.max) + continue; + uint64_t slot = 0; + if (!convertToSlot(entry.type, jsNumber(static_cast(value)), slot, ExpectThrow::No)) + continue; + expectSlotToNumber(entry.type, slot, static_cast(value)); + } + } +} + +static uint64_t s_scratchMemory[64]; + +template +static T slotToNative(uint64_t slot) +{ + if constexpr (std::is_same_v) + return !!slot; + else if constexpr (std::is_pointer_v) + return reinterpret_cast(static_cast(slot)); + else if constexpr (std::is_same_v) + return std::bit_cast(static_cast(slot)); + else if constexpr (std::is_same_v) + return std::bit_cast(slot); + else { + static_assert(std::is_integral_v); + return static_cast(slot); + } +} + +template +static R callNativeFromSlotsImpl(R (*function)(Arguments...), const uint64_t* slots, std::index_sequence) +{ + return function(slotToNative(slots[Indices])...); +} + +template +static R callNativeFromSlots(R (*function)(Arguments...), const uint64_t* slots) +{ + return callNativeFromSlotsImpl(function, slots, std::index_sequence_for { }); +} + +template +static uint64_t nativeReturnRawBits(R value) +{ + if constexpr (std::is_same_v) + return value ? 1 : 0; + else if constexpr (std::is_pointer_v) + return static_cast(reinterpret_cast(value)); + else if constexpr (std::is_same_v) + return static_cast(std::bit_cast(value)); + else if constexpr (std::is_same_v) + return std::bit_cast(value); + else if constexpr (std::is_signed_v) + return static_cast(static_cast(value)); + else + return static_cast(value); +} + +static uint64_t canonicalizeSlot(FFI::Type type, uint64_t raw) +{ + switch (type) { + case FFI::Type::Char: + case FFI::Type::Int8: + return static_cast(static_cast(static_cast(raw))); + case FFI::Type::Uint8: + return static_cast(raw); + case FFI::Type::Int16: + return static_cast(static_cast(static_cast(raw))); + case FFI::Type::Uint16: + return static_cast(raw); + case FFI::Type::Int32: + return static_cast(static_cast(static_cast(raw))); + case FFI::Type::Uint32: + return static_cast(raw); + case FFI::Type::Bool: + return static_cast(raw) ? 1 : 0; + case FFI::Type::Float: + return raw & 0xffffffffull; + case FFI::Type::Int64: + case FFI::Type::Uint64: + case FFI::Type::Int64Fast: + case FFI::Type::Uint64Fast: + case FFI::Type::Double: + case FFI::Type::Pointer: + case FFI::Type::CString: + case FFI::Type::Function: + case FFI::Type::Buffer: + case FFI::Type::BufferLength: + case FFI::Type::RESERVED_WasNapiEnv: + case FFI::Type::JSValue: + case FFI::Type::Void: + return raw; + } + return raw; +} + +static uint64_t edgeBitsForType(FFI::Type type, WeakRandom& random) +{ + switch (type) { + case FFI::Type::Char: + case FFI::Type::Int8: { + static const int8_t values[] = { 0, 1, -1, 127, -128, 64, -64, 100 }; + return static_cast(static_cast(values[random.getUint32(std::size(values))])); + } + case FFI::Type::Uint8: { + static const uint8_t values[] = { 0, 1, 255, 128, 127, 200 }; + return values[random.getUint32(std::size(values))]; + } + case FFI::Type::Int16: { + static const int16_t values[] = { 0, 1, -1, 32767, -32768, 0x1234, -0x1234 }; + return static_cast(static_cast(values[random.getUint32(std::size(values))])); + } + case FFI::Type::Uint16: { + static const uint16_t values[] = { 0, 1, 65535, 32768, 32767, 4660 }; + return values[random.getUint32(std::size(values))]; + } + case FFI::Type::Int32: { + static const int32_t values[] = { 0, 1, -1, 2147483647, -2147483647 - 1, 0x12345678, -0x12345678, 65536 }; + return static_cast(static_cast(values[random.getUint32(std::size(values))])); + } + case FFI::Type::Uint32: { + static const uint32_t values[] = { 0, 1, 4294967295u, 2147483648u, 2147483647u, 0x9abcdef0u }; + return values[random.getUint32(std::size(values))]; + } + case FFI::Type::Int64: + case FFI::Type::Uint64: + case FFI::Type::Int64Fast: + case FFI::Type::Uint64Fast: + case FFI::Type::BufferLength: { // ABI-identical to Uint64 (a byte length) + static const uint64_t values[] = { + 0, 1, ~0ull, 0x8000000000000000ull, 0x7fffffffffffffffull, 0x0123456789abcdefull, + 0xffffffff80000000ull, 0x00000000ffffffffull, 9007199254740992ull, 0xdeadbeefcafebabeull, + }; + return values[random.getUint32(std::size(values))]; + } + case FFI::Type::Double: { + static const uint64_t values[] = { + 0, 0x8000000000000000ull, std::bit_cast(1.5), std::bit_cast(-1.5), + std::bit_cast(std::numeric_limits::infinity()), std::bit_cast(-std::numeric_limits::infinity()), + std::bit_cast(std::numeric_limits::max()), 1 , std::bit_cast(9007199254740993.0), + 0x7ff8000000000001ull, std::bit_cast(3.141592653589793), + }; + return values[random.getUint32(std::size(values))]; + } + case FFI::Type::Float: { + static const uint32_t values[] = { + 0, 0x80000000u, std::bit_cast(1.5f), std::bit_cast(-1.5f), + std::bit_cast(std::numeric_limits::infinity()), std::bit_cast(-std::numeric_limits::infinity()), + std::bit_cast(std::numeric_limits::max()), 1, 0x7fc00001u, std::bit_cast(2.5f), + }; + return values[random.getUint32(std::size(values))]; + } + case FFI::Type::Bool: + return random.getUint32(2); + case FFI::Type::Pointer: + case FFI::Type::CString: { + static const char* const strings[] = { "", "a", "hello", "0123456789", "h\xc3\xa9!" }; + if (!random.getUint32(6)) + return 0; + return reinterpret_cast(strings[random.getUint32(std::size(strings))]); + } + case FFI::Type::Function: + case FFI::Type::Buffer: + case FFI::Type::RESERVED_WasNapiEnv: { + static const uint64_t values[] = { + 0, 0x00007fffdeadbee0ull, ~0ull, 4096, + reinterpret_cast(&s_scratchMemory[0]), reinterpret_cast(&s_scratchMemory[13]), + }; + return values[random.getUint32(std::size(values))]; + } + case FFI::Type::JSValue: + return random.getUint64(); + case FFI::Type::Void: + return 0; + } + return 0; +} + +static uint64_t randomCanonicalSlot(FFI::Type type, WeakRandom& random) +{ + uint64_t raw = random.returnTrueWithProbability(0.7) ? edgeBitsForType(type, random) : random.getUint64(); + return canonicalizeSlot(type, raw); +} + +static void dumpSlots(std::span slots) +{ + dataLog(" slots:"); + for (uint64_t slot : slots) + dataLog(" ", RawHex(slot)); + dataLogLn(); +} + +template +static void differentialCase(ASCIILiteral name, R (*function)(Arguments...), Vector&& argumentTypes, FFI::Type returnType, unsigned iterations, WeakRandom& random) +{ + constexpr size_t nativeArgumentCount = sizeof...(Arguments); + RefPtr signature = FFI::Signature::tryCreate(argumentTypes.span(), returnType); + FFI_CHECK(!!signature); + if (!signature) + return; + FFI_CHECK_EQ(signature->argumentCount(), static_cast(nativeArgumentCount)); + if (signature->argumentCount() != nativeArgumentCount) + return; + + MacroAssemblerCodeRef code = FFI::generateInvokeThunk(*signature); + FFI_CHECK(!!code); + if (!code) + return; + CodePtr cached = signature->invokeThunk(); + FFI_CHECK(!!cached); + + auto freshThunk = reinterpret_cast(code.code().taggedPtr()); + auto cachedThunk = cached ? reinterpret_cast(cached.taggedPtr()) : freshThunk; + + unsigned argumentCount = signature->argumentCount(); + constexpr uint64_t frontGuard = 0xFEEDFACEFEEDFACEull; + constexpr uint64_t backGuard = 0xBADC0FFEEBADC0FFull; + constexpr uint64_t returnPoison = 0xF00DF00DF00DF00Dull; + + unsigned failuresAtEntry = s_failureCount; + for (unsigned iteration = 0; iteration < iterations && s_failureCount - failuresAtEntry < 6; ++iteration) { + Vector buffer(FillWith { }, argumentCount + 3, 0); + buffer[0] = frontGuard; + uint64_t* slots = buffer.mutableSpan().data() + 1; + for (unsigned i = 0; i < argumentCount; ++i) + slots[i] = randomCanonicalSlot(argumentTypes[i], random); + slots[argumentCount] = returnPoison; + buffer[argumentCount + 2] = backGuard; + + Vector nativeSlots(FillWith { }, argumentCount + 1, 0); + memcpy(nativeSlots.mutableSpan().data(), slots, sizeof(uint64_t) * argumentCount); + + auto thunk = (iteration & 1) ? cachedThunk : freshThunk; + thunk(reinterpret_cast(function), slots); + + s_checkCount++; + bool ok = buffer[0] == frontGuard && buffer[argumentCount + 2] == backGuard; + if constexpr (std::is_void_v) { + callNativeFromSlots(function, nativeSlots.span().data()); + ok &= slots[argumentCount] == returnPoison; // Void return slot untouched + } else { + R nativeResult = callNativeFromSlots(function, nativeSlots.span().data()); + uint64_t expected = canonicalizeSlot(returnType, nativeReturnRawBits(nativeResult)); + uint64_t actual = slots[argumentCount]; + ok &= actual == expected; + if (!ok) { + s_failureCount++; + dataLogLn(" FAIL: invoke-thunk differential for ", name.characters(), " (", signature->toString(), ") iteration ", iteration, + ": thunk return slot ", RawHex(actual), " != native ", RawHex(expected)); + dumpSlots(nativeSlots.span().first(argumentCount)); + continue; + } + } + if (!ok) { + s_failureCount++; + dataLogLn(" FAIL: invoke-thunk differential for ", name.characters(), " (", signature->toString(), ") iteration ", iteration, + ": slot buffer guard corrupted or void return slot written (front ", RawHex(buffer[0]), ", back ", RawHex(buffer[argumentCount + 2]), ", ret ", RawHex(slots[argumentCount]), ")"); + dumpSlots(nativeSlots.span().first(argumentCount)); + } + } +} + +static int32_t staticCbI32(int32_t x) { return static_cast(static_cast(x) * 3u + 1u); } +static double staticCbF64x8(double a, double b, double c, double d, double e, double f, double g, double h) +{ + return a + 2 * b + 3 * c + 4 * d + 5 * e + 6 * f + 7 * g + 8 * h; +} +static double staticCbMix(int32_t a, double b, int64_t c, float d, void* e) +{ + return static_cast(a) + 2 * b + 3 * static_cast(c) + 4 * static_cast(d) + 5 * static_cast(reinterpret_cast(e)); +} +static unsigned s_staticCbVoidCount; +static void staticCbVoid() { ++s_staticCbVoidCount; } +static int64_t staticCbI32x9(int32_t a0, int32_t a1, int32_t a2, int32_t a3, int32_t a4, int32_t a5, int32_t a6, int32_t a7, int32_t a8) +{ + return int64_t(a0) + 2 * int64_t(a1) + 3 * int64_t(a2) + 4 * int64_t(a3) + 5 * int64_t(a4) + 6 * int64_t(a5) + 7 * int64_t(a6) + 8 * int64_t(a7) + 9 * int64_t(a8); +} +static double staticCbF64x9(double a0, double a1, double a2, double a3, double a4, double a5, double a6, double a7, double a8) +{ + return a0 + 2 * a1 + 3 * a2 + 4 * a3 + 5 * a4 + 6 * a5 + 7 * a6 + 8 * a7 + 9 * a8; +} +static int64_t staticCbU8x10(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9) +{ + return int64_t(a0) + 2 * int64_t(a1) + 3 * int64_t(a2) + 4 * int64_t(a3) + 5 * int64_t(a4) + 6 * int64_t(a5) + 7 * int64_t(a6) + 8 * int64_t(a7) + 9 * int64_t(a8) + 10 * int64_t(a9); +} +static int8_t staticCbRetI8() { return -3; } +static uint8_t staticCbRetU8() { return 255; } +static int64_t staticCbRetI64() { return std::numeric_limits::min(); } +static uint64_t staticCbRetU64() { return std::numeric_limits::max(); } +static bool staticCbRetBool() { return true; } +static float staticCbRetF32() { return -1.25f; } +static double staticCbRetF64() { return 6.02214076e23; } +static void* staticCbRetPtr() { return &s_scratchMemory[3]; } + +static void testInvokeThunkDifferential() +{ + using T = FFI::Type; + WeakRandom random(0xF00DFEED); + + differentialCase("ffi_echo_char"_s, ffi_echo_char, { T::Char }, T::Char, 40, random); + differentialCase("ffi_echo_char(as i8)"_s, ffi_echo_char, { T::Int8 }, T::Int8, 20, random); + differentialCase("ffi_echo_i8"_s, ffi_echo_i8, { T::Int8 }, T::Int8, 40, random); + differentialCase("ffi_echo_u8"_s, ffi_echo_u8, { T::Uint8 }, T::Uint8, 40, random); + differentialCase("ffi_echo_i16"_s, ffi_echo_i16, { T::Int16 }, T::Int16, 40, random); + differentialCase("ffi_echo_u16"_s, ffi_echo_u16, { T::Uint16 }, T::Uint16, 40, random); + differentialCase("ffi_echo_i32"_s, ffi_echo_i32, { T::Int32 }, T::Int32, 40, random); + differentialCase("ffi_echo_u32"_s, ffi_echo_u32, { T::Uint32 }, T::Uint32, 40, random); + differentialCase("ffi_echo_i64"_s, ffi_echo_i64, { T::Int64 }, T::Int64, 40, random); + differentialCase("ffi_echo_i64(as i64_fast)"_s, ffi_echo_i64, { T::Int64Fast }, T::Int64Fast, 20, random); + differentialCase("ffi_echo_u64"_s, ffi_echo_u64, { T::Uint64 }, T::Uint64, 40, random); + differentialCase("ffi_echo_u64(as u64_fast)"_s, ffi_echo_u64, { T::Uint64Fast }, T::Uint64Fast, 20, random); + differentialCase("ffi_echo_u64(as buffer_length)"_s, ffi_echo_u64, { T::BufferLength }, T::Uint64, 20, random); + differentialCase("ffi_echo_f32"_s, ffi_echo_f32, { T::Float }, T::Float, 40, random); + differentialCase("ffi_echo_f64"_s, ffi_echo_f64, { T::Double }, T::Double, 40, random); + differentialCase("ffi_echo_bool"_s, ffi_echo_bool, { T::Bool }, T::Bool, 20, random); + differentialCase("ffi_echo_ptr"_s, ffi_echo_ptr, { T::Pointer }, T::Pointer, 40, random); + differentialCase("ffi_echo_ptr(as buffer)"_s, ffi_echo_ptr, { T::Buffer }, T::Pointer, 20, random); + differentialCase("ffi_echo_ptr(as function)"_s, ffi_echo_ptr, { T::Function }, T::Pointer, 20, random); + differentialCase("ffi_echo_cstring"_s, ffi_echo_cstring, { T::CString }, T::CString, 30, random); + differentialCase("ffi_echo_jsvalue"_s, ffi_echo_jsvalue, { T::JSValue }, T::JSValue, 40, random); + differentialCase("ffi_widen_char"_s, ffi_widen_char, { T::Char }, T::Int64, 30, random); + differentialCase("ffi_widen_i8"_s, ffi_widen_i8, { T::Int8 }, T::Int64, 30, random); + differentialCase("ffi_widen_u8"_s, ffi_widen_u8, { T::Uint8 }, T::Int64, 30, random); + differentialCase("ffi_widen_i16"_s, ffi_widen_i16, { T::Int16 }, T::Int64, 30, random); + differentialCase("ffi_widen_u16"_s, ffi_widen_u16, { T::Uint16 }, T::Int64, 30, random); + differentialCase("ffi_ret_null_ptr"_s, ffi_ret_null_ptr, { }, T::Pointer, 4, random); + differentialCase("ffi_ret_two_as_bool"_s, ffi_ret_two_as_bool, { }, T::Bool, 4, random); // declared bool, native returns 2 -> slot must be 1 + differentialCase("ffi_ret_two_as_bool(as i8)"_s, ffi_ret_two_as_bool, { }, T::Int8, 4, random); + differentialCase("ffi_ret_neg_one_i8"_s, ffi_ret_neg_one_i8, { }, T::Int8, 4, random); + differentialCase("ffi_ret_neg_one_i16"_s, ffi_ret_neg_one_i16, { }, T::Int16, 4, random); + differentialCase("ffi_ret_neg_one_i32"_s, ffi_ret_neg_one_i32, { }, T::Int32, 4, random); + differentialCase("ffi_ret_neg_one_i64"_s, ffi_ret_neg_one_i64, { }, T::Int64, 4, random); + differentialCase("ffi_ret_neg_one_u8"_s, ffi_ret_neg_one_u8, { }, T::Uint8, 4, random); + differentialCase("ffi_ret_neg_one_u16"_s, ffi_ret_neg_one_u16, { }, T::Uint16, 4, random); + differentialCase("ffi_ret_neg_one_u32"_s, ffi_ret_neg_one_u32, { }, T::Uint32, 4, random); + differentialCase("ffi_ret_neg_one_u64"_s, ffi_ret_neg_one_u64, { }, T::Uint64, 4, random); + differentialCase("ffi_ret_nan_f32"_s, ffi_ret_nan_f32, { }, T::Float, 4, random); + differentialCase("ffi_ret_impure_nan_f64"_s, ffi_ret_impure_nan_f64, { }, T::Double, 4, random); + differentialCase("ffi_ret_neg_zero_f64"_s, ffi_ret_neg_zero_f64, { }, T::Double, 4, random); + differentialCase("ffi_ret_denormal_f32"_s, ffi_ret_denormal_f32, { }, T::Float, 4, random); + differentialCase("ffi_ret_inf_f64"_s, ffi_ret_inf_f64, { }, T::Double, 4, random); + differentialCase("ffi_add_i32"_s, ffi_add_i32, { T::Int32, T::Int32 }, T::Int32, 60, random); + differentialCase("ffi_add_f64"_s, ffi_add_f64, { T::Double, T::Double }, T::Double, 60, random); + differentialCase("ffi_add_i64"_s, ffi_add_i64, { T::Int64, T::Int64 }, T::Int64, 60, random); + differentialCase("ffi_add_u64"_s, ffi_add_u64, { T::Uint64, T::Uint64 }, T::Uint64, 60, random); + differentialCase("ffi_add_f32"_s, ffi_add_f32, { T::Float, T::Float }, T::Float, 60, random); + differentialCase("ffi_sum_i32_0"_s, ffi_sum_i32_0, { }, T::Int64, 4, random); + differentialCase("ffi_sum_i32_1"_s, ffi_sum_i32_1, { T::Int32 }, T::Int64, 20, random); + differentialCase("ffi_sum_i32_2"_s, ffi_sum_i32_2, Vector(FillWith { }, 2, T::Int32), T::Int64, 20, random); + differentialCase("ffi_sum_i32_4"_s, ffi_sum_i32_4, Vector(FillWith { }, 4, T::Int32), T::Int64, 20, random); + differentialCase("ffi_sum_i32_6"_s, ffi_sum_i32_6, Vector(FillWith { }, 6, T::Int32), T::Int64, 30, random); + differentialCase("ffi_sum_i32_7"_s, ffi_sum_i32_7, Vector(FillWith { }, 7, T::Int32), T::Int64, 30, random); + differentialCase("ffi_sum_i32_8"_s, ffi_sum_i32_8, Vector(FillWith { }, 8, T::Int32), T::Int64, 30, random); + differentialCase("ffi_sum_i32_9"_s, ffi_sum_i32_9, Vector(FillWith { }, 9, T::Int32), T::Int64, 30, random); + differentialCase("ffi_sum_i32_12"_s, ffi_sum_i32_12, Vector(FillWith { }, 12, T::Int32), T::Int64, 30, random); + differentialCase("ffi_sum_i32_16"_s, ffi_sum_i32_16, Vector(FillWith { }, 16, T::Int32), T::Int64, 30, random); + differentialCase("ffi_sum_f64_1"_s, ffi_sum_f64_1, { T::Double }, T::Double, 20, random); + differentialCase("ffi_sum_f64_2"_s, ffi_sum_f64_2, Vector(FillWith { }, 2, T::Double), T::Double, 20, random); + differentialCase("ffi_sum_f64_7"_s, ffi_sum_f64_7, Vector(FillWith { }, 7, T::Double), T::Double, 30, random); + differentialCase("ffi_sum_f64_8"_s, ffi_sum_f64_8, Vector(FillWith { }, 8, T::Double), T::Double, 30, random); + differentialCase("ffi_sum_f64_9"_s, ffi_sum_f64_9, Vector(FillWith { }, 9, T::Double), T::Double, 30, random); + differentialCase("ffi_sum_f64_12"_s, ffi_sum_f64_12, Vector(FillWith { }, 12, T::Double), T::Double, 30, random); + differentialCase("ffi_sum_u8_10"_s, ffi_sum_u8_10, Vector(FillWith { }, 10, T::Uint8), T::Int64, 30, random); + differentialCase("ffi_sum_u8_12"_s, ffi_sum_u8_12, Vector(FillWith { }, 12, T::Uint8), T::Int64, 30, random); + differentialCase("ffi_sum_i16_10"_s, ffi_sum_i16_10, Vector(FillWith { }, 10, T::Int16), T::Int64, 30, random); + differentialCase("ffi_sum_i16_12"_s, ffi_sum_i16_12, Vector(FillWith { }, 12, T::Int16), T::Int64, 30, random); + differentialCase("ffi_mix_1"_s, ffi_mix_1, { T::Int32, T::Double, T::Int64, T::Float, T::Pointer, T::Uint8, T::Double, T::Int16, T::Double, T::Int32 }, T::Double, 60, random); + differentialCase("ffi_mix_2"_s, ffi_mix_2, { T::Float, T::Int32, T::Float, T::Int32, T::Float, T::Int32, T::Float, T::Int32, T::Float, T::Int32 }, T::Double, 60, random); + differentialCase("ffi_mix_3"_s, ffi_mix_3, { T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Int32 }, T::Double, 60, random); + differentialCase("ffi_mix_4"_s, ffi_mix_4, { T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Int64, T::Double, T::Int64, T::Double }, T::Double, 60, random); + differentialCase("ffi_mix_5"_s, ffi_mix_5, { T::Uint8, T::Int8, T::Uint16, T::Int16, T::Uint32, T::Int32, T::Uint64, T::Int64 }, T::Double, 60, random); + differentialCase("ffi_mix_6"_s, ffi_mix_6, { T::Bool, T::Bool, T::Int32, T::Bool, T::Double, T::Bool, T::Float, T::Bool, T::Bool, T::Bool, T::Bool, T::Bool, T::Bool }, T::Double, 60, random); + differentialCase("ffi_mix_7"_s, ffi_mix_7, { T::Pointer, T::Char, T::Pointer, T::Char, T::Pointer, T::Char, T::Pointer, T::Char, T::Pointer, T::Char }, T::Double, 60, random); + differentialCase("ffi_mix_8"_s, ffi_mix_8, { T::Float, T::Double, T::Float, T::Double, T::Float, T::Double, T::Float, T::Double, T::Float, T::Double, T::Float, T::Double }, T::Double, 60, random); + differentialCase("ffi_ptr_identity"_s, ffi_ptr_identity, { T::Pointer }, T::Pointer, 30, random); + differentialCase("ffi_high_ptr"_s, ffi_high_ptr, { }, T::Pointer, 4, random); + differentialCase("ffi_align_probe_0"_s, ffi_align_probe_0, { }, T::Double, 8, random); + differentialCase("ffi_align_probe_9"_s, ffi_align_probe_9, Vector(FillWith { }, 9, T::Int32), T::Double, 12, random); + + { + auto runWithCallback = [&](ASCIILiteral name, auto* fixture, Vector&& argumentTypes, FFI::Type returnType, uint64_t callbackSlot, unsigned iterations) { + RefPtr signature = FFI::Signature::tryCreate(argumentTypes.span(), returnType); + FFI_CHECK(!!signature); + if (!signature) + return; + MacroAssemblerCodeRef code = FFI::generateInvokeThunk(*signature); + FFI_CHECK(!!code); + if (!code) + return; + auto thunk = reinterpret_cast(code.code().taggedPtr()); + unsigned argumentCount = signature->argumentCount(); + for (unsigned iteration = 0; iteration < iterations; ++iteration) { + Vector buffer(FillWith { }, argumentCount + 3, 0); + buffer[0] = 0xFEEDFACEFEEDFACEull; + uint64_t* slots = buffer.mutableSpan().data() + 1; + slots[0] = callbackSlot; + for (unsigned i = 1; i < argumentCount; ++i) + slots[i] = randomCanonicalSlot(argumentTypes[i], random); + slots[argumentCount] = 0xF00DF00DF00DF00Dull; + buffer[argumentCount + 2] = 0xBADC0FFEEBADC0FFull; + Vector nativeSlots(FillWith { }, argumentCount + 1, 0); + memcpy(nativeSlots.mutableSpan().data(), slots, sizeof(uint64_t) * argumentCount); + + thunk(reinterpret_cast(fixture), slots); + auto nativeResult = callNativeFromSlots(fixture, nativeSlots.span().data()); + uint64_t expected = canonicalizeSlot(returnType, nativeReturnRawBits(nativeResult)); + s_checkCount++; + bool ok = slots[argumentCount] == expected + && buffer[0] == 0xFEEDFACEFEEDFACEull + && buffer[argumentCount + 2] == 0xBADC0FFEEBADC0FFull; + if (!ok) { + s_failureCount++; + dataLogLn(" FAIL: callback differential for ", name.characters(), " iteration ", iteration, ": thunk ", RawHex(slots[argumentCount]), " native ", RawHex(expected)); + dumpSlots(nativeSlots.span().first(argumentCount)); + } + } + }; + runWithCallback("ffi_call_cb_i32"_s, ffi_call_cb_i32, { T::Function, T::Int32 }, T::Int32, + reinterpret_cast(&staticCbI32), 20); + runWithCallback("ffi_call_cb_f64_x8"_s, ffi_call_cb_f64_x8, { T::Function, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double }, T::Double, + reinterpret_cast(&staticCbF64x8), 20); + runWithCallback("ffi_call_cb_mix"_s, ffi_call_cb_mix, { T::Function, T::Int32, T::Double, T::Int64, T::Float, T::Pointer }, T::Double, + reinterpret_cast(&staticCbMix), 20); + runWithCallback("ffi_call_cb_i32_x9"_s, ffi_call_cb_i32_x9, { T::Function, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32, T::Int32 }, T::Int64, + reinterpret_cast(&staticCbI32x9), 20); + runWithCallback("ffi_call_cb_f64_x9"_s, ffi_call_cb_f64_x9, { T::Function, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double, T::Double }, T::Double, + reinterpret_cast(&staticCbF64x9), 20); + runWithCallback("ffi_call_cb_u8_x10"_s, ffi_call_cb_u8_x10, { T::Function, T::Uint8, T::Uint8, T::Uint8, T::Uint8, T::Uint8, T::Uint8, T::Uint8, T::Uint8, T::Uint8, T::Uint8 }, T::Int64, + reinterpret_cast(&staticCbU8x10), 20); + runWithCallback("ffi_call_cb_ret_i8"_s, ffi_call_cb_ret_i8, { T::Function }, T::Int64, reinterpret_cast(&staticCbRetI8), 3); + runWithCallback("ffi_call_cb_ret_u8"_s, ffi_call_cb_ret_u8, { T::Function }, T::Int64, reinterpret_cast(&staticCbRetU8), 3); + runWithCallback("ffi_call_cb_ret_i64"_s, ffi_call_cb_ret_i64, { T::Function }, T::Int64, reinterpret_cast(&staticCbRetI64), 3); + runWithCallback("ffi_call_cb_ret_u64"_s, ffi_call_cb_ret_u64, { T::Function }, T::Uint64, reinterpret_cast(&staticCbRetU64), 3); + runWithCallback("ffi_call_cb_ret_bool"_s, ffi_call_cb_ret_bool, { T::Function }, T::Int32, reinterpret_cast(&staticCbRetBool), 3); + runWithCallback("ffi_call_cb_ret_f32"_s, ffi_call_cb_ret_f32, { T::Function }, T::Float, reinterpret_cast(&staticCbRetF32), 3); + runWithCallback("ffi_call_cb_ret_f64"_s, ffi_call_cb_ret_f64, { T::Function }, T::Double, reinterpret_cast(&staticCbRetF64), 3); + runWithCallback("ffi_call_cb_ret_ptr"_s, ffi_call_cb_ret_ptr, { T::Function }, T::Pointer, reinterpret_cast(&staticCbRetPtr), 3); + } + + { + Vector argumentTypes { T::Function }; + RefPtr signature = FFI::Signature::tryCreate(argumentTypes.span(), T::Void); + FFI_CHECK(!!signature); + if (signature) { + MacroAssemblerCodeRef code = FFI::generateInvokeThunk(*signature); + FFI_CHECK(!!code); + if (code) { + auto thunk = reinterpret_cast(code.code().taggedPtr()); + s_staticCbVoidCount = 0; + uint64_t slots[2] = { static_cast(reinterpret_cast(&staticCbVoid)), 0 }; + thunk(reinterpret_cast(ffi_call_cb_void), slots); + thunk(reinterpret_cast(ffi_call_cb_void), slots); + ffi_call_cb_void(staticCbVoid); + FFI_CHECK_EQ(s_staticCbVoidCount, 3u); + } + } + } + + { + Vector writeTypes { T::Pointer, T::Uint32 }; + RefPtr writeSignature = FFI::Signature::tryCreate(writeTypes.span(), T::Void); + Vector readTypes { T::Pointer }; + RefPtr readSignature = FFI::Signature::tryCreate(readTypes.span(), T::Uint32); + Vector strlenTypes { T::CString }; + RefPtr strlenSignature = FFI::Signature::tryCreate(strlenTypes.span(), T::Uint64); + FFI_CHECK(!!writeSignature); + FFI_CHECK(!!readSignature); + FFI_CHECK(!!strlenSignature); + if (writeSignature && readSignature && strlenSignature) { + MacroAssemblerCodeRef writeCode = FFI::generateInvokeThunk(*writeSignature); + MacroAssemblerCodeRef readCode = FFI::generateInvokeThunk(*readSignature); + MacroAssemblerCodeRef strlenCode = FFI::generateInvokeThunk(*strlenSignature); + FFI_CHECK(!!writeCode); + FFI_CHECK(!!readCode); + FFI_CHECK(!!strlenCode); + if (writeCode && readCode && strlenCode) { + auto writeThunk = reinterpret_cast(writeCode.code().taggedPtr()); + auto readThunk = reinterpret_cast(readCode.code().taggedPtr()); + auto strlenThunk = reinterpret_cast(strlenCode.code().taggedPtr()); + static uint32_t words[8]; + static const uint32_t patterns[] = { 0, 0xdeadbeefu, 0xffffffffu, 1, 0x80000000u, 0x12345678u }; + for (unsigned i = 0; i < std::size(words); ++i) { + for (uint32_t pattern : patterns) { + words[i] = ~pattern; + uint64_t writeSlots[3] = { static_cast(reinterpret_cast(&words[i])), pattern, 0xdead }; + writeThunk(reinterpret_cast(ffi_ptr_write_u32), writeSlots); + FFI_CHECK_EQ(words[i], pattern); + uint64_t readSlots[2] = { static_cast(reinterpret_cast(&words[i])), 0xF00DF00DF00DF00Dull }; + readThunk(reinterpret_cast(ffi_ptr_read_u32), readSlots); + FFI_CHECK_EQ_HEX(readSlots[1], static_cast(pattern)); // u32 return zero-extended + } + } + static const char* strings[] = { "", "a", "hello, world", "0123456789012345678901234567890123456789012345678901234567890123456789" }; + for (const char* string : strings) { + uint64_t strlenSlots[2] = { static_cast(reinterpret_cast(string)), 0xF00DF00DF00DF00Dull }; + strlenThunk(reinterpret_cast(ffi_strlen), strlenSlots); + FFI_CHECK_EQ(strlenSlots[1], static_cast(strlen(string))); + } + } + } + } +} + +static struct { + FFI::InvokeThunkFunction thunk { nullptr }; + void* target { nullptr }; + uint64_t slots[4]; +} s_canaryInvokeState; + +static void canaryInvokeBody() +{ + s_canaryInvokeState.thunk(s_canaryInvokeState.target, &s_canaryInvokeState.slots[0]); +} + +static unsigned s_hostCanaryHits; + +static JSC_DECLARE_HOST_FUNCTION(functionCanaryTarget); +JSC_DEFINE_HOST_FUNCTION(functionCanaryTarget, (JSGlobalObject*, CallFrame*)) +{ + ++s_hostCanaryHits; + return JSValue::encode(jsUndefined()); +} + +static JSC_DECLARE_HOST_FUNCTION(functionCanaryTargetAllocating); +JSC_DEFINE_HOST_FUNCTION(functionCanaryTargetAllocating, (JSGlobalObject* globalObject, CallFrame*)) +{ + VM& vm = globalObject->vm(); + ++s_hostCanaryHits; + JSObject* garbage = nullptr; + for (unsigned i = 0; i < 200; ++i) { + JSObject* object = constructEmptyObject(globalObject); + object->putDirect(vm, vm.propertyNames->length, jsNumber(i)); + garbage = object; + } + return JSValue::encode(garbage ? JSValue(garbage) : jsUndefined()); +} + +static void testCanaries() +{ + VM& vm = *s_vm; + JSGlobalObject* globalObject = s_globalObject; + + { + Vector argumentTypes { FFI::Type::Int64, FFI::Type::Int64 }; + RefPtr signature = FFI::Signature::tryCreate(argumentTypes.span(), FFI::Type::Int64); + FFI_CHECK(!!signature); + if (signature) { + CodePtr thunk = signature->invokeThunk(); + FFI_CHECK(!!thunk); + if (thunk) { + s_canaryInvokeState.thunk = reinterpret_cast(thunk.taggedPtr()); + s_canaryInvokeState.target = reinterpret_cast(ffi_add_i64); + s_canaryInvokeState.slots[0] = 40; + s_canaryInvokeState.slots[1] = 2; + s_canaryInvokeState.slots[2] = 0; + int32_t mask = ffi_canary_call(canaryInvokeBody); + FFI_CHECK_EQ(mask, 0); + FFI_CHECK_EQ_HEX(s_canaryInvokeState.slots[2], 42); + } + } + } + + { + JSFunction* trivialTarget = JSFunction::create(vm, globalObject, 0, "canaryTarget"_s, functionCanaryTarget, ImplementationVisibility::Public); + JSFunction* allocatingTarget = JSFunction::create(vm, globalObject, 0, "canaryTargetAllocating"_s, functionCanaryTargetAllocating, ImplementationVisibility::Public); + RefPtr voidSignature = FFI::Signature::tryCreate({ }, FFI::Type::Void); + FFI_CHECK(!!voidSignature); + if (voidSignature) { + JSFFICallback* trivialCallback = JSFFICallback::create(vm, globalObject, globalObject->ffiCallbackStructure(), trivialTarget, voidSignature.releaseNonNull()); + FFI_CHECK(!!trivialCallback); + if (trivialCallback) { + gcProtect(trivialCallback); + FFI_CHECK(trivialCallback->nativeEntrypoint()); + if (trivialCallback->nativeEntrypoint()) { + s_hostCanaryHits = 0; + auto entry = reinterpret_cast(trivialCallback->nativeEntrypoint()); + int32_t mask = ffi_canary_call(entry); + FFI_CHECK_EQ(mask, 0); + FFI_CHECK_EQ(s_hostCanaryHits, 1u); + entry(); + FFI_CHECK_EQ(s_hostCanaryHits, 2u); + FFI_CHECK_EQ(ffi_align_probe_0(), 1.0); + } + } + } + RefPtr objectSignature = FFI::Signature::tryCreate({ }, FFI::Type::JSValue); + FFI_CHECK(!!objectSignature); + if (objectSignature) { + JSFFICallback* allocatingCallback = JSFFICallback::create(vm, globalObject, globalObject->ffiCallbackStructure(), allocatingTarget, objectSignature.releaseNonNull()); + FFI_CHECK(!!allocatingCallback); + if (allocatingCallback) { + gcProtect(allocatingCallback); + s_hostCanaryHits = 0; + int32_t mask = ffi_canary_call(reinterpret_cast(allocatingCallback->nativeEntrypoint())); + FFI_CHECK_EQ(mask, 0); + FFI_CHECK_EQ(s_hostCanaryHits, 1u); + gcUnprotect(allocatingCallback); + } + } + } + + { + Vector nine(FillWith { }, 9, FFI::Type::Int32); + RefPtr nineSignature = FFI::Signature::tryCreate(nine.span(), FFI::Type::Double); + RefPtr zeroSignature = FFI::Signature::tryCreate({ }, FFI::Type::Double); + FFI_CHECK(!!nineSignature); + FFI_CHECK(!!zeroSignature); + if (nineSignature && zeroSignature) { + CodePtr nineThunk = nineSignature->invokeThunk(); + CodePtr zeroThunk = zeroSignature->invokeThunk(); + FFI_CHECK(!!nineThunk); + FFI_CHECK(!!zeroThunk); + if (nineThunk && zeroThunk) { + uint64_t nineSlots[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 0 }; + reinterpret_cast(nineThunk.taggedPtr())(reinterpret_cast(ffi_align_probe_9), nineSlots); + FFI_CHECK_EQ(std::bit_cast(nineSlots[9]), 1.0); + uint64_t zeroSlots[1] = { 0 }; + reinterpret_cast(zeroThunk.taggedPtr())(reinterpret_cast(ffi_align_probe_0), zeroSlots); + FFI_CHECK_EQ(std::bit_cast(zeroSlots[0]), 1.0); + } + } + } +} + +static JSC_DECLARE_HOST_FUNCTION(functionTimesThreePlusOne); +JSC_DEFINE_HOST_FUNCTION(functionTimesThreePlusOne, (JSGlobalObject*, CallFrame* callFrame)) +{ + JSValue argument = callFrame->argument(0); + double x = argument.isNumber() ? argument.asNumber() : 0.0; + return JSValue::encode(jsNumber(x * 3.0 + 1.0)); +} + +static JSC_DECLARE_HOST_FUNCTION(functionSumWeightedArguments); +JSC_DEFINE_HOST_FUNCTION(functionSumWeightedArguments, (JSGlobalObject*, CallFrame* callFrame)) +{ + double sum = 0; + for (unsigned i = 0; i < callFrame->argumentCount(); ++i) { + JSValue argument = callFrame->uncheckedArgument(i); + double value = 0; + if (argument.isNumber()) + value = argument.asNumber(); + else if (argument.isBoolean()) + value = argument.asBoolean() ? 1 : 0; + sum += (i + 1) * value; + } + return JSValue::encode(jsNumber(sum)); +} + +static JSC_DECLARE_HOST_FUNCTION(functionReturnTwo); +JSC_DEFINE_HOST_FUNCTION(functionReturnTwo, (JSGlobalObject*, CallFrame*)) +{ + return JSValue::encode(jsNumber(2)); +} + +static JSC_DECLARE_HOST_FUNCTION(functionReturnNaN); +JSC_DEFINE_HOST_FUNCTION(functionReturnNaN, (JSGlobalObject*, CallFrame*)) +{ + return JSValue::encode(jsNaN()); +} + +static JSC_DECLARE_HOST_FUNCTION(functionReturnMinusOne); +JSC_DEFINE_HOST_FUNCTION(functionReturnMinusOne, (JSGlobalObject*, CallFrame*)) +{ + return JSValue::encode(jsNumber(-1)); +} + +static JSValue callFunction(JSValue function, const MarkedArgumentBuffer& arguments) +{ + VM& vm = *s_vm; + JSGlobalObject* globalObject = s_globalObject; + auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + CallData callData = getCallData(function); + if (callData.type == CallData::Type::None) { + s_checkCount++; + s_failureCount++; + dataLogLn(" FAIL: value is not callable"); + return jsUndefined(); + } + JSValue result = call(globalObject, function, callData, jsUndefined(), arguments); + Exception* exception = scope.exception(); + if (exception) { + s_checkCount++; + s_failureCount++; + JSValue thrown = exception->value(); + scope.clearException(); + ErrorInstance* error = dynamicDowncast(thrown); + dataLogLn(" FAIL: unexpected exception from JSFFIFunction call (", error ? "an Error instance" : "a non-Error value", ")"); + return jsUndefined(); + } + return result; +} + +static void testJSFFIFunctionEndToEnd() +{ + VM& vm = *s_vm; + JSGlobalObject* globalObject = s_globalObject; + using T = FFI::Type; + + auto makeFunction = [&](Vector&& argumentTypes, T returnType, void* target, ASCIILiteral name) -> JSFFIFunction* { + RefPtr signature = FFI::Signature::tryCreate(argumentTypes.span(), returnType); + FFI_CHECK(!!signature); + if (!signature) + return nullptr; + JSFFIFunction* function = JSFFIFunction::create(vm, globalObject, globalObject->ffiFunctionStructure(), signature.releaseNonNull(), target, String(name)); + FFI_CHECK(!!function); + if (function) + gcProtect(function); + return function; + }; + + auto icStubsBefore = FFI::g_ffiCompileCounts.icStub.load(); + + if (JSFFIFunction* addI32 = makeFunction({ T::Int32, T::Int32 }, T::Int32, reinterpret_cast(ffi_add_i32), "ffi_add_i32"_s)) { + FFI_CHECK_EQ(addI32->signature().argumentCount(), 2u); + FFI_CHECK(addI32->target() == reinterpret_cast(ffi_add_i32)); + MarkedArgumentBuffer arguments; + arguments.append(jsNumber(40)); + arguments.append(jsNumber(2)); + FFI_CHECK(callFunction(addI32, arguments) == jsNumber(42)); + MarkedArgumentBuffer overflow; + overflow.append(jsNumber(2147483647)); + overflow.append(jsNumber(1)); + FFI_CHECK(callFunction(addI32, overflow) == jsNumber(-2147483647 - 1)); // wraps + MarkedArgumentBuffer missing; + missing.append(jsNumber(7)); + FFI_CHECK(callFunction(addI32, missing) == jsNumber(7)); // missing argument -> 0 + MarkedArgumentBuffer extra; + extra.append(jsNumber(1)); + extra.append(jsNumber(2)); + extra.append(jsNumber(4)); + FFI_CHECK(callFunction(addI32, extra) == jsNumber(3)); // extra arguments ignored + MarkedArgumentBuffer doubles; + doubles.append(jsDoubleNumber(1.9)); + doubles.append(jsDoubleNumber(-2.9)); + FFI_CHECK(callFunction(addI32, doubles) == jsNumber(-1)); // toInt32 truncation + } + + if (JSFFIFunction* echoBool = makeFunction({ T::Bool }, T::Bool, reinterpret_cast(ffi_echo_bool), "ffi_echo_bool"_s)) { + static const struct { double input; bool expected; } cases[] = { + { 0, false }, { 1, true }, { 2, true }, { -1, true }, { 0.5, true }, { -0.0, false }, + }; + for (auto& c : cases) { + MarkedArgumentBuffer arguments; + arguments.append(jsNumber(c.input)); + FFI_CHECK(callFunction(echoBool, arguments) == jsBoolean(c.expected)); + } + MarkedArgumentBuffer booleans; + booleans.append(jsBoolean(true)); + FFI_CHECK(callFunction(echoBool, booleans) == jsBoolean(true)); + MarkedArgumentBuffer nanArguments; + nanArguments.append(jsNaN()); + FFI_CHECK(callFunction(echoBool, nanArguments) == jsBoolean(false)); + } + if (JSFFIFunction* twoAsBool = makeFunction({ }, T::Bool, reinterpret_cast(ffi_ret_two_as_bool), "ffi_ret_two_as_bool"_s)) { + MarkedArgumentBuffer arguments; + FFI_CHECK(callFunction(twoAsBool, arguments) == jsBoolean(true)); + } + + if (JSFFIFunction* nullPtr = makeFunction({ }, T::Pointer, reinterpret_cast(ffi_ret_null_ptr), "ffi_ret_null_ptr"_s)) { + MarkedArgumentBuffer arguments; + FFI_CHECK(callFunction(nullPtr, arguments).isNull()); + } + if (JSFFIFunction* echoPtr = makeFunction({ T::Pointer }, T::Pointer, reinterpret_cast(ffi_echo_ptr), "ffi_echo_ptr"_s)) { + MarkedArgumentBuffer zero; + zero.append(jsNumber(0)); + FFI_CHECK(callFunction(echoPtr, zero).isNull()); + MarkedArgumentBuffer nullArgument; + nullArgument.append(jsNull()); + FFI_CHECK(callFunction(echoPtr, nullArgument).isNull()); + MarkedArgumentBuffer minusOne; + minusOne.append(jsNumber(-1)); + JSValue signExtended = callFunction(echoPtr, minusOne); + FFI_CHECK(signExtended.isBigInt()); + if (signExtended.isBigInt()) + FFI_CHECK_EQ(JSBigInt::toBigUInt64(signExtended), 0xFFFFFFFFFFFFFFFFull); + MarkedArgumentBuffer typedArray; + JSUint8Array* uint8Array = makeUint8Array(16); + if (uint8Array) { + typedArray.append(uint8Array); + JSValue address = callFunction(echoPtr, typedArray); + FFI_CHECK(address.isNumber()); + if (address.isNumber()) + FFI_CHECK_EQ(address.asNumber(), static_cast(reinterpret_cast(uint8Array->vector()))); + } + } + + if (JSFFIFunction* nanF32 = makeFunction({ }, T::Float, reinterpret_cast(ffi_ret_nan_f32), "ffi_ret_nan_f32"_s)) { + MarkedArgumentBuffer arguments; + JSValue result = callFunction(nanF32, arguments); + FFI_CHECK(result.isNumber()); + if (result.isNumber()) + FFI_CHECK(std::isnan(result.asNumber())); + } + if (JSFFIFunction* nanF64 = makeFunction({ }, T::Double, reinterpret_cast(ffi_ret_impure_nan_f64), "ffi_ret_impure_nan_f64"_s)) { + MarkedArgumentBuffer arguments; + JSValue result = callFunction(nanF64, arguments); + FFI_CHECK(result.isNumber()); + if (result.isNumber()) + FFI_CHECK(std::isnan(result.asNumber())); + } + if (JSFFIFunction* denormal = makeFunction({ }, T::Float, reinterpret_cast(ffi_ret_denormal_f32), "ffi_ret_denormal_f32"_s)) { + MarkedArgumentBuffer arguments; + JSValue result = callFunction(denormal, arguments); + FFI_CHECK(result.isNumber()); + if (result.isNumber()) + FFI_CHECK_EQ(result.asNumber(), static_cast(std::bit_cast(0x00000001u))); + } + if (JSFFIFunction* negZero = makeFunction({ }, T::Double, reinterpret_cast(ffi_ret_neg_zero_f64), "ffi_ret_neg_zero_f64"_s)) { + MarkedArgumentBuffer arguments; + JSValue result = callFunction(negZero, arguments); + FFI_CHECK(result.isNumber()); + if (result.isNumber()) + FFI_CHECK(std::signbit(result.asNumber()) && result.asNumber() == 0.0); + } + + if (JSFFIFunction* negOneI64 = makeFunction({ }, T::Int64, reinterpret_cast(ffi_ret_neg_one_i64), "ffi_ret_neg_one_i64"_s)) { + MarkedArgumentBuffer arguments; + JSValue result = callFunction(negOneI64, arguments); + FFI_CHECK(result.isBigInt()); + if (result.isBigInt()) + FFI_CHECK_EQ(JSBigInt::toBigInt64(result), -1); + } + if (JSFFIFunction* negOneU64Fast = makeFunction({ }, T::Uint64Fast, reinterpret_cast(ffi_ret_neg_one_u64), "ffi_ret_neg_one_u64(as u64_fast)"_s)) { + MarkedArgumentBuffer arguments; + JSValue result = callFunction(negOneU64Fast, arguments); + FFI_CHECK(result.isBigInt()); + if (result.isBigInt()) + FFI_CHECK_EQ_HEX(JSBigInt::toBigUInt64(result), 0xffffffffffffffffull); + } + if (JSFFIFunction* widenChar = makeFunction({ T::Char }, T::Int64, reinterpret_cast(ffi_widen_char), "ffi_widen_char"_s)) { + MarkedArgumentBuffer arguments; + arguments.append(jsNumber(255)); + JSValue result = callFunction(widenChar, arguments); + FFI_CHECK(result.isBigInt()); + if (result.isBigInt()) + FFI_CHECK_EQ(JSBigInt::toBigInt64(result), -1); // char is signed + } + + if (JSFFIFunction* strlenFunction = makeFunction({ T::CString }, T::Uint64, reinterpret_cast(ffi_strlen), "ffi_strlen"_s)) { + MarkedArgumentBuffer arguments; + arguments.append(jsString(vm, String("hello"_s))); + JSValue result = callFunction(strlenFunction, arguments); + FFI_CHECK(result.isBigInt()); + if (result.isBigInt()) + FFI_CHECK_EQ(JSBigInt::toBigUInt64(result), 5u); + MarkedArgumentBuffer utf8; + utf8.append(jsString(vm, String::fromUTF8("h\xC3\xA9llo"))); + JSValue utf8Result = callFunction(strlenFunction, utf8); + FFI_CHECK(utf8Result.isBigInt()); + if (utf8Result.isBigInt()) + FFI_CHECK_EQ(JSBigInt::toBigUInt64(utf8Result), 6u); + } + + if (Options::useFFIICStub()) + FFI_CHECK(FFI::g_ffiCompileCounts.icStub.load() > icStubsBefore); + + if (JSFFIFunction* addI32 = makeFunction({ T::Int32, T::Int32 }, T::Int32, reinterpret_cast(ffi_add_i32), "ffi_add_i32"_s)) { + auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + MarkedArgumentBuffer arguments; + arguments.append(jsNumber(1)); + arguments.append(jsNumber(2)); + JSValue constructed = construct(globalObject, addI32, arguments, "not a constructor"_s); + FFI_CHECK(!!scope.exception()); + UNUSED_PARAM(constructed); + scope.clearException(); + } +} + +static void testCallbackThunkEndToEnd() +{ + VM& vm = *s_vm; + JSGlobalObject* globalObject = s_globalObject; + using T = FFI::Type; + + auto makeCallback = [&](Vector&& argumentTypes, T returnType, JSFunction* target) -> JSFFICallback* { + RefPtr signature = FFI::Signature::tryCreate(argumentTypes.span(), returnType); + FFI_CHECK(!!signature); + if (!signature) + return nullptr; + JSFFICallback* callback = JSFFICallback::create(vm, globalObject, globalObject->ffiCallbackStructure(), target, signature.releaseNonNull()); + FFI_CHECK(!!callback); + if (callback) { + gcProtect(callback); + FFI_CHECK(callback->nativeEntrypoint()); + } + return callback; + }; + + JSFunction* timesThreePlusOne = JSFunction::create(vm, globalObject, 1, "timesThreePlusOne"_s, functionTimesThreePlusOne, ImplementationVisibility::Public); + JSFunction* weighted = JSFunction::create(vm, globalObject, 0, "weighted"_s, functionSumWeightedArguments, ImplementationVisibility::Public); + JSFunction* returnsTwo = JSFunction::create(vm, globalObject, 0, "returnsTwo"_s, functionReturnTwo, ImplementationVisibility::Public); + JSFunction* returnsNaN = JSFunction::create(vm, globalObject, 0, "returnsNaN"_s, functionReturnNaN, ImplementationVisibility::Public); + JSFunction* returnsMinusOne = JSFunction::create(vm, globalObject, 0, "returnsMinusOne"_s, functionReturnMinusOne, ImplementationVisibility::Public); + + if (JSFFICallback* callback = makeCallback({ T::Int32 }, T::Int32, timesThreePlusOne)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK_EQ(ffi_call_cb_i32(function, 7), 22); + FFI_CHECK_EQ(ffi_call_cb_i32(function, -1), -2); + FFI_CHECK_EQ(ffi_call_cb_i32(function, 0), 1); + FFI_CHECK_EQ(ffi_call_cb_i32(function, 2147483647), static_cast(static_cast(static_cast(2147483647.0 * 3.0 + 1.0)))); + FFI_CHECK_EQ(ffi_call_cb_reentrant(function, 100), [] { + int32_t sum = 0; + for (int32_t i = 0; i < 100; ++i) + sum += i * 3 + 1; + return sum; + }()); + } + if (JSFFICallback* callback = makeCallback(Vector(FillWith { }, 9, T::Int32), T::Int64, weighted)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + int64_t expected = 1 * 1 + 2 * -2 + 3 * 3 + 4 * -4 + 5 * 5 + 6 * -6 + 7 * 7 + 8 * -8 + 9 * 2147483647ll; + FFI_CHECK_EQ(ffi_call_cb_i32_x9(function, 1, -2, 3, -4, 5, -6, 7, -8, 2147483647), expected); + } + if (JSFFICallback* callback = makeCallback(Vector(FillWith { }, 9, T::Double), T::Double, weighted)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + double expected = 1 * 0.5 + 2 * -0.25 + 3 * 3.5 + 4 * 4.5 + 5 * 5.5 + 6 * 6.5 + 7 * 7.5 + 8 * 8.5 + 9 * 1e10; + FFI_CHECK_EQ(ffi_call_cb_f64_x9(function, 0.5, -0.25, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 1e10), expected); + } + if (JSFFICallback* callback = makeCallback(Vector(FillWith { }, 10, T::Uint8), T::Int64, weighted)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + int64_t expected = 1 * 255 + 2 * 0 + 3 * 128 + 4 * 1 + 5 * 200 + 6 * 17 + 7 * 254 + 8 * 3 + 9 * 99 + 10 * 250; + FFI_CHECK_EQ(ffi_call_cb_u8_x10(function, 255, 0, 128, 1, 200, 17, 254, 3, 99, 250), expected); + } + if (JSFFICallback* callback = makeCallback({ T::Int32, T::Double, T::Int64Fast, T::Float, T::Pointer }, T::Double, weighted)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + void* pointer = reinterpret_cast(static_cast(8192)); + double expected = 1 * -5.0 + 2 * 2.5 + 3 * 4503599627370496.0 + 4 * static_cast(1.25f) + 5 * 8192.0; + FFI_CHECK_EQ(ffi_call_cb_mix(function, -5, 2.5, 4503599627370496ll, 1.25f, pointer), expected); + } + if (JSFFICallback* callback = makeCallback({ }, T::Bool, returnsTwo)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK_EQ(ffi_call_cb_ret_bool(function), 10); // JS 2 -> toBoolean -> true -> exactly 1 in the register + } + if (JSFFICallback* callback = makeCallback({ }, T::Uint8, returnsMinusOne)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK_EQ(ffi_call_cb_ret_u8(function), 255); + } + if (JSFFICallback* callback = makeCallback({ }, T::Int8, returnsMinusOne)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK_EQ(ffi_call_cb_ret_i8(function), -1); + } + if (JSFFICallback* callback = makeCallback({ }, T::Int64, returnsMinusOne)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK_EQ(ffi_call_cb_ret_i64(function), -1); + } + if (JSFFICallback* callback = makeCallback({ }, T::Uint64, returnsMinusOne)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK_EQ_HEX(ffi_call_cb_ret_u64(function), 0xffffffffffffffffull); + } + if (JSFFICallback* callback = makeCallback({ }, T::Float, returnsNaN)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK(std::isnan(ffi_call_cb_ret_f32(function))); + } + if (JSFFICallback* callback = makeCallback({ }, T::Double, returnsNaN)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK(std::isnan(ffi_call_cb_ret_f64(function))); + } + if (JSFFICallback* callback = makeCallback({ }, T::Pointer, returnsMinusOne)) { + auto function = reinterpret_cast(callback->nativeEntrypoint()); + FFI_CHECK_EQ(reinterpret_cast(ffi_call_cb_ret_ptr(function)), static_cast(0xffffffffffffffffull)); + } + if (JSFFICallback* callback = makeCallback({ T::Int32 }, T::Int32, timesThreePlusOne)) { + void* before = callback->nativeEntrypoint(); + callback->close(); + callback->close(); + FFI_CHECK(callback->nativeEntrypoint() == before); + FFI_CHECK(callback->callable() == timesThreePlusOne); + } +} + +static void testFixtureTable() +{ + auto fixtures = ffiTestFixtures(); + FFI_CHECK(fixtures.size() >= 90); + for (const FFIFixtureEntry& entry : fixtures) { + FFI_CHECK(entry.name); + FFI_CHECK(entry.address); + const FFIFixtureEntry* found = ffiTestFixtureNamed(entry.name); + FFI_CHECK(found == &entry); + } + FFI_CHECK(!ffiTestFixtureNamed("ffi_no_such_fixture")); + const FFIFixtureEntry* addI32 = ffiTestFixtureNamed("ffi_add_i32"); + FFI_CHECK(addI32); + if (addI32) + FFI_CHECK(addI32->address == reinterpret_cast(ffi_add_i32)); + + FFI_CHECK_EQ(ffi_widen_char(-1), -1); + FFI_CHECK_EQ(ffi_widen_i8(-1), -1); + FFI_CHECK_EQ(ffi_widen_u8(255), 255); + FFI_CHECK_EQ(ffi_ret_two_as_bool(), 2); + FFI_CHECK(!ffi_ret_null_ptr()); + FFI_CHECK_EQ(reinterpret_cast(ffi_high_ptr()), static_cast(0x00007fffdeadbee0ull)); + FFI_CHECK(std::isnan(ffi_ret_nan_f32())); + FFI_CHECK(std::isnan(ffi_ret_impure_nan_f64())); + FFI_CHECK(std::signbit(ffi_ret_neg_zero_f64())); + FFI_CHECK_EQ(std::bit_cast(ffi_ret_denormal_f32()), 1u); + FFI_CHECK_EQ(ffi_ret_inf_f64(), std::numeric_limits::infinity()); + FFI_CHECK_EQ(ffi_align_probe_0(), 1.0); + FFI_CHECK_EQ(ffi_align_probe_9(1, 2, 3, 4, 5, 6, 7, 8, 9), 1.0); + FFI_CHECK_EQ(ffi_canary_call([] { }), 0); + FFI_CHECK_EQ(ffi_mix_5(1, -1, 3, -4, 5, -6, 7, -8), 1 * 1.0 + 2 * -1.0 + 3 * 3.0 + 4 * -4.0 + 5 * 5.0 + 6 * -6.0 + 7 * 7.0 + 8 * -8.0); +} + +} // anonymous namespace + +static int runAll() +{ + JSC::initialize(); + + if (!Options::useJIT()) { + dataLogLn("testFFI: JIT is disabled; skipping."); + return 0; + } + + s_vm = VM::create(); + { + JSLockHolder locker(*s_vm); + s_globalObject = JSGlobalObject::create(*s_vm, JSGlobalObject::createStructure(*s_vm, jsNull())); + gcProtect(s_globalObject); + + RUN(testTypeTraits()); + RUN(testSignatures()); + RUN(testFixtureTable()); + RUN(testCallLayoutGoldens()); + RUN(testCallLayoutAgainstReferenceModel()); + RUN(testDoubleToInt64()); + RUN(testConversions()); + RUN(testInvokeThunkDifferential()); + RUN(testCanaries()); + RUN(testJSFFIFunctionEndToEnd()); + RUN(testCallbackThunkEndToEnd()); + } + + dataLogLn(s_failureCount ? "FAILED: " : "OK: ", s_checkCount - s_failureCount, " checks passed, ", s_failureCount, " failed."); + VM& leaked = s_vm.releaseNonNull().leakRef(); + UNUSED_PARAM(leaked); + return s_failureCount ? 1 : 0; +} + +#else + +static int runAll() +{ + JSC::initialize(); + dataLogLn("testFFI: bun:ffi is not supported in this configuration (requires USE(BUN_JSC_ADDITIONS), ENABLE(JIT), USE(JSVALUE64), x86-64 or arm64)."); + return 0; +} + +#endif + +int main(int argc, char** argv) +{ +#if USE(BUN_JSC_ADDITIONS) && ENABLE(JIT) && USE(JSVALUE64) && (CPU(X86_64) || CPU(ARM64)) + if (argc == 2) + s_filter = argv[1]; + else if (argc > 2) { + dataLogLn("Usage: testFFI []"); + return 1; + } +#else + UNUSED_PARAM(argc); + UNUSED_PARAM(argv); +#endif + return runAll(); +} + +#if OS(WINDOWS) +extern "C" __declspec(dllexport) int WINAPI dllLauncherEntryPoint(int argc, const char* argv[]) +{ + return main(argc, const_cast(argv)); +} +#endif + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END diff --git a/Source/JavaScriptCore/ftl/FTLCapabilities.cpp b/Source/JavaScriptCore/ftl/FTLCapabilities.cpp index 7c390c902f944..ce610f254853f 100644 --- a/Source/JavaScriptCore/ftl/FTLCapabilities.cpp +++ b/Source/JavaScriptCore/ftl/FTLCapabilities.cpp @@ -211,6 +211,7 @@ inline CapabilityLevel canCompile(DFG::Node* node) case ConstructForwardVarargs: case CallWasm: case TailCallInlinedCallerWasm: + case CallFFI: case CallCustomAccessorGetter: case CallCustomAccessorSetter: case VarargsLength: diff --git a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp index 3866991e70c0c..b6d59a5526019 100644 --- a/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp +++ b/Source/JavaScriptCore/ftl/FTLLowerDFGToB3.cpp @@ -120,6 +120,13 @@ #include "WasmModuleInformation.h" #include "WebAssemblyFunction.h" #include "YarrJITRegisters.h" +#if USE(BUN_JSC_ADDITIONS) +#include "FFIContext.h" +#include "FFIConversions.h" +#include "FFISignature.h" +#include "FFIType.h" +#include "JSFFIFunction.h" +#endif #include #include #include @@ -1565,6 +1572,13 @@ class LowerDFGToB3 { case TailCallInlinedCallerWasm: compileCallWasm(); break; + case CallFFI: +#if USE(BUN_JSC_ADDITIONS) + compileCallFFI(); +#else + DFG_CRASH(m_graph, m_node, "CallFFI is unreachable without USE(BUN_JSC_ADDITIONS)"); +#endif + break; case CallCustomAccessorGetter: compileCallCustomAccessorGetter(); break; @@ -15177,6 +15191,423 @@ IGNORE_CLANG_WARNINGS_END } } +#if USE(BUN_JSC_ADDITIONS) + + template + void compileCallFFIImpl() + { + Node* node = m_node; + JSFFIFunction* ffiFunction = node->ffiFunction(); + JSGlobalObject* globalObject = ffiFunction->globalObject(); + FFI::Signature& signature = node->ffiSignature(); + void* target = ffiFunction->target(); + unsigned nativeArgumentCount = signature.argumentCount(); + + DFG_ASSERT(m_graph, node, node->numChildren() >= 2); + DFG_ASSERT(m_graph, node, node->numChildren() - 2 == nativeArgumentCount, node->numChildren(), nativeArgumentCount); + + CodePtr invokeThunk = signature.invokeThunk(); + if (!invokeThunk) { + m_ftlState.allocationFailed = true; + setJSValue(m_out.constInt64(JSValue::encode(jsUndefined()))); + return; + } + + FFI::g_ffiCompileCounts.ftlCallFFI++; // spec 11.2: proves an FFI call reached the FTL. + + FFI::FFIContext& context = globalObject->ffiContext(); + + LValue slots = m_out.lockedStackSlot(signature.slotBufferBytes()); + auto slotOffset = [](unsigned index) -> ptrdiff_t { + return static_cast(index * FFI::slotSize); + }; + auto slotPointer = [&](unsigned index, ptrdiff_t extraOffset = 0) -> TypedPointer { + return m_out.address(m_heaps.root, slots, slotOffset(index) + extraOffset); + }; + auto slotAddress = [&](unsigned index) -> LValue { + return m_out.addPtr(slots, slotOffset(index)); + }; + + bool needsArena = false; + { + for (unsigned i = 0; i < nativeArgumentCount; ++i) { + FFI::Type type = signature.argumentType(i); + Edge edge = m_graph.varArgChild(node, 2 + i); + if (edge.useKind() != UntypedUse) + continue; + switch (type) { + case FFI::Type::CString: + needsArena = true; + break; + default: + break; + } + } + } + + auto exceptionCheckWithArenaExit = [&](LValue exception) { + if (!exception) + exception = m_out.load64(m_vmValue, m_heaps.VM_exception); +#if ASSERT_ENABLED + LValue vmException = m_out.load64(m_vmValue, m_heaps.VM_exception); + m_out.verify(m_out.equal(exception, vmException)); +#endif + LBasicBlock cleanup = m_out.newBlock(); + LBasicBlock continuation = m_out.newBlock(); + m_out.branch(m_out.notZero64(exception), rarely(cleanup), usually(continuation)); + + LBasicBlock lastNext = m_out.appendTo(cleanup, continuation); + callPreflight(); + m_out.call(Void, m_out.operation(operationFFIArenaExit), weakPointer(globalObject)); + operationExceptionCheck(nullptr); + m_out.jump(continuation); + + m_out.appendTo(continuation, lastNext); + }; + + if (needsArena) + vmCall(Void, operationFFIArenaEnter, weakPointer(globalObject)); + + Vector keepAliveValues; + + constexpr bool directCall = DirectCall; + Vector directOperands; + + for (unsigned i = 0; i < nativeArgumentCount; ++i) { + FFI::Type type = signature.argumentType(i); + TypedPointer slot = slotPointer(i); + LValue directOperand = nullptr; // set by the single-value cases; else reloaded from the slot + + Edge edge = m_graph.varArgChild(node, 2 + i); + switch (edge.useKind()) { + case KnownInt32Use: { + LValue value = lowInt32(edge); + LValue slotValue = nullptr; + switch (type) { + case FFI::Type::Char: + case FFI::Type::Int8: + slotValue = m_out.signExt32To64(m_out.aShr(m_out.shl(value, m_out.constInt32(24)), m_out.constInt32(24))); + break; + case FFI::Type::Uint8: + slotValue = m_out.zeroExt(m_out.bitAnd(value, m_out.constInt32(0xff)), Int64); + break; + case FFI::Type::Int16: + slotValue = m_out.signExt32To64(m_out.aShr(m_out.shl(value, m_out.constInt32(16)), m_out.constInt32(16))); + break; + case FFI::Type::Uint16: + slotValue = m_out.zeroExt(m_out.bitAnd(value, m_out.constInt32(0xffff)), Int64); + break; + case FFI::Type::Int32: + slotValue = m_out.signExt32To64(value); + break; + case FFI::Type::Uint32: + slotValue = m_out.zeroExt(value, Int64); + break; + case FFI::Type::Bool: + slotValue = m_out.zeroExt(m_out.notEqual(value, m_out.int32Zero), Int64); + break; + default: + DFG_CRASH(m_graph, node, "Bad FFI argument type for a KnownInt32Use CallFFI child"); + break; + } + if constexpr (directCall) + directOperand = m_out.castToInt32(slotValue); + else + m_out.store64(slotValue, slot); + break; + } + case KnownBooleanUse: { + DFG_ASSERT(m_graph, node, type == FFI::Type::Bool); + if constexpr (directCall) + directOperand = m_out.zeroExt(lowBoolean(edge), Int32); + else + m_out.store64(m_out.zeroExt(lowBoolean(edge), Int64), slot); + break; + } + case DoubleRepUse: { + LValue value = lowDouble(edge); + if (type == FFI::Type::Float) { + if constexpr (directCall) + directOperand = m_out.doubleToFloat(value); // real float operand -> FPR + else { + m_out.storeFloat(m_out.doubleToFloat(value), slot); + m_out.store32(m_out.int32Zero, slotPointer(i, 4)); + } + } else { + DFG_ASSERT(m_graph, node, type == FFI::Type::Double); + if constexpr (directCall) + directOperand = value; + else + m_out.storeDouble(value, slot); + } + break; + } + case UntypedUse: { + LValue value = lowJSValue(edge); + keepAliveValues.append(value); + LValue typeTag = m_out.constInt32(static_cast(static_cast(type))); + TypedPointer slot = slotPointer(i, 0); + + auto emitSlowConversion = [&] { + if (needsArena) { + callPreflight(); + LValue exception = m_out.call(toOperationType(Void), m_out.operation(operationFFIWriteSlot), weakPointer(globalObject), m_out.constIntPtr(&context), typeTag, value, slotAddress(i)); + exceptionCheckWithArenaExit(exception); + } else + vmCall(Void, operationFFIWriteSlot, weakPointer(globalObject), m_out.constIntPtr(&context), typeTag, value, slotAddress(i)); + }; + + bool numbersInline = type == FFI::Type::Pointer || type == FFI::Type::CString || type == FFI::Type::Function; + bool viewsInline = numbersInline || type == FFI::Type::Buffer; + if (!viewsInline) { + emitSlowConversion(); + break; + } + + LBasicBlock int32Case = m_out.newBlock(); + LBasicBlock notInt32Case = m_out.newBlock(); + LBasicBlock doubleCase = m_out.newBlock(); + LBasicBlock cellCase = m_out.newBlock(); + LBasicBlock viewCase = m_out.newBlock(); + LBasicBlock vectorCase = m_out.newBlock(); + LBasicBlock slowCase = m_out.newBlock(); + LBasicBlock done = m_out.newBlock(); + + if (numbersInline) + m_out.branch(isInt32(value, provenType(edge)), unsure(int32Case), unsure(notInt32Case)); + else + m_out.jump(cellCase); // buffer: only views convert inline; numbers throw in C++. + + LBasicBlock lastNext = m_out.appendTo(int32Case, notInt32Case); + m_out.store64(m_out.signExt32To64(unboxInt32(value)), slot); + m_out.jump(done); + + m_out.appendTo(notInt32Case, doubleCase); + m_out.branch(isNumber(value, provenType(edge)), unsure(doubleCase), unsure(cellCase)); + + m_out.appendTo(doubleCase, cellCase); + m_out.store64(m_out.doubleToInt64(unboxDouble(value)), slot); + m_out.jump(done); + + m_out.appendTo(cellCase, viewCase); + if (!numbersInline) + m_out.branch(isNotCell(value, provenType(edge)), rarely(slowCase), usually(viewCase)); + else + m_out.branch(isNotCell(value, provenType(edge)), unsure(slowCase), unsure(viewCase)); + + m_out.appendTo(viewCase, vectorCase); + LValue jsType = m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoType); + LValue isView = m_out.belowOrEqual( + m_out.sub(jsType, m_out.constInt32(FirstTypedArrayType)), + m_out.constInt32(LastTypedArrayType - FirstTypedArrayType)); + LValue mode = m_out.load8ZeroExt32(value, m_heaps.JSArrayBufferView_mode); + LValue isPlainMode = m_out.isZero32(m_out.bitAnd(mode, m_out.constInt32(isResizableOrGrowableSharedMode))); + m_out.branch(m_out.bitAnd(isView, isPlainMode), usually(vectorCase), rarely(slowCase)); + + m_out.appendTo(vectorCase, slowCase); + LValue vector = m_out.loadPtr(value, m_heaps.JSArrayBufferView_vector); + LValue storage = caged(Gigacage::Primitive, vector, value); + LBasicBlock storeVector = m_out.newBlock(); + m_out.branch(m_out.isNull(vector), rarely(slowCase), usually(storeVector)); + m_out.appendTo(storeVector, slowCase); + m_out.store64(storage, slot); + m_out.jump(done); + + m_out.appendTo(slowCase, done); + emitSlowConversion(); + m_out.jump(done); + + m_out.appendTo(done, lastNext); + break; + } + default: + DFG_CRASH(m_graph, node, "Bad use kind for a CallFFI argument"); + break; + } + if constexpr (directCall) { + if (directOperand) + directOperands.append(directOperand); + else if (type == FFI::Type::Double) + directOperands.append(m_out.loadDouble(slotPointer(i))); + else if (type == FFI::Type::Float) + directOperands.append(m_out.loadFloat(slotPointer(i))); + else if (FFI::nativeSizeInBytes(type) <= 4) + directOperands.append(m_out.castToInt32(m_out.load64(slotPointer(i)))); + else + directOperands.append(m_out.load64(slotPointer(i))); + } + } + + FFI::Type returnType = signature.returnType(); + LValue targetValue = m_out.constIntPtr(target); + if constexpr (directCall) { + callPreflight(); + m_out.storePtr(m_callFrame, m_out.absolute(&vm().topCallFrame)); + LValue callee = m_out.constIntPtr(tagCFunctionPtr(target)); + LType returnLType = Void; + switch (returnType) { + case FFI::Type::Void: returnLType = Void; break; + case FFI::Type::Float: returnLType = Float; break; + case FFI::Type::Double: returnLType = Double; break; + case FFI::Type::Char: + case FFI::Type::Int8: + case FFI::Type::Uint8: + case FFI::Type::Int16: + case FFI::Type::Uint16: + case FFI::Type::Int32: + case FFI::Type::Uint32: + case FFI::Type::Bool: + returnLType = Int32; + break; + default: + returnLType = Int64; + break; + } + LValue rawReturn = m_out.call(returnLType, callee, directOperands); + TypedPointer returnSlot = slotPointer(nativeArgumentCount); + switch (returnType) { + case FFI::Type::Void: + break; + case FFI::Type::Char: case FFI::Type::Int8: + m_out.store64(m_out.signExt32To64(m_out.aShr(m_out.shl(rawReturn, m_out.constInt32(24)), m_out.constInt32(24))), returnSlot); + break; + case FFI::Type::Uint8: + m_out.store64(m_out.zeroExt(m_out.bitAnd(rawReturn, m_out.constInt32(0xff)), Int64), returnSlot); + break; + case FFI::Type::Int16: + m_out.store64(m_out.signExt32To64(m_out.aShr(m_out.shl(rawReturn, m_out.constInt32(16)), m_out.constInt32(16))), returnSlot); + break; + case FFI::Type::Uint16: + m_out.store64(m_out.zeroExt(m_out.bitAnd(rawReturn, m_out.constInt32(0xffff)), Int64), returnSlot); + break; + case FFI::Type::Int32: + m_out.store64(m_out.signExt32To64(rawReturn), returnSlot); + break; + case FFI::Type::Uint32: + m_out.store64(m_out.zeroExt(rawReturn, Int64), returnSlot); + break; + case FFI::Type::Bool: + m_out.store64(m_out.zeroExt(m_out.notEqual(m_out.bitAnd(rawReturn, m_out.constInt32(0xff)), m_out.int32Zero), Int64), returnSlot); + break; + case FFI::Type::Float: + m_out.storeFloat(rawReturn, returnSlot); + m_out.store32(m_out.int32Zero, slotPointer(nativeArgumentCount, 4)); + break; + case FFI::Type::Double: + m_out.storeDouble(rawReturn, returnSlot); + break; + default: // 64-bit integers, pointer family, jsvalue: raw 64 bits are the encoding. + m_out.store64(rawReturn, returnSlot); + break; + } + if (needsArena) + exceptionCheckWithArenaExit(nullptr); + else + operationExceptionCheck(nullptr); + } else if (needsArena) { + callPreflight(); + m_out.storePtr(m_callFrame, m_out.absolute(&vm().topCallFrame)); + m_out.call(Void, m_out.constIntPtr(invokeThunk.taggedPtr()), targetValue, slots); + exceptionCheckWithArenaExit(nullptr); + } else { + callPreflight(); + m_out.storePtr(m_callFrame, m_out.absolute(&vm().topCallFrame)); + m_out.call(Void, m_out.constIntPtr(invokeThunk.taggedPtr()), targetValue, slots); + operationExceptionCheck(nullptr); + } + + if (!keepAliveValues.isEmpty()) + ensureStillAliveHere(keepAliveValues); + + TypedPointer returnSlot = slotPointer(nativeArgumentCount); + switch (returnType) { + case FFI::Type::Void: + setJSValue(m_out.constInt64(JSValue::encode(jsUndefined()))); + break; + case FFI::Type::Char: + case FFI::Type::Int8: + case FFI::Type::Uint8: + case FFI::Type::Int16: + case FFI::Type::Uint16: + case FFI::Type::Int32: + setJSValue(boxInt32(m_out.load32(returnSlot))); + break; + case FFI::Type::Uint32: + setJSValue(strictInt52ToJSValue(m_out.load64(returnSlot))); + break; + case FFI::Type::Bool: + setJSValue(boxBoolean(m_out.load32(returnSlot))); + break; + case FFI::Type::Double: + setJSValue(boxDouble(m_out.purifyNaN(m_out.loadDouble(returnSlot)))); + break; + case FFI::Type::Float: + setJSValue(boxDouble(m_out.purifyNaN(m_out.floatToDouble(m_out.loadFloat(returnSlot))))); + break; + case FFI::Type::JSValue: + setJSValue(m_out.load64(returnSlot)); + break; + case FFI::Type::Int64: + case FFI::Type::Uint64: + case FFI::Type::Int64Fast: + case FFI::Type::Uint64Fast: + case FFI::Type::Pointer: + case FFI::Type::CString: + case FFI::Type::Function: + case FFI::Type::Buffer: { + LValue slotValue = m_out.load64(returnSlot); + LValue typeTag = m_out.constInt32(static_cast(static_cast(returnType))); + LValue boxed; + if (needsArena) { + callPreflight(); + LValue result = m_out.call(toOperationType(Int64), m_out.operation(operationFFIBoxSlot), weakPointer(globalObject), typeTag, slotValue, m_out.constInt32(0)); + boxed = m_out.extract(result, 0); + exceptionCheckWithArenaExit(m_out.extract(result, 1)); + } else + boxed = vmCall(Int64, operationFFIBoxSlot, weakPointer(globalObject), typeTag, slotValue, m_out.constInt32(0)); + setJSValue(boxed); + break; + } + case FFI::Type::RESERVED_WasNapiEnv: + case FFI::Type::BufferLength: + DFG_CRASH(m_graph, node, "the reserved tag / buffer_length is never a valid FFI return type"); + break; + } + + if (needsArena) + vmCall(Void, operationFFIArenaExit, weakPointer(globalObject)); + } + + void compileCallFFI() + { + bool directCall = Options::useFFIDirectCall(); +#if OS(WINDOWS) + directCall = false; +#elif CPU(ARM64) && OS(DARWIN) + if (directCall) { + FFI::Signature& signature = m_node->ffiFunction()->signature(); + unsigned gprCount = 0; + for (unsigned i = 0; i < signature.argumentCount(); ++i) { + FFI::Type t = signature.argumentType(i); + if (t == FFI::Type::Float || t == FFI::Type::Double) + continue; + bool subWord = t == FFI::Type::Char || t == FFI::Type::Int8 || t == FFI::Type::Uint8 + || t == FFI::Type::Int16 || t == FFI::Type::Uint16 || t == FFI::Type::Bool; + if (subWord && gprCount >= GPRInfo::numberOfArgumentRegisters) { + directCall = false; + break; + } + ++gprCount; + } + } +#endif + if (directCall) + compileCallFFIImpl(); + else + compileCallFFIImpl(); + } +#endif // USE(BUN_JSC_ADDITIONS) + void compileCallCustomAccessorGetter() { // The following function is not an operation: we directly call a custom accessor getter. diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index decf60d0df217..431c12824bcfc 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -117,6 +117,8 @@ #include "InternalFieldTuple.h" #if USE(BUN_JSC_ADDITIONS) +#include "JSFFICallback.h" +#include "JSFFIFunction.h" #include "JSString.h" #include #endif @@ -415,6 +417,10 @@ Heap::Heap(VM& vm, HeapType heapType) , intlSegmentIteratorHeapCellType(IsoHeapCellType::Args()) , intlSegmenterHeapCellType(IsoHeapCellType::Args()) , intlSegmentsHeapCellType(IsoHeapCellType::Args()) +#if USE(BUN_JSC_ADDITIONS) + , ffiFunctionHeapCellType(IsoHeapCellType::Args()) + , ffiCallbackHeapCellType(IsoHeapCellType::Args()) +#endif #if ENABLE(WEBASSEMBLY) , webAssemblyExceptionHeapCellType(IsoHeapCellType::Args()) , webAssemblyFunctionHeapCellType(IsoHeapCellType::Args()) diff --git a/Source/JavaScriptCore/heap/Heap.h b/Source/JavaScriptCore/heap/Heap.h index ec349ffe0c17f..4e52e1167970e 100644 --- a/Source/JavaScriptCore/heap/Heap.h +++ b/Source/JavaScriptCore/heap/Heap.h @@ -212,9 +212,18 @@ class Heap; #define FOR_EACH_JSC_WEBASSEMBLY_DYNAMIC_NON_ISO_SUBSPACE(v) #endif +#if USE(BUN_JSC_ADDITIONS) +#define FOR_EACH_JSC_FFI_DYNAMIC_ISO_SUBSPACE(v) \ + v(ffiFunctionSpace, ffiFunctionHeapCellType, JSFFIFunction) \ + v(ffiCallbackSpace, ffiCallbackHeapCellType, JSFFICallback) +#else +#define FOR_EACH_JSC_FFI_DYNAMIC_ISO_SUBSPACE(v) +#endif + #define FOR_EACH_JSC_DYNAMIC_ISO_SUBSPACE(v) \ FOR_EACH_JSC_OBJC_API_DYNAMIC_ISO_SUBSPACE(v) \ FOR_EACH_JSC_GLIB_API_DYNAMIC_ISO_SUBSPACE(v) \ + FOR_EACH_JSC_FFI_DYNAMIC_ISO_SUBSPACE(v) \ \ v(apiGlobalObjectSpace, apiGlobalObjectHeapCellType, JSAPIGlobalObject) \ v(apiValueWrapperSpace, cellHeapCellType, JSAPIValueWrapper) \ @@ -1101,6 +1110,10 @@ class Heap { IsoHeapCellType intlSegmentIteratorHeapCellType; IsoHeapCellType intlSegmenterHeapCellType; IsoHeapCellType intlSegmentsHeapCellType; +#if USE(BUN_JSC_ADDITIONS) + IsoHeapCellType ffiFunctionHeapCellType; + IsoHeapCellType ffiCallbackHeapCellType; +#endif #if ENABLE(WEBASSEMBLY) IsoHeapCellType webAssemblyExceptionHeapCellType; IsoHeapCellType webAssemblyFunctionHeapCellType; diff --git a/Source/JavaScriptCore/runtime/JSGlobalObject.cpp b/Source/JavaScriptCore/runtime/JSGlobalObject.cpp index e5970de99501c..e24afa8f374a4 100644 --- a/Source/JavaScriptCore/runtime/JSGlobalObject.cpp +++ b/Source/JavaScriptCore/runtime/JSGlobalObject.cpp @@ -35,6 +35,11 @@ #include "AggregateError.h" #include "SuppressedError.h" #include "InternalFieldTuple.h" +#if USE(BUN_JSC_ADDITIONS) +#include "FFIContext.h" +#include "JSFFICallback.h" +#include "JSFFIFunction.h" +#endif #include "AggregateErrorConstructorInlines.h" #include "SuppressedErrorConstructorInlines.h" #include "AggregateErrorPrototypeInlines.h" @@ -2217,6 +2222,15 @@ capitalName ## Constructor* lowerName ## Constructor = featureFlag ? capitalName vm, vm.propertyNames->builtinNames().asyncContextPrivateName(), asyncContext, PropertyAttribute::DontEnum | PropertyAttribute::DontDelete | PropertyAttribute::ReadOnly); m_asyncContextData.set(vm, this, asyncContext); + + m_ffiFunctionStructure.initLater( + [] (const Initializer& init) { + init.set(JSFFIFunction::createStructure(init.vm, init.owner, init.owner->m_functionPrototype.get())); + }); + m_ffiCallbackStructure.initLater( + [] (const Initializer& init) { + init.set(JSFFICallback::createStructure(init.vm, init.owner, JSFFICallback::createPrototype(init.vm, init.owner))); + }); #endif m_performProxyObjectHasFunction.set(vm, this, uncheckedDowncast(linkTimeConstant(LinkTimeConstant::performProxyObjectHas))); @@ -2983,6 +2997,10 @@ void JSGlobalObject::visitChildrenImpl(JSCell* cell, Visitor& visitor) #if USE(BUN_JSC_ADDITIONS) visitor.append(thisObject->m_asyncContextData); visitor.append(thisObject->m_internalFieldTupleStructure); + thisObject->m_ffiFunctionStructure.visit(visitor); + thisObject->m_ffiCallbackStructure.visit(visitor); + if (thisObject->m_ffiContext) + thisObject->m_ffiContext->visitLiveCallbacks(*thisObject, visitor); #endif visitor.append(thisObject->m_globalLexicalEnvironment); @@ -3786,8 +3804,18 @@ void JSGlobalObject::queueMicrotask(VM& vm, InternalMicrotask job, uint8_t paylo { queueMicrotask(vm, QueuedTask { nullptr, job, payload, this, argument0, argument1, argument2, argument3 }); } -#endif +FFI::FFIContext& JSGlobalObject::ffiContext() +{ + if (!m_ffiContext) [[unlikely]] { + ASSERT(!isCompilationThread()); + auto context = makeUnique(vm()); + WTF::storeStoreFence(); + m_ffiContext = WTF::move(context); + } + return *m_ffiContext; +} +#endif void JSGlobalObject::setMicrotaskQueue(Ref&& queue) { diff --git a/Source/JavaScriptCore/runtime/JSGlobalObject.h b/Source/JavaScriptCore/runtime/JSGlobalObject.h index 1c095af20a46c..72c06c159d22b 100644 --- a/Source/JavaScriptCore/runtime/JSGlobalObject.h +++ b/Source/JavaScriptCore/runtime/JSGlobalObject.h @@ -131,6 +131,12 @@ class SymbolTable; class WrapperMap; class WrapForValidIteratorPrototype; +#if USE(BUN_JSC_ADDITIONS) +namespace FFI { +class FFIContext; +} // namespace FFI +#endif + enum class ArrayBufferSharingMode : bool; enum class CodeGenerationMode : uint8_t; enum class ErrorType : uint8_t; @@ -382,6 +388,8 @@ class JSGlobalObject : public JSSegmentedVariableObject { #if USE(BUN_JSC_ADDITIONS) WriteBarrierStructureID m_internalFieldTupleStructure; + LazyProperty m_ffiFunctionStructure; + LazyProperty m_ffiCallbackStructure; #endif // Lists the actual structures used for having these particular indexing shapes. @@ -505,6 +513,7 @@ class JSGlobalObject : public JSSegmentedVariableObject { #if USE(BUN_JSC_ADDITIONS) bool m_isAsyncContextTrackingEnabled { false }; WriteBarrier m_asyncContextData; + std::unique_ptr m_ffiContext; #endif #if ENABLE(REMOTE_INSPECTOR) @@ -1099,6 +1108,9 @@ class JSGlobalObject : public JSSegmentedVariableObject { #if USE(BUN_JSC_ADDITIONS) Structure* internalFieldTupleStructure() const { return m_internalFieldTupleStructure.get(); } + Structure* ffiFunctionStructure() const { return m_ffiFunctionStructure.get(this); } + Structure* ffiCallbackStructure() const { return m_ffiCallbackStructure.get(this); } + JS_EXPORT_PRIVATE FFI::FFIContext& ffiContext(); #endif JS_EXPORT_PRIVATE void setInspectable(bool); diff --git a/Source/JavaScriptCore/runtime/OptionsList.h b/Source/JavaScriptCore/runtime/OptionsList.h index d872e04613335..094edc1b27e3d 100644 --- a/Source/JavaScriptCore/runtime/OptionsList.h +++ b/Source/JavaScriptCore/runtime/OptionsList.h @@ -77,6 +77,17 @@ bool hasCapacityToUseLargeGigacage(); // On instantiation of the first VM instance, the Options will be write protected // and cannot be modified thereafter. +#if USE(BUN_JSC_ADDITIONS) +#define FOR_EACH_JSC_FFI_OPTION(v) \ + v(Bool, useFFIICStub, true, Normal, "install per-function FFI IC stubs"_s) \ + v(Bool, useFFICallInDFG, true, Normal, "allow Call -> CallFFI in DFG/FTL"_s) \ + v(Bool, useFFIDirectCall, true, Normal, "FTL calls the native FFI target directly (no invoke thunk)"_s) \ + v(Bool, dumpFFIDisassembly, false, Normal, "disassemble generated FFI thunks/stubs"_s) \ + v(Bool, verboseFFI, false, Normal, "dataLog on FFI thunk/stub/signature creation"_s) +#else +#define FOR_EACH_JSC_FFI_OPTION(v) +#endif + #define FOR_EACH_JSC_OPTION(v) \ v(Bool, useKernTCSM, defaultTCSMValue(), Normal, "Note: this needs to go before other options since they depend on this value."_s) \ v(Bool, validateOptions, false, Normal, "crashes if mis-typed JSC options were passed to the VM"_s) \ @@ -95,6 +106,7 @@ bool hasCapacityToUseLargeGigacage(); v(Bool, useV8DateParser, false, Normal, nullptr) \ v(Bool, showPrivateScriptsInStackTraces, false, Normal, "Show private scripts in stack traces."_s) \ v(Bool, evalMode, false, Normal, "Set to true for less aggressive function call completion value discarding."_s) \ + FOR_EACH_JSC_FFI_OPTION(v) \ \ v(Unsigned, maxPerThreadStackUsage, 5 * MB, Normal, "Max allowed stack usage by the VM"_s) \ v(Unsigned, softReservedZoneSize, 128 * KB, Normal, "A buffer greater than reservedZoneSize that reserves space for stringifying exceptions."_s) \ diff --git a/Source/JavaScriptCore/runtime/VM.h b/Source/JavaScriptCore/runtime/VM.h index e57882627c62c..4ba6229bb3def 100644 --- a/Source/JavaScriptCore/runtime/VM.h +++ b/Source/JavaScriptCore/runtime/VM.h @@ -140,6 +140,7 @@ class NativeExecutable; #if USE(BUN_JSC_ADDITIONS) class QueuedTask; enum class InternalMicrotask : uint8_t; +namespace FFI { class CallbackEntryScope; } #endif class Debugger; class DeferredWorkTimer; @@ -1368,6 +1369,9 @@ class VM : public ThreadSafeRefCountedWithSuppressingSaferCPPChecking { friend class JSDollarVMHelper; friend class LLIntOffsetsExtractor; friend class SuspendExceptionScope; +#if USE(BUN_JSC_ADDITIONS) + friend class FFI::CallbackEntryScope; +#endif friend class VMTraps; }; diff --git a/Source/JavaScriptCore/shell/CMakeLists.txt b/Source/JavaScriptCore/shell/CMakeLists.txt index 2a3970cd0d66f..2c8ec3d92fd64 100644 --- a/Source/JavaScriptCore/shell/CMakeLists.txt +++ b/Source/JavaScriptCore/shell/CMakeLists.txt @@ -31,6 +31,38 @@ endif () WEBKIT_EXECUTABLE_DECLARE(jsc) +if (USE_BUN_JSC_ADDITIONS) + # bun:ffi (JSC FFI) C++ unit / ABI test executable, modeled on the + # testmasm_* variables below. Build and run it with: + # ninja testFFI && ./bin/testFFI + # Unlike testmasm this is NOT additionally gated on DEVELOPER_MODE: the + # standard `bun build.ts debug` configuration never enables DEVELOPER_MODE, + # and testFFI must build from that same build directory. build.ts and the + # release scripts build it alongside jsc and ship it in the artifact's bin/. + # The native test fixtures (ffi/tests/FFITestFixtures.cpp) are NOT compiled + # here: they live in libJavaScriptCore itself (Sources.txt) so that + # $vm.ffiFixture in tools/JSDollarVM.cpp can link, and testFFI reaches them + # through the framework it links. + # FFI-SPEC-GAP: the target is gated on USE_BUN_JSC_ADDITIONS because + # testFFI.cpp compiles to nothing (no main) without the feature. + set(testFFI_SOURCES ../ffi/tests/testFFI.cpp) + set(testFFI_DEFINITIONS ${jsc_PRIVATE_DEFINITIONS}) + set(testFFI_PRIVATE_INCLUDE_DIRECTORIES ${jsc_PRIVATE_INCLUDE_DIRECTORIES}) + set(testFFI_FRAMEWORKS ${jsc_FRAMEWORKS}) + set(testFFI_LIBRARIES ${CMAKE_DL_LIBS}) + if (WIN32) + list(APPEND testFFI_LIBRARIES Winmm) + endif () + if (USE_EXTERNAL_MIMALLOC) + # Same as jsc above: the shipped static archives leave mi_* unresolved + # under USE_EXTERNAL_MIMALLOC, so any executable that links the + # framework must supply the vendored mimalloc objects itself. + list(APPEND testFFI_LIBRARIES $) + endif () + + WEBKIT_EXECUTABLE_DECLARE(testFFI) +endif () + if (DEVELOPER_MODE) set(testapi_SOURCES ../API/tests/CompareAndSwapTest.cpp @@ -60,6 +92,8 @@ if (DEVELOPER_MODE) set(testmasm_PRIVATE_INCLUDE_DIRECTORIES ${jsc_PRIVATE_INCLUDE_DIRECTORIES}) set(testmasm_FRAMEWORKS ${jsc_FRAMEWORKS}) + # (testFFI is declared above, outside DEVELOPER_MODE; see the comment there.) + set(testRegExp_SOURCES ../testRegExp.cpp) set(testRegExp_DEFINITIONS ${jsc_PRIVATE_DEFINITIONS}) set(testRegExp_PRIVATE_INCLUDE_DIRECTORIES ${jsc_PRIVATE_INCLUDE_DIRECTORIES}) @@ -133,6 +167,10 @@ WEBKIT_INCLUDE_CONFIG_FILES_IF_EXISTS() WEBKIT_EXECUTABLE(jsc) set_target_properties(jsc PROPERTIES POSITION_INDEPENDENT_CODE ON) +if (USE_BUN_JSC_ADDITIONS) + WEBKIT_EXECUTABLE(testFFI) +endif () + if (SHOULD_INSTALL_JS_SHELL) install(TARGETS jsc DESTINATION "${LIBEXEC_INSTALL_DIR}") endif () diff --git a/Source/JavaScriptCore/tools/JSDollarVM.cpp b/Source/JavaScriptCore/tools/JSDollarVM.cpp index 21779cdd7bddc..65827df6c57cc 100644 --- a/Source/JavaScriptCore/tools/JSDollarVM.cpp +++ b/Source/JavaScriptCore/tools/JSDollarVM.cpp @@ -67,6 +67,7 @@ #include "SnippetParams.h" #include "Strong.h" #include "StructureCreateInlines.h" +#include "TopExceptionScope.h" #include "TypeProfiler.h" #include "TypeProfilerLog.h" #include "VMEntryScopeInlines.h" @@ -107,6 +108,22 @@ WTF_ALLOW_UNSAFE_BUFFER_USAGE_END #include "WasmDebugServer.h" #endif +#if USE(BUN_JSC_ADDITIONS) && USE(JSVALUE64) +#include "BunFFI.h" +#include "FFIContext.h" +#include "FFIConversions.h" +#include "FFISignature.h" +#include "FFITestFixtures.h" +#include "FFIType.h" +#include "JSFFICallback.h" +#include "JSFFIFunction.h" +#include "ObjectConstructor.h" +#include +#include +#include +#include +#endif + #if PLATFORM(COCOA) #include #endif @@ -2265,6 +2282,18 @@ static JSC_DECLARE_HOST_FUNCTION(functionCallFromCPP); static JSC_DECLARE_HOST_FUNCTION(functionCachedCallFromCPP); static JSC_DECLARE_HOST_FUNCTION(functionDumpLineBreakData); static JSC_DECLARE_HOST_FUNCTION(functionWeakCreate); +#if USE(BUN_JSC_ADDITIONS) && USE(JSVALUE64) +static JSC_DECLARE_HOST_FUNCTION(functionFFIFunction); +static JSC_DECLARE_HOST_FUNCTION(functionFFICallback); +static JSC_DECLARE_HOST_FUNCTION(functionFFIFixture); +static JSC_DECLARE_HOST_FUNCTION(functionFFIFixtures); +static JSC_DECLARE_HOST_FUNCTION(functionFFISignatureString); +static JSC_DECLARE_HOST_FUNCTION(functionFFIRead); +static JSC_DECLARE_HOST_FUNCTION(functionFFIWrite); +static JSC_DECLARE_HOST_FUNCTION(functionFFICString); +static JSC_DECLARE_HOST_FUNCTION(functionFFIArenaDepth); +static JSC_DECLARE_HOST_FUNCTION(functionFFICompileCounts); +#endif const ClassInfo JSDollarVM::s_info = { "DollarVM"_s, &Base::s_info, nullptr, nullptr, CREATE_METHOD_TABLE(JSDollarVM) }; @@ -4495,6 +4524,419 @@ JSC_DEFINE_HOST_FUNCTION(functionWeakCreate, (JSGlobalObject* globalObject, Call return JSValue::encode(jsUndefined()); } +#if USE(BUN_JSC_ADDITIONS) && USE(JSVALUE64) + +static bool dollarVMFFIJITIsUnavailable() +{ + return !Options::useJIT() || !VM::canUseAssembler(); +} + +static std::optional dollarVMParseFFIType(JSGlobalObject* globalObject, JSValue value) +{ + return FFI::typeFromJS(globalObject, value); +} + +static void* dollarVMFFIPointerFromJS(JSGlobalObject* globalObject, JSValue value) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (auto* callback = dynamicDowncast(value); callback && callback->isClosed()) { + throwTypeError(globalObject, scope, "bun:ffi: the JSFFICallback has been closed"_s); + return nullptr; + } + + uint64_t slot = 0; + FFI::writeSlotFromJSValue(globalObject, globalObject->ffiContext(), FFI::Type::Pointer, value, slot, nullptr); + RETURN_IF_EXCEPTION(scope, nullptr); + return reinterpret_cast(static_cast(slot)); +} + +static bool dollarVMFFIIsRawMemoryType(FFI::Type type) +{ + switch (type) { + case FFI::Type::Char: + case FFI::Type::Int8: + case FFI::Type::Uint8: + case FFI::Type::Int16: + case FFI::Type::Uint16: + case FFI::Type::Int32: + case FFI::Type::Uint32: + case FFI::Type::Int64: + case FFI::Type::Uint64: + case FFI::Type::Double: + case FFI::Type::Float: + case FFI::Type::Bool: + case FFI::Type::Pointer: + case FFI::Type::Int64Fast: + case FFI::Type::Uint64Fast: + return true; + case FFI::Type::Void: + case FFI::Type::CString: + case FFI::Type::Function: + case FFI::Type::RESERVED_WasNapiEnv: + case FFI::Type::JSValue: + case FFI::Type::Buffer: + case FFI::Type::BufferLength: + return false; + } + return false; +} + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN + +static uint64_t dollarVMFFILoadSlot(FFI::Type type, const void* address) +{ + switch (type) { + case FFI::Type::Char: + case FFI::Type::Int8: { + int8_t value; + memcpy(&value, address, sizeof(value)); + return static_cast(static_cast(value)); + } + case FFI::Type::Uint8: { + uint8_t value; + memcpy(&value, address, sizeof(value)); + return value; + } + case FFI::Type::Bool: { + uint8_t value; + memcpy(&value, address, sizeof(value)); + return value ? 1 : 0; + } + case FFI::Type::Int16: { + int16_t value; + memcpy(&value, address, sizeof(value)); + return static_cast(static_cast(value)); + } + case FFI::Type::Uint16: { + uint16_t value; + memcpy(&value, address, sizeof(value)); + return value; + } + case FFI::Type::Int32: { + int32_t value; + memcpy(&value, address, sizeof(value)); + return static_cast(static_cast(value)); + } + case FFI::Type::Uint32: + case FFI::Type::Float: { + uint32_t value; + memcpy(&value, address, sizeof(value)); + return value; + } + case FFI::Type::Int64: + case FFI::Type::Uint64: + case FFI::Type::Int64Fast: + case FFI::Type::Uint64Fast: + case FFI::Type::Double: + case FFI::Type::Pointer: { + uint64_t value; + memcpy(&value, address, sizeof(value)); + return value; + } + case FFI::Type::Void: + case FFI::Type::CString: + case FFI::Type::Function: + case FFI::Type::RESERVED_WasNapiEnv: + case FFI::Type::JSValue: + case FFI::Type::Buffer: + case FFI::Type::BufferLength: + break; + } + RELEASE_ASSERT_NOT_REACHED(); + return 0; +} + +static void dollarVMFFIStoreSlot(FFI::Type type, uint64_t slot, void* address) +{ + static_assert(std::endian::native == std::endian::little, "bun:ffi $vm raw memory helpers assume little-endian"); + memcpy(address, &slot, FFI::nativeSizeInBytes(type)); +} + +WTF_ALLOW_UNSAFE_BUFFER_USAGE_END + +static void* dollarVMTestHookBefore(JSGlobalObject* globalObject, CallFrame* callFrame) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + static uintptr_t tokenCounter = 0; + uintptr_t token = ++tokenCounter; + if (JSObject* owner = uncheckedDowncast(callFrame->jsCallee())->owner()) { + JSValue logValue = owner->get(globalObject, Identifier::fromString(vm, "hookLog"_s)); + RETURN_IF_EXCEPTION(scope, nullptr); + if (auto* log = dynamicDowncast(logValue)) { + log->push(globalObject, jsString(vm, makeString("before:"_s, token))); + RETURN_IF_EXCEPTION(scope, nullptr); + } + } + return reinterpret_cast(token); +} +static void dollarVMTestHookAfter(JSGlobalObject* globalObject, CallFrame* callFrame, void* token) +{ + VM& vm = globalObject->vm(); + auto scope = DECLARE_TOP_EXCEPTION_SCOPE(vm); + if (JSObject* owner = uncheckedDowncast(callFrame->jsCallee())->owner()) { + JSValue logValue = owner->get(globalObject, Identifier::fromString(vm, "hookLog"_s)); + if (scope.exception()) { + scope.clearException(); + return; + } + if (auto* log = dynamicDowncast(logValue)) { + log->push(globalObject, jsString(vm, makeString("after:"_s, reinterpret_cast(token)))); + if (scope.exception()) + scope.clearException(); + } + } +} +static const FFI::CallHooks dollarVMTestHooks { dollarVMTestHookBefore, dollarVMTestHookAfter }; + +JSC_DEFINE_HOST_FUNCTION(functionFFIFunction, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (dollarVMFFIJITIsUnavailable()) + return throwVMTypeError(globalObject, scope, "bun:ffi requires the JIT"_s); + + RefPtr signature = FFI::signatureFromJS(globalObject, callFrame->argument(0)); + RETURN_IF_EXCEPTION(scope, { }); + RELEASE_ASSERT(signature); // FFI::signatureFromJS throws on every failure. + + void* target = dollarVMFFIPointerFromJS(globalObject, callFrame->argument(1)); + RETURN_IF_EXCEPTION(scope, { }); + if (!target) + return throwVMTypeError(globalObject, scope, "$vm.ffiFunction: null pointer"_s); + + String name; + JSValue nameValue = callFrame->argument(2); + if (nameValue.isUndefinedOrNull()) + name = signature->toString(); + else { + name = nameValue.toWTFString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + } + + JSObject* owner = nullptr; + const FFI::CallHooks* hooks = nullptr; + if (JSObject* options = callFrame->argument(3).getObject()) { + JSValue ownerValue = options->get(globalObject, Identifier::fromString(vm, "owner"_s)); + RETURN_IF_EXCEPTION(scope, { }); + if (!ownerValue.isUndefinedOrNull()) { + owner = ownerValue.getObject(); + if (!owner) + return throwVMTypeError(globalObject, scope, "$vm.ffiFunction: owner must be an object"_s); + } + JSValue hooksValue = options->get(globalObject, Identifier::fromString(vm, "hooks"_s)); + RETURN_IF_EXCEPTION(scope, { }); + if (!hooksValue.isUndefinedOrNull()) { + String kind = hooksValue.toWTFString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + if (kind != "test"_s) + return throwVMTypeError(globalObject, scope, "$vm.ffiFunction: hooks must be \"test\""_s); + hooks = &dollarVMTestHooks; + } + } + + RELEASE_AND_RETURN(scope, JSValue::encode(JSFFIFunction::create(vm, globalObject, globalObject->ffiFunctionStructure(), signature.releaseNonNull(), target, name, owner, hooks))); +} + +static void dollarVMThreadsafeDispatch(FFI::ThreadsafeInvocation&); // defined below with the queue/drain model +JSC_DEFINE_HOST_FUNCTION(functionFFICallback, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + if (dollarVMFFIJITIsUnavailable()) + return throwVMTypeError(globalObject, scope, "bun:ffi requires the JIT"_s); + + RefPtr signature = FFI::signatureFromJS(globalObject, callFrame->argument(0)); + RETURN_IF_EXCEPTION(scope, { }); + RELEASE_ASSERT(signature); // FFI::signatureFromJS throws on every failure. + + JSValue callableValue = callFrame->argument(1); + if (!callableValue.isCallable()) + return throwVMTypeError(globalObject, scope, "$vm.ffiCallback: expected a callable"_s); + + bool threadsafe = false; + if (JSObject* options = callFrame->argument(2).getObject()) { + JSValue threadsafeValue = options->get(globalObject, Identifier::fromString(vm, "threadsafe"_s)); + RETURN_IF_EXCEPTION(scope, { }); + threadsafe = threadsafeValue.toBoolean(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + } + if (threadsafe && !FFI::FFIContext::threadsafeDispatch()) + FFI::FFIContext::setThreadsafeDispatch(dollarVMThreadsafeDispatch); + + RELEASE_AND_RETURN(scope, JSValue::encode(JSFFICallback::create(vm, globalObject, globalObject->ffiCallbackStructure(), asObject(callableValue), signature.releaseNonNull(), threadsafe, nullptr))); +} + +static Lock s_threadsafeQueueLock; +static Vector>& threadsafeQueue() +{ + static NeverDestroyed>> queue; + return queue.get(); +} +static void dollarVMThreadsafeDispatch(FFI::ThreadsafeInvocation& invocation) +{ + Locker locker { s_threadsafeQueueLock }; + threadsafeQueue().append(&invocation); +} +static JSC_DECLARE_HOST_FUNCTION(functionDrainThreadsafeCallbacks); +JSC_DEFINE_HOST_FUNCTION(functionDrainThreadsafeCallbacks, (JSGlobalObject* globalObject, CallFrame*)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + Vector> pending; + { + Locker locker { s_threadsafeQueueLock }; + pending = std::exchange(threadsafeQueue(), { }); + } + unsigned index = 0; + for (; index < pending.size(); ++index) { + FFI::runThreadsafeInvocation(*pending[index]); + if (scope.exception()) [[unlikely]] { + ++index; + break; + } + } + for (; index < pending.size(); ++index) { + JSFFICallback* callback = pending[index]->callback(); + if (callback->endThreadsafeInvocation()) + callback->unroot(); + } + RETURN_IF_EXCEPTION(scope, { }); + return JSValue::encode(jsNumber(pending.size())); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFIArenaDepth, (JSGlobalObject* globalObject, CallFrame*)) +{ + DollarVMAssertScope assertScope; + return JSValue::encode(jsNumber(globalObject->ffiContext().arena().depth())); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFIFixture, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + String name = callFrame->argument(0).toWTFString(globalObject); + RETURN_IF_EXCEPTION(scope, { }); + + for (auto& entry : ffiTestFixtures()) { + if (name == String::fromLatin1(entry.name)) + return JSValue::encode(jsNumber(static_cast(reinterpret_cast(entry.address)))); + } + + return throwVMTypeError(globalObject, scope, makeString("Unknown FFI fixture '"_s, name, "'"_s)); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFIFixtures, (JSGlobalObject* globalObject, CallFrame*)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + JSArray* result = constructEmptyArray(globalObject, nullptr); + RETURN_IF_EXCEPTION(scope, { }); + for (auto& entry : ffiTestFixtures()) { + result->push(globalObject, jsString(vm, String::fromUTF8(entry.name))); + RETURN_IF_EXCEPTION(scope, { }); + } + return JSValue::encode(result); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFISignatureString, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + RefPtr signature = FFI::signatureFromJS(globalObject, callFrame->argument(0)); + RETURN_IF_EXCEPTION(scope, { }); + RELEASE_ASSERT(signature); // FFI::signatureFromJS throws on every failure. + + return JSValue::encode(jsString(vm, signature->toString())); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFIRead, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + void* address = dollarVMFFIPointerFromJS(globalObject, callFrame->argument(0)); + RETURN_IF_EXCEPTION(scope, { }); + if (!address) + return throwVMTypeError(globalObject, scope, "$vm.ffiRead: null pointer"_s); + + auto type = dollarVMParseFFIType(globalObject, callFrame->argument(1)); + RETURN_IF_EXCEPTION(scope, { }); + ASSERT(type); + if (!dollarVMFFIIsRawMemoryType(*type)) + return throwVMTypeError(globalObject, scope, makeString("$vm.ffiRead: unsupported type "_s, FFI::name(*type))); + + uint64_t slot = dollarVMFFILoadSlot(*type, address); + RELEASE_AND_RETURN(scope, JSValue::encode(FFI::jsValueFromSlot(globalObject, globalObject->ffiContext(), *type, slot))); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFIWrite, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + void* address = dollarVMFFIPointerFromJS(globalObject, callFrame->argument(0)); + RETURN_IF_EXCEPTION(scope, { }); + if (!address) + return throwVMTypeError(globalObject, scope, "$vm.ffiWrite: null pointer"_s); + + auto type = dollarVMParseFFIType(globalObject, callFrame->argument(1)); + RETURN_IF_EXCEPTION(scope, { }); + ASSERT(type); + if (!dollarVMFFIIsRawMemoryType(*type)) + return throwVMTypeError(globalObject, scope, makeString("$vm.ffiWrite: unsupported type "_s, FFI::name(*type))); + + uint64_t slot = 0; + FFI::writeSlotFromJSValue(globalObject, globalObject->ffiContext(), *type, callFrame->argument(2), slot, nullptr); + RETURN_IF_EXCEPTION(scope, { }); + + dollarVMFFIStoreSlot(*type, slot, address); + return JSValue::encode(jsUndefined()); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFICString, (JSGlobalObject* globalObject, CallFrame* callFrame)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + auto scope = DECLARE_THROW_SCOPE(vm); + + void* address = dollarVMFFIPointerFromJS(globalObject, callFrame->argument(0)); + RETURN_IF_EXCEPTION(scope, { }); + if (!address) + return JSValue::encode(jsNull()); + + return JSValue::encode(jsString(vm, String::fromUTF8(static_cast(address)))); +} + +JSC_DEFINE_HOST_FUNCTION(functionFFICompileCounts, (JSGlobalObject* globalObject, CallFrame*)) +{ + DollarVMAssertScope assertScope; + VM& vm = globalObject->vm(); + + JSObject* counts = constructEmptyObject(globalObject); + counts->putDirect(vm, Identifier::fromString(vm, "icStub"_s), jsNumber(static_cast(FFI::g_ffiCompileCounts.icStub.load()))); + counts->putDirect(vm, Identifier::fromString(vm, "dfgCallFFI"_s), jsNumber(static_cast(FFI::g_ffiCompileCounts.dfgCallFFI.load()))); + counts->putDirect(vm, Identifier::fromString(vm, "ftlCallFFI"_s), jsNumber(static_cast(FFI::g_ffiCompileCounts.ftlCallFFI.load()))); + return JSValue::encode(counts); +} + +#endif // USE(BUN_JSC_ADDITIONS) && USE(JSVALUE64) + constexpr unsigned jsDollarVMPropertyAttributes = PropertyAttribute::ReadOnly | PropertyAttribute::DontEnum | PropertyAttribute::DontDelete; void JSDollarVM::finishCreation(VM& vm) @@ -4713,6 +5155,20 @@ void JSDollarVM::finishCreation(VM& vm) addFunction(vm, alwaysAllow, "dumpLineBreakData"_s, functionDumpLineBreakData, 0); addFunction(vm, alwaysAllow, "weakCreate"_s, functionWeakCreate, 0); +#if USE(BUN_JSC_ADDITIONS) && USE(JSVALUE64) + addFunction(vm, allowIfNotFuzz, "ffiFunction"_s, functionFFIFunction, 4); + addFunction(vm, allowIfNotFuzz, "ffiCallback"_s, functionFFICallback, 3); + addFunction(vm, allowIfNotFuzz, "drainThreadsafeCallbacks"_s, functionDrainThreadsafeCallbacks, 0); + addFunction(vm, allowIfNotFuzz, "ffiFixture"_s, functionFFIFixture, 1); + addFunction(vm, allowIfNotFuzz, "ffiFixtures"_s, functionFFIFixtures, 0); + addFunction(vm, allowIfNotFuzz, "ffiSignatureString"_s, functionFFISignatureString, 1); + addFunction(vm, allowIfNotFuzz, "ffiRead"_s, functionFFIRead, 2); + addFunction(vm, allowIfNotFuzz, "ffiWrite"_s, functionFFIWrite, 3); + addFunction(vm, allowIfNotFuzz, "ffiCString"_s, functionFFICString, 1); + addFunction(vm, allowIfNotFuzz, "ffiArenaDepth"_s, functionFFIArenaDepth, 0); + addFunction(vm, allowIfNotFuzz, "ffiCompileCounts"_s, functionFFICompileCounts, 0); +#endif + if (allowIfNotFuzz) { m_objectDoingSideEffectPutWithoutCorrectSlotStatusStructureID.set(vm, this, ObjectDoingSideEffectPutWithoutCorrectSlotStatus::createStructure(vm, globalObject, jsNull())); m_testCustomGetterSetterStructureID.set(vm, this, JSTestCustomGetterSetter::createStructure(vm, globalObject)); diff --git a/build.ts b/build.ts index bea9f80682085..8d41c314d2fa1 100755 --- a/build.ts +++ b/build.ts @@ -286,7 +286,7 @@ function buildJSC() { runCommand( "cmake", - ["--build", buildDir, "--config", buildType, "--target", "jsc"], + ["--build", buildDir, "--config", buildType, "--target", "jsc", "--target", "testFFI"], { cwd: buildDir, env, diff --git a/mac-release.bash b/mac-release.bash index e4d0f5998bc81..5b1f16b827869 100755 --- a/mac-release.bash +++ b/mac-release.bash @@ -65,7 +65,7 @@ cmake \ -DENABLE_SANITIZERS="$ENABLE_SANITIZERS" \ $THIS_DIR \ $RUNNER_TEMP/webkit-release && - cmake --build $RUNNER_TEMP/webkit-release --config $CMAKE_BUILD_TYPE --target jsc + cmake --build $RUNNER_TEMP/webkit-release --config $CMAKE_BUILD_TYPE --target jsc --target testFFI # JSC compares ClassInfo by address; LTO can fold byte-identical ones together. python3 $THIS_DIR/Tools/Scripts/check-classinfo-uniqueness.py $RUNNER_TEMP/webkit-release/bin/jsc diff --git a/windows-release.ps1 b/windows-release.ps1 index 9cca115ee9e5d..9421f70ef8bd7 100644 --- a/windows-release.ps1 +++ b/windows-release.ps1 @@ -138,7 +138,7 @@ foreach ($file in $batFiles) { } Write-Host ":: Building WebKit" -cmake --build $WebKitBuild --config Release --target jsc --verbose +cmake --build $WebKitBuild --config Release --target jsc --target testFFI --verbose if ($LASTEXITCODE -ne 0) { throw "cmake --build failed with exit code $LASTEXITCODE" } Write-Host ":: Packaging ${output}"