diff --git a/crates/base64-simd/src/decode.rs b/crates/base64-simd/src/decode.rs index 41d9034..363dd19 100644 --- a/crates/base64-simd/src/decode.rs +++ b/crates/base64-simd/src/decode.rs @@ -4,7 +4,7 @@ use crate::{Config, Error, Extra, Kind}; use crate::{STANDARD_CHARSET, URL_SAFE_CHARSET}; use vsimd::alsw::AlswLut; -use vsimd::isa::{NEON, SSSE3, WASM128}; +use vsimd::isa::{NEON, SSSE3, VSX, WASM128}; use vsimd::mask::u8x32_highbit_any; use vsimd::matches_isa; use vsimd::tools::{read, write}; @@ -249,7 +249,7 @@ fn merge_bits_x2(s: S, x: V256) -> V256 { let m2 = s.u32x8_splat(u32::from_le_bytes([0x00, 0x10, 0x01, 0x00])); s.i16x16_madd(x1, m2) // {ccdddddd|bbbbcccc|aaaaaabb|00000000} x8 - } else if matches_isa!(S, NEON | WASM128) { + } else if matches_isa!(S, NEON | WASM128 | VSX) { let m1 = s.u32x8_splat(u32::from_le_bytes([0x3f, 0x00, 0x3f, 0x00])); let x1 = s.v256_and(x, m1); // x1: {00aaaaaa|00000000|00cccccc|00000000} x8 diff --git a/crates/base64-simd/src/encode.rs b/crates/base64-simd/src/encode.rs index cea8649..146a004 100644 --- a/crates/base64-simd/src/encode.rs +++ b/crates/base64-simd/src/encode.rs @@ -1,7 +1,7 @@ use crate::{Config, Kind}; use crate::{STANDARD_CHARSET, URL_SAFE_CHARSET}; -use vsimd::isa::{NEON, SSE2, WASM128}; +use vsimd::isa::{NEON, SSE2, VSX, WASM128}; use vsimd::tools::{read, write}; use vsimd::vector::{V128, V256}; use vsimd::{matches_isa, POD}; @@ -200,7 +200,7 @@ fn split_bits_x2(s: S, x: V256) -> V256 { // {00aaaaaa|00bbbbbb|00cccccc|00dddddd} x8 } - if matches_isa!(S, NEON | WASM128) { + if matches_isa!(S, NEON | WASM128 | VSX) { let m1 = s.u32x8_splat(u32::from_le_bytes([0x00, 0xfc, 0x00, 0x00])); let x1 = s.u16x16_shr::<10>(s.v256_and(x0, m1)); // x1: {00aaaaaa|000000000|00000000|00000000} x8 @@ -248,7 +248,7 @@ fn split_bits_x1(s: S, x: V128) -> V128 { return s.v128_or(x3, x4); } - if matches_isa!(S, NEON | WASM128) { + if matches_isa!(S, NEON | WASM128 | VSX) { let m1 = s.u32x4_splat(u32::from_le_bytes([0x00, 0xfc, 0x00, 0x00])); let x1 = s.u16x8_shr::<10>(s.v128_and(x0, m1)); diff --git a/crates/base64-simd/src/lib.rs b/crates/base64-simd/src/lib.rs index d03a8cb..88d76b1 100644 --- a/crates/base64-simd/src/lib.rs +++ b/crates/base64-simd/src/lib.rs @@ -20,6 +20,10 @@ // #![cfg_attr(not(any(feature = "std", test)), no_std)] #![cfg_attr(feature = "unstable", feature(arm_target_feature))] +#![cfg_attr( + all(feature = "unstable", target_arch = "powerpc64"), + feature(powerpc_target_feature) +)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(test, deny(warnings))] // diff --git a/crates/base64-simd/src/multiversion.rs b/crates/base64-simd/src/multiversion.rs index ebe9f25..5e6bcdc 100644 --- a/crates/base64-simd/src/multiversion.rs +++ b/crates/base64-simd/src/multiversion.rs @@ -5,8 +5,8 @@ vsimd::dispatch!( signature = {pub(crate) unsafe fn(src: *const u8, len: usize, dst: *mut u8, config: Config) -> ()}, fallback = {crate::encode::encode_fallback}, simd = {crate::encode::encode_simd}, - targets = {"avx2", "ssse3", "neon", "simd128"}, - fastest = {"avx2", "neon", "simd128"}, + targets = {"avx2", "ssse3", "neon", "simd128", "vsx"}, + fastest = {"avx2", "neon", "simd128", "vsx"}, ); vsimd::dispatch!( @@ -14,8 +14,8 @@ vsimd::dispatch!( signature = {pub(crate) unsafe fn(src: *const u8, dst: *mut u8, n: usize, config: Config) -> Result<(), Error>}, fallback = {crate::decode::decode_fallback}, simd = {crate::decode::decode_simd}, - targets = {"avx2", "ssse3", "neon", "simd128"}, - fastest = {"avx2", "neon", "simd128"}, + targets = {"avx2", "ssse3", "neon", "simd128", "vsx"}, + fastest = {"avx2", "neon", "simd128", "vsx"}, ); vsimd::dispatch!( @@ -23,8 +23,8 @@ vsimd::dispatch!( signature = {pub(crate) unsafe fn(src: *const u8, n: usize, config: Config) -> Result<(), Error>}, fallback = {crate::check::check_fallback}, simd = {crate::check::check_simd}, - targets = {"avx2", "ssse3", "neon", "simd128"}, - fastest = {"avx2", "neon", "simd128"}, + targets = {"avx2", "ssse3", "neon", "simd128", "vsx"}, + fastest = {"avx2", "neon", "simd128", "vsx"}, ); vsimd::dispatch!( @@ -32,6 +32,6 @@ vsimd::dispatch!( signature = {pub unsafe fn(src: *const u8, len: usize) -> usize}, fallback = {crate::ascii::find_non_ascii_whitespace_fallback}, simd = {crate::ascii::find_non_ascii_whitespace_simd}, - targets = {"avx2", "sse2", "neon", "simd128"}, - fastest = {"avx2", "neon", "simd128"}, + targets = {"avx2", "sse2", "neon", "simd128", "vsx"}, + fastest = {"avx2", "neon", "simd128", "vsx"}, ); diff --git a/crates/vsimd/src/isa.rs b/crates/vsimd/src/isa.rs index 95dba94..0798a28 100644 --- a/crates/vsimd/src/isa.rs +++ b/crates/vsimd/src/isa.rs @@ -24,6 +24,7 @@ pub enum InstructionSetTypeId { AVX2, NEON, WASM128, + VSX, } #[doc(hidden)] @@ -46,6 +47,7 @@ where AVX2 => matches!(super_ty, Fallback | SSE2 | SSSE3 | SSE41 | AVX2), NEON => matches!(super_ty, Fallback | NEON), WASM128 => matches!(super_ty, Fallback | WASM128), + VSX => matches!(super_ty, Fallback | VSX), }; S::ARCH && U::ARCH && inherits @@ -115,11 +117,16 @@ macro_rules! is_feature_detected { { std::arch::is_aarch64_feature_detected!($feature) } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + { + std::arch::is_powerpc64_feature_detected!($feature) + } #[cfg(not(any( target_arch = "x86", target_arch = "x86_64", target_arch = "arm", - target_arch = "aarch64" + target_arch = "aarch64", + all(feature = "unstable", target_arch = "powerpc64"), )))] { false @@ -303,3 +310,39 @@ unsafe impl InstructionSet for WASM128 { unsafe impl SIMD64 for WASM128 {} unsafe impl SIMD128 for WASM128 {} unsafe impl SIMD256 for WASM128 {} + +macro_rules! ppc64_is_enabled { + ($feature:tt) => {{ + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + { + is_feature_detected!($feature) + } + #[cfg(not(all(feature = "unstable", target_arch = "powerpc64")))] + { + false + } + }}; +} + +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy)] +pub struct VSX(()); + +unsafe impl InstructionSet for VSX { + const ID: InstructionSetTypeId = InstructionSetTypeId::VSX; + const ARCH: bool = cfg!(all(feature = "unstable", target_arch = "powerpc64")); + + #[inline(always)] + unsafe fn new() -> Self { + Self(()) + } + + #[inline(always)] + fn is_enabled() -> bool { + ppc64_is_enabled!("vsx") + } +} + +unsafe impl SIMD64 for VSX {} +unsafe impl SIMD128 for VSX {} +unsafe impl SIMD256 for VSX {} diff --git a/crates/vsimd/src/lib.rs b/crates/vsimd/src/lib.rs index 8ccd8ca..63adfda 100644 --- a/crates/vsimd/src/lib.rs +++ b/crates/vsimd/src/lib.rs @@ -1,12 +1,21 @@ //! ⚠️ This crate contains shared implementation details. Do not directly depend on it. #![cfg_attr(not(any(test, feature = "std")), no_std)] -#![cfg_attr(feature = "unstable", feature(portable_simd))] +#![cfg_attr(all(feature = "unstable", not(target_arch = "powerpc64")), feature(portable_simd))] #![cfg_attr( all(feature = "unstable", target_arch = "arm"), feature(arm_target_feature), feature(stdarch_arm_feature_detection), feature(stdarch_arm_neon_intrinsics) )] +#![cfg_attr( + all(feature = "unstable", target_arch = "powerpc64"), + feature(stdarch_powerpc), + feature(powerpc_target_feature) +)] +#![cfg_attr( + all(feature = "unstable", feature = "detect", target_arch = "powerpc64"), + feature(stdarch_powerpc_feature_detection) +)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(test, deny(warnings))] // @@ -81,5 +90,5 @@ pub mod mask; pub mod native; pub mod table; -#[cfg(feature = "unstable")] +#[cfg(all(feature = "unstable", not(target_arch = "powerpc64")))] pub mod unstable; diff --git a/crates/vsimd/src/macros.rs b/crates/vsimd/src/macros.rs index 7ee1ef0..57e6dbd 100644 --- a/crates/vsimd/src/macros.rs +++ b/crates/vsimd/src/macros.rs @@ -209,6 +209,17 @@ macro_rules! dispatch { } }; + (@resolve_static, "vsx", $($arg_name: ident),*) => { + #[cfg(all( + feature = "unstable", + target_arch = "powerpc64", + target_feature = "vsx", + ))] + { + return unsafe { vsx($($arg_name),*) } + } + }; + ( @iter_resolve_dynamic, targets = {$x:tt, $($xs:tt),+}, @@ -266,6 +277,13 @@ macro_rules! dispatch { } }; + (@resolve_dynamic, "vsx") => { + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if $crate::isa::VSX::is_enabled() { + return vsx; + } + }; + ( @iter_compile, signature = {$vis:vis unsafe fn($($arg_name: ident: $arg_type: ty),*) -> $ret:ty}, @@ -389,5 +407,20 @@ macro_rules! dispatch { use $crate::isa::{WASM128, InstructionSet as _}; $simd_fn(WASM128::new() $(,$arg_name)*) } + }; + + ( + @compile, + signature = {$vis:vis unsafe fn($($arg_name: ident: $arg_type: ty),*) -> $ret:ty}, + simd = {$simd_fn:path}, + target = {"vsx"}, + ) => { + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + #[inline] + #[target_feature(enable = "vsx")] + $vis unsafe fn vsx($($arg_name:$arg_type),*) -> $ret { + use $crate::isa::{VSX, InstructionSet as _}; + $simd_fn(VSX::new() $(,$arg_name)*) + } } } diff --git a/crates/vsimd/src/mask.rs b/crates/vsimd/src/mask.rs index d30ef45..f14b55f 100644 --- a/crates/vsimd/src/mask.rs +++ b/crates/vsimd/src/mask.rs @@ -1,4 +1,4 @@ -use crate::isa::{AVX2, NEON, SSE2, WASM128}; +use crate::isa::{AVX2, NEON, SSE2, VSX, WASM128}; use crate::vector::{V128, V256}; use crate::{SIMD128, SIMD256}; @@ -17,6 +17,9 @@ pub fn mask8x16_all(s: S, x: V128) -> bool { return s.u8x16_reduce_min(x) != 0; } } + if matches_isa!(S, VSX) { + return s.u8x16_any_zero(x).not(); + } unreachable!() } @@ -25,7 +28,7 @@ pub fn mask8x32_all(s: S, x: V256) -> bool { if matches_isa!(S, AVX2) { return s.u8x32_bitmask(x) == u32::MAX; } - if matches_isa!(S, SSE2 | WASM128 | NEON) { + if matches_isa!(S, SSE2 | WASM128 | NEON | VSX) { let x = x.to_v128x2(); let x = s.v128_and(x.0, x.1); return mask8x16_all(s, x); @@ -41,6 +44,9 @@ pub fn mask8x16_any(s: S, x: V128) -> bool { if matches_isa!(S, NEON) { return s.v128_all_zero(x).not(); } + if matches_isa!(S, VSX) { + return s.v128_all_zero(x).not(); + } unreachable!() } @@ -49,7 +55,7 @@ pub fn mask8x32_any(s: S, x: V256) -> bool { if matches_isa!(S, AVX2) { return s.u8x32_bitmask(x) != 0; } - if matches_isa!(S, SSE2 | WASM128 | NEON) { + if matches_isa!(S, SSE2 | WASM128 | NEON | VSX) { let x = x.to_v128x2(); let x = s.v128_or(x.0, x.1); return mask8x16_any(s, x); @@ -70,6 +76,9 @@ pub fn u8x16_highbit_all(s: S, x: V128) -> bool { return s.u8x16_reduce_min(x) >= 0x80; } } + if matches_isa!(S, VSX) { + return mask8x16_all(s, s.i8x16_lt(x, s.v128_create_zero())); + } unreachable!() } @@ -78,7 +87,7 @@ pub fn u8x32_highbit_all(s: S, x: V256) -> bool { if matches_isa!(S, AVX2) { return s.u8x32_bitmask(x) == u32::MAX; } - if matches_isa!(S, SSE2 | WASM128 | NEON) { + if matches_isa!(S, SSE2 | WASM128 | NEON | VSX) { let x = x.to_v128x2(); let x = s.v128_and(x.0, x.1); return u8x16_highbit_all(s, x); @@ -99,6 +108,9 @@ pub fn u8x16_highbit_any(s: S, x: V128) -> bool { return s.u8x16_reduce_max(x) >= 0x80; } } + if matches_isa!(S, VSX) { + return mask8x16_any(s, s.i8x16_lt(x, s.v128_create_zero())); + } unreachable!() } @@ -107,7 +119,7 @@ pub fn u8x32_highbit_any(s: S, x: V256) -> bool { if matches_isa!(S, AVX2) { return s.u8x32_bitmask(x) != 0; } - if matches_isa!(S, SSE2 | WASM128 | NEON) { + if matches_isa!(S, SSE2 | WASM128 | NEON | VSX) { let x = x.to_v128x2(); let x = s.v128_or(x.0, x.1); return u8x16_highbit_any(s, x); diff --git a/crates/vsimd/src/native.rs b/crates/vsimd/src/native.rs index d99db38..4c5eb44 100644 --- a/crates/vsimd/src/native.rs +++ b/crates/vsimd/src/native.rs @@ -18,6 +18,9 @@ enum Arch { #[cfg(target_arch = "wasm32")] Simd128, + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + Vsx, + Fallback, } @@ -51,6 +54,12 @@ impl Native { return Self(Arch::Simd128); } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + { + if is_feature_detected!("vsx") { + return Self(Arch::Vsx); + } + } Self(Arch::Fallback) } @@ -82,10 +91,18 @@ impl Native { Arch::Fallback => f(), } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + { + match self.0 { + Arch::Vsx => unsafe { ppc64::vsx(f) }, + Arch::Fallback => f(), + } + } #[cfg(not(any( // any(target_arch = "x86", target_arch = "x86_64"), // any(all(feature = "unstable", target_arch = "arm"), target_arch = "aarch64"), // - target_arch = "wasm32" // + target_arch = "wasm32", // + all(feature = "unstable", target_arch = "powerpc64"), // )))] { f() @@ -123,3 +140,8 @@ mod arm { mod wasm { generic_dispatch!(simd128, "simd128"); } + +#[cfg(all(feature = "unstable", target_arch = "powerpc64"))] +mod ppc64 { + generic_dispatch!(vsx, "vsx"); +} diff --git a/crates/vsimd/src/simd128.rs b/crates/vsimd/src/simd128.rs index 867eca3..0a6a333 100644 --- a/crates/vsimd/src/simd128.rs +++ b/crates/vsimd/src/simd128.rs @@ -1,6 +1,6 @@ #![allow(clippy::missing_transmute_annotations)] -use crate::isa::{NEON, SSE2, SSE41, WASM128}; +use crate::isa::{NEON, SSE2, SSE41, VSX, WASM128}; use crate::unified; use crate::vector::V128; use crate::SIMD64; @@ -11,7 +11,8 @@ use crate::isa::SSSE3; #[cfg(any( any(target_arch = "x86", target_arch = "x86_64"), any(all(feature = "unstable", target_arch = "arm"), target_arch = "aarch64"), - target_arch = "wasm32" + target_arch = "wasm32", + all(feature = "unstable", target_arch = "powerpc64"), ))] use core::mem::transmute as t; @@ -30,6 +31,9 @@ use core::arch::aarch64::*; #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; +#[cfg(all(feature = "unstable", target_arch = "powerpc64"))] +use core::arch::powerpc64::*; + pub unsafe trait SIMD128: SIMD64 { /// T1: SSE2, NEON, WASM128 #[inline(always)] @@ -48,6 +52,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return self.v128_load_unaligned(addr); } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return self.v128_load_unaligned(addr); + } { let _ = addr; unreachable!() @@ -69,6 +77,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return t(v128_load(addr.cast())); } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return t(vec_xl(0, addr)); + } { let _ = addr; unreachable!() @@ -92,6 +104,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return self.v128_store_unaligned(addr, a); } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return self.v128_store_unaligned(addr, a); + } { let _ = (addr, a); unreachable!() @@ -116,6 +132,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return v128_store(addr.cast(), t(a)); } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return vec_xst(t::<_, vector_unsigned_char>(a), 0, addr); + } { let _ = (addr, a); unreachable!() @@ -137,6 +157,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(u8x16_splat(0)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { t(vec_splats(0u8)) }; + } { unreachable!() } @@ -162,6 +186,13 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(v128_not(t(a))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_char = t(a); + t(vec_nor(a, a)) + }; + } { let _ = a; unreachable!() @@ -220,6 +251,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { !v128_any_true(t(a)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { vec_all_eq(t::<_, vector_unsigned_char>(a), vec_splats(0u8)) }; + } { let _ = a; unreachable!() @@ -361,6 +396,15 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(i16x8_mul(t(a), t(b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_signed_short = t(a); + let b: vector_signed_short = t(b); + let zero = vec_splats(0i16); + t(vec_mladd(a, b, zero)) + }; + } { let _ = (a, b); unreachable!() @@ -382,6 +426,14 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(i32x4_mul(t(a), t(b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_signed_int = t(a); + let b: vector_signed_int = t(b); + t(vec_mul(a, b)) + }; + } { let _ = (a, b); unreachable!() @@ -406,6 +458,14 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(u16x8_shl(t(a), IMM8 as u32)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_short = t(a); + let shift = vec_splats(IMM8 as u16); + t(vec_sl(a, shift)) + }; + } { let _ = a; unreachable!() @@ -427,6 +487,14 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(u32x4_shl(t(a), IMM8 as u32)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_int = t(a); + let shift = vec_splats(IMM8 as u32); + t(vec_sl(a, shift)) + }; + } { let _ = a; unreachable!() @@ -451,6 +519,14 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(u16x8_shr(t(a), IMM8 as u32)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_short = t(a); + let shift = vec_splats(IMM8 as u16); + t(vec_sr(a, shift)) + }; + } { let _ = a; unreachable!() @@ -472,6 +548,14 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(u32x4_shr(t(a), IMM8 as u32)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_int = t(a); + let shift = vec_splats(IMM8 as u32); + t(vec_sr(a, shift)) + }; + } { let _ = a; unreachable!() @@ -637,6 +721,29 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(u8x16_swizzle(t(a), t(b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + // vec_perm selects from the concatenation of its first two args (32 bytes) + // using only the low 5 bits of each index (& 0x1f). To emulate + // SSSE3/NEON/WASM swizzle semantics (produce zero for index >= 16), + // we place a zero vector in the second slot and remap any out-of-range + // index (>= 16, which includes sentinel values like 0x80) to 16 so that + // vec_perm reads from the zero vector instead of wrapping into `a`. + return unsafe { + let a: vector_unsigned_char = t(a); + let b: vector_unsigned_char = t(b); + let zero = vec_splats(0u8); + let threshold = vec_splats(16u8); + // For each lane: if index >= 16, replace with 16 (selects zero); + // otherwise keep the original index (selects from `a`). + // Note: vec_cmpgt(b, 15) is equivalent to b >= 16 for unsigned bytes. + // vec_cmpge only supports vector_float, not vector_unsigned_char. + let limit = vec_splats(15u8); + let oob: vector_bool_char = vec_cmpgt(b, limit); + let idx: vector_unsigned_char = vec_sel(b, threshold, oob); + t(vec_perm(a, zero, idx)) + }; + } { let _ = (a, b); unreachable!() @@ -646,7 +753,7 @@ pub unsafe trait SIMD128: SIMD64 { /// T1: SSE41, NEON, WASM128 #[inline(always)] fn u16x8_bswap(self, a: V128) -> V128 { - if matches_isa!(Self, SSE41 | WASM128) { + if matches_isa!(Self, SSE41 | WASM128 | VSX) { return self.u8x16_swizzle(a, crate::bswap::SHUFFLE_U16X8); } @@ -664,7 +771,7 @@ pub unsafe trait SIMD128: SIMD64 { /// T1: SSE41, NEON, WASM128 #[inline(always)] fn u32x4_bswap(self, a: V128) -> V128 { - if matches_isa!(Self, SSE41 | WASM128) { + if matches_isa!(Self, SSE41 | WASM128 | VSX) { return self.u8x16_swizzle(a, crate::bswap::SHUFFLE_U32X4); } @@ -682,7 +789,7 @@ pub unsafe trait SIMD128: SIMD64 { /// T1: SSE41, NEON, WASM128 #[inline(always)] fn u64x2_bswap(self, a: V128) -> V128 { - if matches_isa!(Self, SSE41 | WASM128) { + if matches_isa!(Self, SSE41 | WASM128 | VSX) { return self.u8x16_swizzle(a, crate::bswap::SHUFFLE_U64X2); } @@ -724,6 +831,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { !u8x16_all_true(t(a)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { vec_any_eq(t::<_, vector_unsigned_char>(a), vec_splats(0u8)) }; + } { let _ = a; unreachable!() @@ -744,6 +855,10 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, NEON) { unimplemented!() } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + unimplemented!() + } #[cfg(target_arch = "wasm32")] if matches_isa!(Self, WASM128) { return unsafe { u8x16_bitmask(t(a)) }; @@ -757,7 +872,7 @@ pub unsafe trait SIMD128: SIMD64 { /// T1: NEON-A64 #[inline(always)] fn u8x16_reduce_max(self, a: V128) -> u8 { - if matches_isa!(Self, SSE41 | WASM128) { + if matches_isa!(Self, SSE41 | WASM128 | VSX) { unimplemented!() } #[cfg(all(feature = "unstable", target_arch = "arm"))] @@ -777,7 +892,7 @@ pub unsafe trait SIMD128: SIMD64 { /// T1: NEON-A64 #[inline(always)] fn u8x16_reduce_min(self, a: V128) -> u8 { - if matches_isa!(Self, SSE41 | WASM128) { + if matches_isa!(Self, SSE41 | WASM128 | VSX) { unimplemented!() } #[cfg(all(feature = "unstable", target_arch = "arm"))] @@ -799,7 +914,7 @@ pub unsafe trait SIMD128: SIMD64 { /// T2: SSE2, WASM128 #[inline(always)] fn v128_bsl(self, a: V128, b: V128, c: V128) -> V128 { - if matches_isa!(Self, SSE2 | WASM128) { + if matches_isa!(Self, SSE2 | WASM128 | VSX) { return self.v128_xor(self.v128_and(self.v128_xor(b, c), a), c); } #[cfg(any(all(feature = "unstable", target_arch = "arm"), target_arch = "aarch64"))] @@ -833,6 +948,14 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_char = t(a); + let b: vector_unsigned_char = t(b); + t(vec_mergeh(a, b)) + }; + } { let _ = (a, b); unreachable!() @@ -860,6 +983,14 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_char = t(a); + let b: vector_unsigned_char = t(b); + t(vec_mergel(a, b)) + }; + } { let _ = (a, b); unreachable!() @@ -887,6 +1018,14 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_short = t(a); + let b: vector_unsigned_short = t(b); + t(vec_mergeh(a, b)) + }; + } { let _ = (a, b); unreachable!() @@ -914,6 +1053,14 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_short = t(a); + let b: vector_unsigned_short = t(b); + t(vec_mergel(a, b)) + }; + } { let _ = (a, b); unreachable!() @@ -941,6 +1088,14 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u32x4_shuffle::<0, 4, 1, 5>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_int = t(a); + let b: vector_unsigned_int = t(b); + t(vec_mergeh(a, b)) + }; + } { let _ = (a, b); unreachable!() @@ -968,6 +1123,14 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u32x4_shuffle::<2, 6, 3, 7>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_int = t(a); + let b: vector_unsigned_int = t(b); + t(vec_mergel(a, b)) + }; + } { let _ = (a, b); unreachable!() @@ -994,6 +1157,13 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u64x2_shuffle::<0, 2>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let (a, b): ([u64; 2], [u64; 2]) = (t(a), t(b)); + t([a[0], b[0]]) + }; + } { let _ = (a, b); unreachable!() @@ -1020,6 +1190,13 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u64x2_shuffle::<1, 3>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let (a, b): ([u64; 2], [u64; 2]) = (t(a), t(b)); + t([a[1], b[1]]) + }; + } { let _ = (a, b); unreachable!() @@ -1047,6 +1224,17 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_char = t(a); + let b: vector_unsigned_char = t(b); + // Select even bytes: 0,2,4,6,8,10,12,14 from a, then 0,2,4,6,8,10,12,14 from b + // On LE ppc64, vec_perm indices: bytes from a are 0-15, bytes from b are 16-31 + let perm: vector_unsigned_char = t([0u8, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]); + t(vec_perm(a, b, perm)) + }; + } { let _ = (a, b); unreachable!() @@ -1074,6 +1262,16 @@ pub unsafe trait SIMD128: SIMD64 { let ans = u8x16_shuffle::<1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31>(a, b); return unsafe { t(ans) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_char = t(a); + let b: vector_unsigned_char = t(b); + // Select odd bytes: 1,3,5,7,9,11,13,15 from a, then 1,3,5,7,9,11,13,15 from b + let perm: vector_unsigned_char = t([1u8, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31]); + t(vec_perm(a, b, perm)) + }; + } { let _ = (a, b); unreachable!() @@ -1087,7 +1285,7 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, SSE2) { return unsafe { t(_mm_mulhi_epu16(t(a), t(b))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { @@ -1103,7 +1301,7 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, SSE2) { return unsafe { t(_mm_mulhi_epi16(t(a), t(b))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { @@ -1119,7 +1317,7 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, SSSE3) { return unsafe { t(_mm_maddubs_epi16(t(a), t(b))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { @@ -1135,7 +1333,7 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, SSE41) { return unsafe { t(_mm_blend_epi16::(t(a), t(b))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { @@ -1153,7 +1351,7 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, SSE41) { return unsafe { t(_mm_blendv_epi8(t(a), t(b), t(c))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { @@ -1169,7 +1367,7 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, SSE2) { return unsafe { t(_mm_madd_epi16(t(a), t(b))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { @@ -1193,6 +1391,14 @@ pub unsafe trait SIMD128: SIMD64 { if matches_isa!(Self, WASM128) { return unsafe { t(u8x16_avgr(t(a), t(b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + return unsafe { + let a: vector_unsigned_char = t(a); + let b: vector_unsigned_char = t(b); + t(vec_avg(a, b)) + }; + } { let _ = (a, b); unreachable!() diff --git a/crates/vsimd/src/simd256.rs b/crates/vsimd/src/simd256.rs index adb4f9b..f609383 100644 --- a/crates/vsimd/src/simd256.rs +++ b/crates/vsimd/src/simd256.rs @@ -1,6 +1,6 @@ #![allow(clippy::missing_transmute_annotations)] -use crate::isa::{AVX2, NEON, SSE2, WASM128}; +use crate::isa::{AVX2, NEON, SSE2, VSX, WASM128}; use crate::vector::{V128, V256}; use crate::{unified, SIMD128}; @@ -503,7 +503,7 @@ pub unsafe trait SIMD256: SIMD128 { #[inline(always)] fn u8x32_swizzle(self, a: V256, b: V256) -> V256 { - if matches_isa!(Self, SSE2 | WASM128) { + if matches_isa!(Self, SSE2 | WASM128 | VSX) { let _ = (a, b); unimplemented!() } @@ -569,7 +569,7 @@ pub unsafe trait SIMD256: SIMD128 { /// ans = ((b ^ c) & a) ^ c #[inline(always)] fn v256_bsl(self, a: V256, b: V256, c: V256) -> V256 { - if matches_isa!(Self, NEON) { + if matches_isa!(Self, NEON | VSX) { return simd256_vop!(self, Self::v128_bsl, a, b, c); } { @@ -608,6 +608,13 @@ pub unsafe trait SIMD256: SIMD128 { V256::from_v128x2((low, high)) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(Self, VSX) { + let zero = self.v128_create_zero(); + let lo = self.u8x16_zip_lo(a, zero); + let hi = self.u8x16_zip_hi(a, zero); + return V256::from_v128x2((lo, hi)); + } { let _ = a; unreachable!() @@ -708,7 +715,7 @@ pub unsafe trait SIMD256: SIMD128 { if matches_isa!(Self, AVX2) { return unsafe { t(_mm256_permute2x128_si256::<0b0010_0000>(t(a), t(b))) }; } - if matches_isa!(Self, SSE2 | NEON | WASM128) { + if matches_isa!(Self, SSE2 | NEON | WASM128 | VSX) { let ((a, _), (c, _)) = (a.to_v128x2(), b.to_v128x2()); return V256::from_v128x2((a, c)); } @@ -723,7 +730,7 @@ pub unsafe trait SIMD256: SIMD128 { if matches_isa!(Self, AVX2) { return unsafe { t(_mm256_permute2x128_si256::<0b0011_0001>(t(a), t(b))) }; } - if matches_isa!(Self, SSE2 | NEON | WASM128) { + if matches_isa!(Self, SSE2 | NEON | WASM128 | VSX) { let ((_, b), (_, d)) = (a.to_v128x2(), b.to_v128x2()); return V256::from_v128x2((b, d)); } @@ -738,7 +745,7 @@ pub unsafe trait SIMD256: SIMD128 { if matches_isa!(Self, AVX2) { return unsafe { t(_mm256_permute4x64_epi64::(t(a))) }; } - if matches_isa!(Self, SSE2 | NEON | WASM128) { + if matches_isa!(Self, SSE2 | NEON | WASM128 | VSX) { let _ = a; unimplemented!() } @@ -843,7 +850,7 @@ pub unsafe trait SIMD256: SIMD128 { if matches_isa!(Self, AVX2) { return unsafe { t(_mm256_blend_epi32::(t(a), t(b))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { @@ -859,7 +866,7 @@ pub unsafe trait SIMD256: SIMD128 { if matches_isa!(Self, AVX2) { return unsafe { t(_mm256_blendv_epi8(t(a), t(b), t(c))) }; } - if matches_isa!(Self, NEON | WASM128) { + if matches_isa!(Self, NEON | WASM128 | VSX) { unimplemented!() } { diff --git a/crates/vsimd/src/table.rs b/crates/vsimd/src/table.rs index bb5f423..ac63055 100644 --- a/crates/vsimd/src/table.rs +++ b/crates/vsimd/src/table.rs @@ -1,4 +1,4 @@ -use crate::isa::{NEON, SSSE3, WASM128}; +use crate::isa::{NEON, SSSE3, VSX, WASM128}; use crate::pod::POD; use crate::Scalable; @@ -17,5 +17,19 @@ where return s.u8x16xn_swizzle(lut, idx); } + // VSX vec_perm uses idx & 0x1f, so indices 0-15 select from lut (first vector), + // and indices 16-31 select from the zero vector (second vector). + // Lookup semantics: return lut[x & 0x0f] when x < 128, return 0 when x >= 128. + // Strategy: do the lookup with masked indices, then zero out results for x >= 128. + if matches_isa!(S, VSX) { + let lo_nibble = s.and(x, s.u8xn_splat(0x0f)); + let result = s.u8x16xn_swizzle(lut, lo_nibble); + // For x >= 128, viewed as signed i8, the value is negative. + // i8xn_lt(x, 0) gives 0xFF for x >= 128, 0x00 for x < 128. + let is_negative = s.i8xn_lt(x, s.u8xn_splat(0)); + // Zero out entries where x >= 128: result & !is_negative + return s.andnot(result, is_negative); + } + unreachable!() } diff --git a/crates/vsimd/src/unified.rs b/crates/vsimd/src/unified.rs index 46a61c8..b445cde 100644 --- a/crates/vsimd/src/unified.rs +++ b/crates/vsimd/src/unified.rs @@ -14,6 +14,9 @@ use crate::isa::NEON; #[cfg(target_arch = "wasm32")] use crate::isa::WASM128; +#[cfg(all(feature = "unstable", target_arch = "powerpc64"))] +use crate::isa::VSX; + #[cfg(target_arch = "x86")] use core::arch::x86::*; @@ -29,6 +32,9 @@ use core::arch::aarch64::*; #[cfg(target_arch = "wasm32")] use core::arch::wasm32::*; +#[cfg(all(feature = "unstable", target_arch = "powerpc64"))] +use core::arch::powerpc64::*; + #[inline(always)] pub fn splat(s: S, x: T) -> V { if is_pod_type!(V, V256) { @@ -89,6 +95,24 @@ pub fn splat(s: S, x: T) -> V { return unsafe { tc(&u64x2_splat(tc(&x))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, u8 | i8) { + return unsafe { tc(&vec_splats(tc::<_, u8>(&x))) }; + } + if is_pod_type!(T, u16 | i16) { + return unsafe { tc(&vec_splats(tc::<_, u16>(&x))) }; + } + if is_pod_type!(T, u32 | i32) { + return unsafe { tc(&vec_splats(tc::<_, u32>(&x))) }; + } + if is_pod_type!(T, u64 | i64) { + return unsafe { + let val: u64 = tc(&x); + tc(&core::mem::transmute::<[u64; 2], vector_unsigned_char>([val, val])) + }; + } + } } { let _ = (s, x); @@ -168,6 +192,40 @@ pub fn add(s: S, a: V, b: V) -> V { return unsafe { tc(&u64x2_add(tc(&a), tc(&b))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, u8 | i8) { + return unsafe { + tc(&vec_add( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16 | i16) { + return unsafe { + tc(&vec_add( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + if is_pod_type!(T, u32 | i32) { + return unsafe { + tc(&vec_add( + tc::<_, vector_unsigned_int>(&a), + tc::<_, vector_unsigned_int>(&b), + )) + }; + } + if is_pod_type!(T, u64 | i64) { + return unsafe { + let a: [u64; 2] = tc(&a); + let b: [u64; 2] = tc(&b); + tc(&[a[0].wrapping_add(b[0]), a[1].wrapping_add(b[1])]) + }; + } + } } { let _ = (s, a, b); @@ -247,6 +305,40 @@ pub fn sub(s: S, a: V, b: V) -> V { return unsafe { tc(&u64x2_sub(tc(&a), tc(&b))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, u8 | i8) { + return unsafe { + tc(&vec_sub( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16 | i16) { + return unsafe { + tc(&vec_sub( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + if is_pod_type!(T, u32 | i32) { + return unsafe { + tc(&vec_sub( + tc::<_, vector_unsigned_int>(&a), + tc::<_, vector_unsigned_int>(&b), + )) + }; + } + if is_pod_type!(T, u64 | i64) { + return unsafe { + let a: [u64; 2] = tc(&a); + let b: [u64; 2] = tc(&b); + tc(&[a[0].wrapping_sub(b[0]), a[1].wrapping_sub(b[1])]) + }; + } + } } { let _ = (s, a, b); @@ -324,6 +416,42 @@ pub fn eq(s: S, a: V, b: V) -> V { return unsafe { tc(&u64x2_eq(tc(&a), tc(&b))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, u8 | i8) { + return unsafe { + tc(&vec_cmpeq( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16 | i16) { + return unsafe { + tc(&vec_cmpeq( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + if is_pod_type!(T, u32 | i32) { + return unsafe { + tc(&vec_cmpeq( + tc::<_, vector_unsigned_int>(&a), + tc::<_, vector_unsigned_int>(&b), + )) + }; + } + if is_pod_type!(T, u64 | i64) { + return unsafe { + let a: [u64; 2] = tc(&a); + let b: [u64; 2] = tc(&b); + let r0: u64 = if a[0] == b[0] { u64::MAX } else { 0 }; + let r1: u64 = if a[1] == b[1] { u64::MAX } else { 0 }; + tc(&[r0, r1]) + }; + } + } } { let _ = (s, a, b); @@ -470,6 +598,57 @@ pub fn lt(s: S, a: V, b: V) -> V { // return unsafe { tc(&u64x2_lt(tc(&a), tc(&b))) }; // } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, i8) { + return unsafe { + tc(&vec_cmplt( + tc::<_, vector_signed_char>(&a), + tc::<_, vector_signed_char>(&b), + )) + }; + } + if is_pod_type!(T, i16) { + return unsafe { + tc(&vec_cmplt( + tc::<_, vector_signed_short>(&a), + tc::<_, vector_signed_short>(&b), + )) + }; + } + if is_pod_type!(T, i32) { + return unsafe { + tc(&vec_cmplt( + tc::<_, vector_signed_int>(&a), + tc::<_, vector_signed_int>(&b), + )) + }; + } + if is_pod_type!(T, u8) { + return unsafe { + tc(&vec_cmplt( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16) { + return unsafe { + tc(&vec_cmplt( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + if is_pod_type!(T, u32) { + return unsafe { + tc(&vec_cmplt( + tc::<_, vector_unsigned_int>(&a), + tc::<_, vector_unsigned_int>(&b), + )) + }; + } + } } { let _ = (s, a, b); @@ -555,6 +734,41 @@ pub fn add_sat(s: S, a: V, b: V) -> V { return unsafe { tc(&u16x8_add_sat(tc(&a), tc(&b))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, i8) { + return unsafe { + tc(&vec_adds( + tc::<_, vector_signed_char>(&a), + tc::<_, vector_signed_char>(&b), + )) + }; + } + if is_pod_type!(T, i16) { + return unsafe { + tc(&vec_adds( + tc::<_, vector_signed_short>(&a), + tc::<_, vector_signed_short>(&b), + )) + }; + } + if is_pod_type!(T, u8) { + return unsafe { + tc(&vec_adds( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16) { + return unsafe { + tc(&vec_adds( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + } } { let _ = (s, a, b); @@ -640,6 +854,41 @@ pub fn sub_sat(s: S, a: V, b: V) -> V { return unsafe { tc(&u16x8_sub_sat(tc(&a), tc(&b))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, i8) { + return unsafe { + tc(&vec_subs( + tc::<_, vector_signed_char>(&a), + tc::<_, vector_signed_char>(&b), + )) + }; + } + if is_pod_type!(T, i16) { + return unsafe { + tc(&vec_subs( + tc::<_, vector_signed_short>(&a), + tc::<_, vector_signed_short>(&b), + )) + }; + } + if is_pod_type!(T, u8) { + return unsafe { + tc(&vec_subs( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16) { + return unsafe { + tc(&vec_subs( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + } } { let _ = (s, a, b); @@ -769,6 +1018,52 @@ pub fn max(s: S, a: V, b: V) -> V { return unsafe { tc(&f32x4_max(tc(&a), tc(&b))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, i8) { + return unsafe { + tc(&vec_max( + tc::<_, vector_signed_char>(&a), + tc::<_, vector_signed_char>(&b), + )) + }; + } + if is_pod_type!(T, i16) { + return unsafe { + tc(&vec_max( + tc::<_, vector_signed_short>(&a), + tc::<_, vector_signed_short>(&b), + )) + }; + } + if is_pod_type!(T, i32) { + return unsafe { tc(&vec_max(tc::<_, vector_signed_int>(&a), tc::<_, vector_signed_int>(&b))) }; + } + if is_pod_type!(T, u8) { + return unsafe { + tc(&vec_max( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16) { + return unsafe { + tc(&vec_max( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + if is_pod_type!(T, u32) { + return unsafe { + tc(&vec_max( + tc::<_, vector_unsigned_int>(&a), + tc::<_, vector_unsigned_int>(&b), + )) + }; + } + } } { let _ = (s, a, b); @@ -900,6 +1195,52 @@ pub fn min(s: S, a: V, b: V) -> V { return unsafe { tc(&f32x4_min(tc(&a), tc(&b))) }; } } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + if is_pod_type!(T, i8) { + return unsafe { + tc(&vec_min( + tc::<_, vector_signed_char>(&a), + tc::<_, vector_signed_char>(&b), + )) + }; + } + if is_pod_type!(T, i16) { + return unsafe { + tc(&vec_min( + tc::<_, vector_signed_short>(&a), + tc::<_, vector_signed_short>(&b), + )) + }; + } + if is_pod_type!(T, i32) { + return unsafe { tc(&vec_min(tc::<_, vector_signed_int>(&a), tc::<_, vector_signed_int>(&b))) }; + } + if is_pod_type!(T, u8) { + return unsafe { + tc(&vec_min( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } + if is_pod_type!(T, u16) { + return unsafe { + tc(&vec_min( + tc::<_, vector_unsigned_short>(&a), + tc::<_, vector_unsigned_short>(&b), + )) + }; + } + if is_pod_type!(T, u32) { + return unsafe { + tc(&vec_min( + tc::<_, vector_unsigned_int>(&a), + tc::<_, vector_unsigned_int>(&b), + )) + }; + } + } } { let _ = (s, a, b); @@ -939,6 +1280,15 @@ where if matches_isa!(S, WASM128) { return unsafe { tc(&v128_and(tc(&a), tc(&b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + return unsafe { + tc(&vec_and( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } } { let _ = (s, a, b); @@ -978,6 +1328,15 @@ where if matches_isa!(S, WASM128) { return unsafe { tc(&v128_or(tc(&a), tc(&b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + return unsafe { + tc(&vec_or( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } } { let _ = (s, a, b); @@ -1017,6 +1376,15 @@ where if matches_isa!(S, WASM128) { return unsafe { tc(&v128_xor(tc(&a), tc(&b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + return unsafe { + tc(&vec_xor( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } } { let _ = (s, a, b); @@ -1058,6 +1426,16 @@ where if matches_isa!(S, WASM128) { return unsafe { tc(&v128_andnot(tc(&a), tc(&b))) }; } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if matches_isa!(S, VSX) { + // andnot(a, b) = a & !b = vec_andc(a, b) + return unsafe { + tc(&vec_andc( + tc::<_, vector_unsigned_char>(&a), + tc::<_, vector_unsigned_char>(&b), + )) + }; + } } { let _ = (s, a, b); diff --git a/crates/vsimd/src/vector.rs b/crates/vsimd/src/vector.rs index a6e7518..746d865 100644 --- a/crates/vsimd/src/vector.rs +++ b/crates/vsimd/src/vector.rs @@ -4,7 +4,7 @@ use core::mem::transmute; // vectors should have `repr(simd)` if possible. -#[cfg(feature = "unstable")] +#[cfg(all(feature = "unstable", not(target_arch = "powerpc64")))] item_group! { use core::simd::{u8x16, u8x32, u8x64, u8x8}; @@ -93,6 +93,27 @@ item_group! { pub struct V512(v128, v128, v128, v128); } +#[cfg(all(feature = "unstable", target_arch = "powerpc64"))] +item_group! { + use core::arch::powerpc64::*; + + #[derive(Debug, Clone, Copy)] + #[repr(transparent)] + pub struct V64(u64); + + #[derive(Debug, Clone, Copy)] + #[repr(transparent)] + pub struct V128(vector_unsigned_char); + + #[derive(Debug, Clone, Copy)] + #[repr(C, align(32))] + pub struct V256(vector_unsigned_char, vector_unsigned_char); + + #[derive(Debug, Clone, Copy)] + #[repr(C, align(64))] + pub struct V512(vector_unsigned_char, vector_unsigned_char, vector_unsigned_char, vector_unsigned_char); +} + #[cfg(all( not(feature = "unstable"), not(any( diff --git a/crates/vsimd/tests/it.rs b/crates/vsimd/tests/it.rs index 6222749..916a87f 100644 --- a/crates/vsimd/tests/it.rs +++ b/crates/vsimd/tests/it.rs @@ -3,6 +3,12 @@ use vsimd::isa::{NEON, SSE2, WASM128}; use vsimd::vector::V128; use vsimd::SIMD128; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +use vsimd::isa::SSSE3; + +#[cfg(all(feature = "unstable", target_arch = "powerpc64"))] +use vsimd::isa::VSX; + use const_str::hex; #[cfg(not(miri))] @@ -55,3 +61,96 @@ fn u8x16_any_zero() { test(hex!("00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F"), true); test(hex!("10 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F"), false); } + +#[cfg_attr(not(target_arch = "wasm32"), test)] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] +fn u8x16_swizzle_out_of_range_produces_zero() { + /// Reference implementation: index in 0..15 selects from `a`; + /// index with high bit set (>= 128) yields 0. + /// Indices 16..127 are implementation-defined and not tested. + fn swizzle_scalar(a: [u8; 16], b: [u8; 16]) -> [u8; 16] { + let mut out = [0u8; 16]; + for i in 0..16 { + out[i] = if b[i] & 0x80 != 0 { 0 } else { a[(b[i] & 0x0f) as usize] }; + } + out + } + + fn f(a: [u8; 16], b: [u8; 16]) -> [u8; 16] { + let va = V128::from_bytes(a); + let vb = V128::from_bytes(b); + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + if let Some(s) = detect::() { + return *s.u8x16_swizzle(va, vb).as_bytes(); + } + if let Some(s) = detect::() { + return *s.u8x16_swizzle(va, vb).as_bytes(); + } + if let Some(s) = detect::() { + return *s.u8x16_swizzle(va, vb).as_bytes(); + } + #[cfg(all(feature = "unstable", target_arch = "powerpc64"))] + if let Some(s) = detect::() { + return *s.u8x16_swizzle(va, vb).as_bytes(); + } + + swizzle_scalar(a, b) + } + + fn test(a: [u8; 16], b: [u8; 16], expected: [u8; 16]) { + let result = f(a, b); + assert_eq!(result, expected, "a={a:02x?}, b={b:02x?}"); + } + + let data: [u8; 16] = [ + 0x10, 0x21, 0x32, 0x43, 0x54, 0x65, 0x76, 0x87, 0x98, 0xA9, 0xBA, 0xCB, 0xDC, 0xED, 0xFE, 0x0F, + ]; + + // Identity shuffle: indices 0..15 select each byte in order. + test(data, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], data); + + // Reverse shuffle. + test( + data, + [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0], + [ + 0x0F, 0xFE, 0xED, 0xDC, 0xCB, 0xBA, 0xA9, 0x98, 0x87, 0x76, 0x65, 0x54, 0x43, 0x32, 0x21, 0x10, + ], + ); + + // All out-of-range (0x80): every lane must be zero. + test(data, [0x80; 16], [0x00; 16]); + + // All out-of-range (0xFF): every lane must be zero. + test(data, [0xFF; 16], [0x00; 16]); + + // Mix of valid indices and 0x80 sentinels. + test( + data, + [ + 0, 0x80, 2, 0x80, 4, 0x80, 6, 0x80, 8, 0x80, 10, 0x80, 12, 0x80, 14, 0x80, + ], + [ + 0x10, 0x00, 0x32, 0x00, 0x54, 0x00, 0x76, 0x00, 0x98, 0x00, 0xBA, 0x00, 0xDC, 0x00, 0xFE, 0x00, + ], + ); + + // Broadcast byte 0 everywhere, except lane 7 which is out of range. + test( + data, + [0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0, 0, 0], + [ + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x00, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + ], + ); + + // Various high-bit-set values: 0x80, 0x90, 0xA0, 0xC8, 0xFF. + test( + data, + [0, 0x80, 2, 0x90, 4, 0xA0, 6, 0xC8, 8, 0xFF, 10, 0x80, 12, 15, 14, 0xFE], + [ + 0x10, 0x00, 0x32, 0x00, 0x54, 0x00, 0x76, 0x00, 0x98, 0x00, 0xBA, 0x00, 0xDC, 0x0F, 0xFE, 0x00, + ], + ); +} diff --git a/scripts/testgen.py b/scripts/testgen.py index 332cad6..1246695 100755 --- a/scripts/testgen.py +++ b/scripts/testgen.py @@ -35,11 +35,11 @@ "-C target-feature=+simd128", "", ], + "mips": [""], "powerpc": [ "-C target-feature=+vsx", "", ], - "mips": [""], } TARGETS = { @@ -52,8 +52,8 @@ "armv7-unknown-linux-gnueabihf", ], "wasm": ["wasm32-unknown-unknown"], - "powerpc": ["powerpc64le-unknown-linux-gnu"], "mips": ["mips-unknown-linux-gnu"], + "powerpc": ["powerpc64le-unknown-linux-gnu"], } TARGET_REMAP = { @@ -62,11 +62,11 @@ "aarch64-unknown-linux-gnu": "arm", "armv7-unknown-linux-gnueabihf": "arm", "wasm32-unknown-unknown": "wasm", - "powerpc64le-unknown-linux-gnu": "powerpc", "mips-unknown-linux-gnu": "mips", + "powerpc64le-unknown-linux-gnu": "powerpc", } -TEST_MODES = ["x86", "arm", "wasm", "powerpc", "mips"] +TEST_MODES = ["x86", "arm", "wasm", "mips", "powerpc"] def gen(mode: str, target: str, rustflag: str, host: str): @@ -76,12 +76,16 @@ def gen(mode: str, target: str, rustflag: str, host: str): feat = "--features " + feat if mode == "wasm": - print(f'RUSTFLAGS="{rustflag}" wasm-pack test --node -- --no-default-features {feat} $@') + print( + f'RUSTFLAGS="{rustflag}" wasm-pack test --node -- --no-default-features {feat} $@' + ) continue prog = "cross" if target != host else "cargo" skip_others = "--lib --tests" if mode == "x86" else "" - print(f'RUSTFLAGS="{rustflag}" {prog} test --target {target} {skip_others} --no-default-features {feat} $@') + print( + f'RUSTFLAGS="{rustflag}" {prog} test --target {target} {skip_others} --no-default-features {feat} $@' + ) def get_rustc_host():