prepare stdsimd for publishing
diff --git a/crates/stdsimd/Cargo.toml b/crates/stdsimd/Cargo.toml index 726ccfa..512fc7a 100644 --- a/crates/stdsimd/Cargo.toml +++ b/crates/stdsimd/Cargo.toml
@@ -20,7 +20,7 @@ [dependencies] #coresimd = { version = "0.0.3", path = "../coresimd" } -coresimd = "0.1.1" +coresimd = "0.1.2" libc = "0.2" cfg-if = "0.1"
diff --git a/crates/stdsimd/README.md b/crates/stdsimd/README.md new file mode 100644 index 0000000..e56d4de --- /dev/null +++ b/crates/stdsimd/README.md
@@ -0,0 +1,69 @@ +stdsimd - Rust's standard library SIMD components +======= + +[![Travis-CI Status]][travis] [![Appveyor Status]][appveyor] [![Latest Version]][crates.io] [![docs]][docs.rs] + +# Usage + +`stdsimd` is now shipped with Rust's `std` library - its is part of `libcore` +and `libstd`. + +The easiest way to use it is just to import it via `use std::arch`. + +The `std::arch` component for `x86` is available in stable Rust. The `std::arch` +components for other architectures and the `std::simd` component require nightly +Rust. + +Using `stdsimd` master branch is not recommended. It requires nightly Rust, it +only works with particular Rust nightly versions, and it can (and does) break +often. If you need to use `stdsimd` master branch, you can add it to your +`Cargo.toml` as follows: + +```toml +#[dependencies] +stdsimd = { git = "https://github.com/rust-lang-nursery/stdsimd.git" } +``` + +# Documentation + +* [Documentation - i686][i686] +* [Documentation - x86\_64][x86_64] +* [Documentation - arm][arm] +* [Documentation - aarch64][aarch64] +* [Documentation - powerpc][powerpc] +* [Documentation - powerpc64][powerpc64] +* [How to get started][contrib] +* [How to help implement intrinsics][help-implement] + +[contrib]: https://github.com/rust-lang-nursery/stdsimd/blob/master/CONTRIBUTING.md +[help-implement]: https://github.com/rust-lang-nursery/stdsimd/issues/40 +[i686]: https://rust-lang-nursery.github.io/stdsimd/i686/stdsimd/ +[x86_64]: https://rust-lang-nursery.github.io/stdsimd/x86_64/stdsimd/ +[arm]: https://rust-lang-nursery.github.io/stdsimd/arm/stdsimd/ +[aarch64]: https://rust-lang-nursery.github.io/stdsimd/aarch64/stdsimd/ +[powerpc]: https://rust-lang-nursery.github.io/stdsimd/powerpc/stdsimd/ +[powerpc64]: https://rust-lang-nursery.github.io/stdsimd/powerpc64/stdsimd/ + +### Approach + +The main goal is to expose APIs defined by *vendors* with the least amount of +abstraction possible. On x86, for example, the API should correspond to that +provided by `emmintrin.h`. + +# License + +`stdsimd` is primarily distributed under the terms of both the MIT license and +the Apache License (Version 2.0), with portions covered by various BSD-like +licenses. + +See LICENSE-APACHE, and LICENSE-MIT for details. + + +[travis]: https://travis-ci.org/rust-lang-nursery/stdsimd +[Travis-CI Status]: https://travis-ci.org/rust-lang-nursery/stdsimd.svg?branch=master +[appveyor]: https://ci.appveyor.com/project/rust-lang-libs/stdsimd/branch/master +[Appveyor Status]: https://ci.appveyor.com/api/projects/status/ix74qhmilpibn00x/branch/master?svg=true +[Latest Version]: https://img.shields.io/crates/v/stdsimd.svg +[crates.io]: https://crates.io/crates/stdsimd +[docs]: https://docs.rs/stdsimd/badge.svg +[docs.rs]: https://docs.rs/stdsimd/
diff --git a/crates/stdsimd/src/lib.rs b/crates/stdsimd/src/lib.rs index 021dc06..25479d3 100644 --- a/crates/stdsimd/src/lib.rs +++ b/crates/stdsimd/src/lib.rs
@@ -21,7 +21,7 @@ #[macro_use(println, print)] extern crate std; -#[path = "../../../stdsimd/mod.rs"] +// #[path = "../../../stdsimd/mod.rs"] mod stdsimd; pub use stdsimd::*;
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/arch/aarch64.rs b/crates/stdsimd/src/stdsimd/arch/detect/arch/aarch64.rs new file mode 100644 index 0000000..a7f6d35 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/arch/aarch64.rs
@@ -0,0 +1,102 @@ +//! Aarch64 run-time features. + +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +#[allow_internal_unstable] +macro_rules! is_aarch64_feature_detected { + ("neon") => { + // FIXME: this should be removed once we rename Aarch64 neon to asimd + cfg!(target_feature = "neon") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::asimd) + }; + ("asimd") => { + cfg!(target_feature = "neon") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::asimd) + }; + ("pmull") => { + cfg!(target_feature = "pmull") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::pmull) + }; + ("fp") => { + cfg!(target_feature = "fp") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::fp) + }; + ("fp16") => { + cfg!(target_feature = "fp16") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::fp16) + }; + ("sve") => { + cfg!(target_feature = "sve") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::sve) + }; + ("crc") => { + cfg!(target_feature = "crc") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::crc) + }; + ("crypto") => { + cfg!(target_feature = "crypto") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::crypto) + }; + ("lse") => { + cfg!(target_feature = "lse") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::lse) + }; + ("rdm") => { + cfg!(target_feature = "rdm") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::rdm) + }; + ("rcpc") => { + cfg!(target_feature = "rcpc") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::rcpc) + }; + ("dotprod") => { + cfg!(target_feature = "dotprod") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::dotprod) + }; + ("ras") => { + compile_error!("\"ras\" feature cannot be detected at run-time") + }; + ("v8.1a") => { + compile_error!("\"v8.1a\" feature cannot be detected at run-time") + }; + ("v8.2a") => { + compile_error!("\"v8.2a\" feature cannot be detected at run-time") + }; + ("v8.3a") => { + compile_error!("\"v8.3a\" feature cannot be detected at run-time") + }; + ($t:tt) => { compile_error!(concat!("unknown aarch64 target feature: ", $t)) }; +} + +/// ARM Aarch64 CPU Feature enum. Each variant denotes a position in a bitset +/// for a particular feature. +/// +/// PLEASE: do not use this, it is an implementation detail subject to change. +#[doc(hidden)] +#[allow(non_camel_case_types)] +#[repr(u8)] +#[unstable(feature = "stdsimd_internal", issue = "0")] +pub enum Feature { + /// ARM Advanced SIMD (ASIMD) + asimd, + /// Polynomial Multiply + pmull, + /// Floating point support + fp, + /// Half-float support. + fp16, + /// Scalable Vector Extension (SVE) + sve, + /// CRC32 (Cyclic Redundancy Check) + crc, + /// Crypto: AES + PMULL + SHA1 + SHA2 + crypto, + /// Atomics (Large System Extension) + lse, + /// Rounding Double Multiply (ASIMDRDM) + rdm, + /// Release consistent Processor consistent (RcPc) + rcpc, + /// Vector Dot-Product (ASIMDDP) + dotprod, +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/arch/arm.rs b/crates/stdsimd/src/stdsimd/arch/detect/arch/arm.rs new file mode 100644 index 0000000..8a5048a --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/arch/arm.rs
@@ -0,0 +1,35 @@ +//! Run-time feature detection on ARM Aarch32. + +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +#[allow_internal_unstable] +macro_rules! is_arm_feature_detected { + ("neon") => { + cfg!(target_feature = "neon") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::neon) + }; + ("pmull") => { + cfg!(target_feature = "pmull") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::pmull) + }; + ("v7") => { compile_error!("\"v7\" feature cannot be detected at run-time") }; + ("vfp2") => { compile_error!("\"vfp2\" feature cannot be detected at run-time") }; + ("vfp3") => { compile_error!("\"vfp3\" feature cannot be detected at run-time") }; + ("vfp4") => { compile_error!("\"vfp4\" feature cannot be detected at run-time") }; + ($t:tt) => { compile_error!(concat!("unknown arm target feature: ", $t)) }; +} + +/// ARM CPU Feature enum. Each variant denotes a position in a bitset for a +/// particular feature. +/// +/// PLEASE: do not use this, it is an implementation detail subject to change. +#[doc(hidden)] +#[allow(non_camel_case_types)] +#[repr(u8)] +#[unstable(feature = "stdsimd_internal", issue = "0")] +pub enum Feature { + /// ARM Advanced SIMD (NEON) - Aarch32 + neon, + /// Polynomial Multiply + pmull, +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/arch/mips.rs b/crates/stdsimd/src/stdsimd/arch/detect/arch/mips.rs new file mode 100644 index 0000000..58c9e4c --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/arch/mips.rs
@@ -0,0 +1,25 @@ +//! Run-time feature detection on MIPS. + +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +#[allow_internal_unstable] +macro_rules! is_mips_feature_detected { + ("msa") => { + cfg!(target_feature = "msa") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::msa) + }; + ($t:tt) => { compile_error!(concat!("unknown mips target feature: ", $t)) }; +} + +/// MIPS CPU Feature enum. Each variant denotes a position in a bitset for a +/// particular feature. +/// +/// PLEASE: do not use this, it is an implementation detail subject to change. +#[doc(hidden)] +#[allow(non_camel_case_types)] +#[repr(u8)] +#[unstable(feature = "stdsimd_internal", issue = "0")] +pub enum Feature { + /// MIPS SIMD Architecture (MSA) + msa, +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/arch/mips64.rs b/crates/stdsimd/src/stdsimd/arch/detect/arch/mips64.rs new file mode 100644 index 0000000..de77659 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/arch/mips64.rs
@@ -0,0 +1,25 @@ +//! Run-time feature detection on MIPS64. + +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +#[allow_internal_unstable] +macro_rules! is_mips64_feature_detected { + ("msa") => { + cfg!(target_feature = "msa") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::msa) + }; + ($t:tt) => { compile_error!(concat!("unknown mips64 target feature: ", $t)) }; +} + +/// MIPS64 CPU Feature enum. Each variant denotes a position in a bitset +/// for a particular feature. +/// +/// PLEASE: do not use this, it is an implementation detail subject to change. +#[doc(hidden)] +#[allow(non_camel_case_types)] +#[repr(u8)] +#[unstable(feature = "stdsimd_internal", issue = "0")] +pub enum Feature { + /// MIPS SIMD Architecture (MSA) + msa, +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/arch/powerpc.rs b/crates/stdsimd/src/stdsimd/arch/detect/arch/powerpc.rs new file mode 100644 index 0000000..e59f4a8 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/arch/powerpc.rs
@@ -0,0 +1,38 @@ +//! Run-time feature detection on PowerPC. + +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +#[allow_internal_unstable] +macro_rules! is_powerpc_feature_detected { + ("altivec") => { + cfg!(target_feature = "altivec") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::altivec) + }; + ("vsx") => { + cfg!(target_feature = "vsx") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::vsx) + }; + ("power8") => { + cfg!(target_feature = "power8") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::power8) + }; + ($t:tt) => { compile_error!(concat!("unknown powerpc target feature: ", $t)) }; +} + + +/// PowerPC CPU Feature enum. Each variant denotes a position in a bitset +/// for a particular feature. +/// +/// PLEASE: do not use this, it is an implementation detail subject to change. +#[doc(hidden)] +#[allow(non_camel_case_types)] +#[repr(u8)] +#[unstable(feature = "stdsimd_internal", issue = "0")] +pub enum Feature { + /// Altivec + altivec, + /// VSX + vsx, + /// Power8 + power8, +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/arch/powerpc64.rs b/crates/stdsimd/src/stdsimd/arch/detect/arch/powerpc64.rs new file mode 100644 index 0000000..2695f6b --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/arch/powerpc64.rs
@@ -0,0 +1,38 @@ +//! Run-time feature detection on PowerPC64. + +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +#[allow_internal_unstable] +macro_rules! is_powerpc64_feature_detected { + ("altivec") => { + cfg!(target_feature = "altivec") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::altivec) + }; + ("vsx") => { + cfg!(target_feature = "vsx") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::vsx) + }; + ("power8") => { + cfg!(target_feature = "power8") || + $crate::arch::detect::check_for($crate::arch::detect::Feature::power8) + }; + ($t:tt) => { compile_error!(concat!("unknown powerpc64 target feature: ", $t)) }; +} + + +/// PowerPC64 CPU Feature enum. Each variant denotes a position in a bitset +/// for a particular feature. +/// +/// PLEASE: do not use this, it is an implementation detail subject to change. +#[doc(hidden)] +#[allow(non_camel_case_types)] +#[repr(u8)] +#[unstable(feature = "stdsimd_internal", issue = "0")] +pub enum Feature { + /// Altivec + altivec, + /// VSX + vsx, + /// Power8 + power8, +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/arch/x86.rs b/crates/stdsimd/src/stdsimd/arch/detect/arch/x86.rs new file mode 100644 index 0000000..5faaff3 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/arch/x86.rs
@@ -0,0 +1,319 @@ +//! This module implements minimal run-time feature detection for x86. +//! +//! The features are detected using the `detect_features` function below. +//! This function uses the CPUID instruction to read the feature flags from the +//! CPU and encodes them in an `usize` where each bit position represents +//! whether a feature is available (bit is set) or unavaiable (bit is cleared). +//! +//! The enum `Feature` is used to map bit positions to feature names, and the +//! the `__crate::arch::detect::check_for!` macro is used to map string literals (e.g. +//! "avx") to these bit positions (e.g. `Feature::avx`). +//! +//! +//! The run-time feature detection is performed by the +//! `__crate::arch::detect::check_for(Feature) -> bool` function. On its first call, +//! this functions queries the CPU for the available features and stores them +//! in a global `AtomicUsize` variable. The query is performed by just checking +//! whether the feature bit in this global variable is set or cleared. + +/// A macro to test at *runtime* whether a CPU feature is available on +/// x86/x86-64 platforms. +/// +/// This macro is provided in the standard library and will detect at runtime +/// whether the specified CPU feature is detected. This does *not* resolve at +/// compile time unless the specified feature is already enabled for the entire +/// crate. Runtime detection currently relies mostly on the `cpuid` instruction. +/// +/// This macro only takes one argument which is a string literal of the feature +/// being tested for. The feature names supported are the lowercase versions of +/// the ones defined by Intel in [their documentation][docs]. +/// +/// ## Supported arguments +/// +/// This macro supports the same names that `#[target_feature]` supports. Unlike +/// `#[target_feature]`, however, this macro does not support names separated +/// with a comma. Instead testing for multiple features must be done through +/// separate macro invocations for now. +/// +/// Supported arguments are: +/// +/// * `"aes"` +/// * `"pclmulqdq"` +/// * `"rdrand"` +/// * `"rdseed"` +/// * `"tsc"` +/// * `"mmx"` +/// * `"sse"` +/// * `"sse2"` +/// * `"sse3"` +/// * `"ssse3"` +/// * `"sse4.1"` +/// * `"sse4.2"` +/// * `"sse4a"` +/// * `"sha"` +/// * `"avx"` +/// * `"avx2"` +/// * `"avx512f"` +/// * `"avx512cd"` +/// * `"avx512er"` +/// * `"avx512pf"` +/// * `"avx512bw"` +/// * `"avx512dq"` +/// * `"avx512vl"` +/// * `"avx512ifma"` +/// * `"avx512vbmi"` +/// * `"avx512vpopcntdq"` +/// * `"fma"` +/// * `"bmi1"` +/// * `"bmi2"` +/// * `"abm"` +/// * `"lzcnt"` +/// * `"tbm"` +/// * `"popcnt"` +/// * `"fxsr"` +/// * `"xsave"` +/// * `"xsaveopt"` +/// * `"xsaves"` +/// * `"xsavec"` +/// +/// [docs]: https://software.intel.com/sites/landingpage/IntrinsicsGuide +#[macro_export] +#[stable(feature = "simd_x86", since = "1.27.0")] +#[allow_internal_unstable] +macro_rules! is_x86_feature_detected { + ("aes") => { + cfg!(target_feature = "aes") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::aes) }; + ("pclmulqdq") => { + cfg!(target_feature = "pclmulqdq") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::pclmulqdq) }; + ("rdrand") => { + cfg!(target_feature = "rdrand") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::rdrand) }; + ("rdseed") => { + cfg!(target_feature = "rdseed") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::rdseed) }; + ("tsc") => { + cfg!(target_feature = "tsc") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::tsc) }; + ("mmx") => { + cfg!(target_feature = "mmx") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::mmx) }; + ("sse") => { + cfg!(target_feature = "sse") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::sse) }; + ("sse2") => { + cfg!(target_feature = "sse2") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::sse2) + }; + ("sse3") => { + cfg!(target_feature = "sse3") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::sse3) + }; + ("ssse3") => { + cfg!(target_feature = "ssse3") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::ssse3) + }; + ("sse4.1") => { + cfg!(target_feature = "sse4.1") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::sse4_1) + }; + ("sse4.2") => { + cfg!(target_feature = "sse4.2") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::sse4_2) + }; + ("sse4a") => { + cfg!(target_feature = "sse4a") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::sse4a) + }; + ("sha") => { + cfg!(target_feature = "sha") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::sha) + }; + ("avx") => { + cfg!(target_feature = "avx") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx) + }; + ("avx2") => { + cfg!(target_feature = "avx2") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx2) + }; + ("avx512f") => { + cfg!(target_feature = "avx512f") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512f) + }; + ("avx512cd") => { + cfg!(target_feature = "avx512cd") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512cd) + }; + ("avx512er") => { + cfg!(target_feature = "avx512er") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512er) + }; + ("avx512pf") => { + cfg!(target_feature = "avx512pf") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512pf) + }; + ("avx512bw") => { + cfg!(target_feature = "avx512bw") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512bw) + }; + ("avx512dq") => { + cfg!(target_feature = "avx512dq") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512dq) + }; + ("avx512vl") => { + cfg!(target_Feature = "avx512vl") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512vl) + }; + ("avx512ifma") => { + cfg!(target_feature = "avx512ifma") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512_ifma) + }; + ("avx512vbmi") => { + cfg!(target_feature = "avx512vbmi") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512_vbmi) + }; + ("avx512vpopcntdq") => { + cfg!(target_feature = "avx512vpopcntdq") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::avx512_vpopcntdq) + }; + ("fma") => { + cfg!(target_feature = "fma") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::fma) + }; + ("bmi1") => { + cfg!(target_feature = "bmi1") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::bmi) + }; + ("bmi2") => { + cfg!(target_feature = "bmi2") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::bmi2) + }; + ("abm") => { + cfg!(target_feature = "abm") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::abm) + }; + ("lzcnt") => { + cfg!(target_feature = "lzcnt") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::abm) + }; + ("tbm") => { + cfg!(target_feature = "tbm") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::tbm) + }; + ("popcnt") => { + cfg!(target_feature = "popcnt") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::popcnt) + }; + ("fxsr") => { + cfg!(target_feature = "fxsr") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::fxsr) + }; + ("xsave") => { + cfg!(target_feature = "xsave") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::xsave) + }; + ("xsaveopt") => { + cfg!(target_feature = "xsaveopt") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::xsaveopt) + }; + ("xsaves") => { + cfg!(target_feature = "xsaves") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::xsaves) + }; + ("xsavec") => { + cfg!(target_feature = "xsavec") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::xsavec) + }; + ($t:tt) => { + compile_error!(concat!("unknown target feature: ", $t)) + }; +} + +/// X86 CPU Feature enum. Each variant denotes a position in a bitset for a +/// particular feature. +/// +/// This is an unstable implementation detail subject to change. +#[allow(non_camel_case_types)] +#[repr(u8)] +#[doc(hidden)] +#[unstable(feature = "stdsimd_internal", issue = "0")] +pub enum Feature { + /// AES (Advanced Encryption Standard New Instructions AES-NI) + aes, + /// CLMUL (Carry-less Multiplication) + pclmulqdq, + /// RDRAND + rdrand, + /// RDSEED + rdseed, + /// TSC (Time Stamp Counter) + tsc, + /// MMX + mmx, + /// SSE (Streaming SIMD Extensions) + sse, + /// SSE2 (Streaming SIMD Extensions 2) + sse2, + /// SSE3 (Streaming SIMD Extensions 3) + sse3, + /// SSSE3 (Supplemental Streaming SIMD Extensions 3) + ssse3, + /// SSE4.1 (Streaming SIMD Extensions 4.1) + sse4_1, + /// SSE4.2 (Streaming SIMD Extensions 4.2) + sse4_2, + /// SSE4a (Streaming SIMD Extensions 4a) + sse4a, + /// SHA + sha, + /// AVX (Advanced Vector Extensions) + avx, + /// AVX2 (Advanced Vector Extensions 2) + avx2, + /// AVX-512 F (Foundation) + avx512f, + /// AVX-512 CD (Conflict Detection Instructions) + avx512cd, + /// AVX-512 ER (Exponential and Reciprocal Instructions) + avx512er, + /// AVX-512 PF (Prefetch Instructions) + avx512pf, + /// AVX-512 BW (Byte and Word Instructions) + avx512bw, + /// AVX-512 DQ (Doubleword and Quadword) + avx512dq, + /// AVX-512 VL (Vector Length Extensions) + avx512vl, + /// AVX-512 IFMA (Integer Fused Multiply Add) + avx512_ifma, + /// AVX-512 VBMI (Vector Byte Manipulation Instructions) + avx512_vbmi, + /// AVX-512 VPOPCNTDQ (Vector Population Count Doubleword and + /// Quadword) + avx512_vpopcntdq, + /// FMA (Fused Multiply Add) + fma, + /// BMI1 (Bit Manipulation Instructions 1) + bmi, + /// BMI1 (Bit Manipulation Instructions 2) + bmi2, + /// ABM (Advanced Bit Manipulation) on AMD / LZCNT (Leading Zero + /// Count) on Intel + abm, + /// TBM (Trailing Bit Manipulation) + tbm, + /// POPCNT (Population Count) + popcnt, + /// FXSR (Floating-point context fast save and restor) + fxsr, + /// XSAVE (Save Processor Extended States) + xsave, + /// XSAVEOPT (Save Processor Extended States Optimized) + xsaveopt, + /// XSAVES (Save Processor Extended States Supervisor) + xsaves, + /// XSAVEC (Save Processor Extended States Compacted) + xsavec, +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/bit.rs b/crates/stdsimd/src/stdsimd/arch/detect/bit.rs new file mode 100644 index 0000000..578f0b1 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/bit.rs
@@ -0,0 +1,9 @@ +//! Bit manipulation utilities. + +/// Tests the `bit` of `x`. +#[allow(dead_code)] +#[inline] +pub(crate) fn test(x: usize, bit: u32) -> bool { + debug_assert!(bit < 32, "bit index out-of-bounds"); + x & (1 << bit) != 0 +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/cache.rs b/crates/stdsimd/src/stdsimd/arch/detect/cache.rs new file mode 100644 index 0000000..ead91ad --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/cache.rs
@@ -0,0 +1,161 @@ +//! Caches run-time feature detection so that it only needs to be computed +//! once. + +#![allow(dead_code)] // not used on all platforms + +use core::sync::atomic::Ordering; + +#[cfg(target_pointer_width = "64")] +use core::sync::atomic::AtomicU64; + +#[cfg(target_pointer_width = "32")] +use core::sync::atomic::AtomicU32; + +/// Sets the `bit` of `x`. +#[inline] +const fn set_bit(x: u64, bit: u32) -> u64 { + x | 1 << bit +} + +/// Tests the `bit` of `x`. +#[inline] +const fn test_bit(x: u64, bit: u32) -> bool { + x & (1 << bit) != 0 +} + +/// Maximum number of features that can be cached. +const CACHE_CAPACITY: u32 = 63; + +/// This type is used to initialize the cache +#[derive(Copy, Clone)] +pub(crate) struct Initializer(u64); + +impl Default for Initializer { + fn default() -> Self { + Initializer(0) + } +} + +impl Initializer { + /// Tests the `bit` of the cache. + #[allow(dead_code)] + #[inline] + pub(crate) fn test(&self, bit: u32) -> bool { + // FIXME: this way of making sure that the cache is large enough is + // brittle. + debug_assert!( + bit < CACHE_CAPACITY, + "too many features, time to increase the cache size!" + ); + test_bit(self.0, bit) + } + + /// Sets the `bit` of the cache. + #[inline] + pub(crate) fn set(&mut self, bit: u32) { + // FIXME: this way of making sure that the cache is large enough is + // brittle. + debug_assert!( + bit < CACHE_CAPACITY, + "too many features, time to increase the cache size!" + ); + let v = self.0; + self.0 = set_bit(v, bit); + } +} + +/// This global variable is a cache of the features supported by the CPU. +static CACHE: Cache = Cache::uninitialized(); + +/// Feature cache with capacity for `CACHE_CAPACITY` features. +/// +/// Note: the last feature bit is used to represent an +/// uninitialized cache. +#[cfg(target_pointer_width = "64")] +struct Cache(AtomicU64); + +#[cfg(target_pointer_width = "64")] +impl Cache { + /// Creates an uninitialized cache. + const fn uninitialized() -> Self { + Cache(AtomicU64::new(u64::max_value())) + } + /// Is the cache uninitialized? + #[inline] + pub(crate) fn is_uninitialized(&self) -> bool { + self.0.load(Ordering::Relaxed) == u64::max_value() + } + + /// Is the `bit` in the cache set? + #[inline] + pub(crate) fn test(&self, bit: u32) -> bool { + test_bit(CACHE.0.load(Ordering::Relaxed), bit) + } + + /// Initializes the cache. + #[inline] + pub(crate) fn initialize(&self, value: Initializer) { + self.0.store(value.0, Ordering::Relaxed); + } +} + +/// Feature cache with capacity for `CACHE_CAPACITY` features. +/// +/// Note: the last feature bit is used to represent an +/// uninitialized cache. +#[cfg(target_pointer_width = "32")] +struct Cache(AtomicU32, AtomicU32); + +#[cfg(target_pointer_width = "32")] +impl Cache { + /// Creates an uninitialized cache. + const fn uninitialized() -> Self { + Cache( + AtomicU32::new(u32::max_value()), + AtomicU32::new(u32::max_value()), + ) + } + /// Is the cache uninitialized? + #[inline] + pub(crate) fn is_uninitialized(&self) -> bool { + self.1.load(Ordering::Relaxed) == u32::max_value() + } + + /// Is the `bit` in the cache set? + #[inline] + pub(crate) fn test(&self, bit: u32) -> bool { + if bit < 32 { + test_bit(CACHE.0.load(Ordering::Relaxed) as u64, bit) + } else { + test_bit(CACHE.1.load(Ordering::Relaxed) as u64, bit - 32) + } + } + + /// Initializes the cache. + #[inline] + pub(crate) fn initialize(&self, value: Initializer) { + let lo: u32 = value.0 as u32; + let hi: u32 = (value.0 >> 32) as u32; + self.0.store(lo, Ordering::Relaxed); + self.1.store(hi, Ordering::Relaxed); + } +} + +/// Test the `bit` of the storage. If the storage has not been initialized, +/// initializes it with the result of `f()`. +/// +/// On its first invocation, it detects the CPU features and caches them in the +/// `CACHE` global variable as an `AtomicU64`. +/// +/// It uses the `Feature` variant to index into this variable as a bitset. If +/// the bit is set, the feature is enabled, and otherwise it is disabled. +#[inline] +pub(crate) fn test<F>(bit: u32, f: F) -> bool +where + F: FnOnce() -> Initializer, +{ + if CACHE.is_uninitialized() { + CACHE.initialize(f()); + } + CACHE.test(bit) +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/error_macros.rs b/crates/stdsimd/src/stdsimd/arch/detect/error_macros.rs new file mode 100644 index 0000000..743f7ea --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/error_macros.rs
@@ -0,0 +1,136 @@ +//! The `is_{target_arch}_feature_detected!` macro are only available on their +//! architecture. These macros provide a better error messages when the user +//! attempts to call them in a different architecture. + +#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +macro_rules! is_x86_feature_detected { + ($t: tt) => { + compile_error!( + r#" + is_x86_feature_detected can only be used on x86 and x86_64 targets. + You can prevent it from being used in other architectures by + guarding it behind a cfg(target_arch) as follows: + + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + if is_x86_feature_detected(...) { ... } + } + "# + ) + }; +} + +#[cfg(not(target_arch = "arm"))] +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +macro_rules! is_arm_feature_detected { + ($t:tt) => { + compile_error!( + r#" + is_arm_feature_detected can only be used on ARM targets. + You can prevent it from being used in other architectures by + guarding it behind a cfg(target_arch) as follows: + + #[cfg(target_arch = "arm")] { + if is_arm_feature_detected(...) { ... } + } + "# + ) + }; +} + +#[cfg(not(target_arch = "aarch64"))] +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +macro_rules! is_aarch64_feature_detected { + ($t: tt) => { + compile_error!( + r#" + is_aarch64_feature_detected can only be used on AArch64 targets. + You can prevent it from being used in other architectures by + guarding it behind a cfg(target_arch) as follows: + + #[cfg(target_arch = "aarch64")] { + if is_aarch64_feature_detected(...) { ... } + } + "# + ) + }; +} + +#[cfg(not(target_arch = "powerpc"))] +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +macro_rules! is_powerpc_feature_detected { + ($t:tt) => { + compile_error!( + r#" +is_powerpc_feature_detected can only be used on PowerPC targets. +You can prevent it from being used in other architectures by +guarding it behind a cfg(target_arch) as follows: + + #[cfg(target_arch = "powerpc")] { + if is_powerpc_feature_detected(...) { ... } + } +"# + ) + }; +} + +#[cfg(not(target_arch = "powerpc64"))] +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +macro_rules! is_powerpc64_feature_detected { + ($t:tt) => { + compile_error!( + r#" +is_powerpc64_feature_detected can only be used on PowerPC64 targets. +You can prevent it from being used in other architectures by +guarding it behind a cfg(target_arch) as follows: + + #[cfg(target_arch = "powerpc64")] { + if is_powerpc64_feature_detected(...) { ... } + } +"# + ) + }; +} + +#[cfg(not(target_arch = "mips"))] +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +macro_rules! is_mips_feature_detected { + ($t:tt) => { + compile_error!( + r#" + is_mips_feature_detected can only be used on MIPS targets. + You can prevent it from being used in other architectures by + guarding it behind a cfg(target_arch) as follows: + + #[cfg(target_arch = "mips")] { + if is_mips_feature_detected(...) { ... } + } + "# + ) + }; +} + +#[cfg(not(target_arch = "mips64"))] +#[macro_export] +#[unstable(feature = "stdsimd", issue = "27731")] +macro_rules! is_mips64_feature_detected { + ($t:tt) => { + compile_error!( + r#" + is_mips64_feature_detected can only be used on MIPS64 targets. + You can prevent it from being used in other architectures by + guarding it behind a cfg(target_arch) as follows: + + #[cfg(target_arch = "mips64")] { + if is_mips64_feature_detected(...) { ... } + } + "# + ) + }; +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/mod.rs b/crates/stdsimd/src/stdsimd/arch/detect/mod.rs new file mode 100644 index 0000000..57cab4f --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/mod.rs
@@ -0,0 +1,79 @@ +//! This module implements run-time feature detection. +//! +//! The `is_{arch}_feature_detected!("feature-name")` macros take the name of a +//! feature as a string-literal, and return a boolean indicating whether the +//! feature is enabled at run-time or not. +//! +//! These macros do two things: +//! * map the string-literal into an integer stored as a `Feature` enum, +//! * call a `os::check_for(x: Feature)` function that returns `true` if the +//! feature is enabled. +//! +//! The `Feature` enums are also implemented in the `arch/{target_arch}.rs` +//! modules. +//! +//! The `check_for` functions are, in general, Operating System dependent. Most +//! architectures do not allow user-space programs to query the feature bits +//! due to security concerns (x86 is the big exception). These functions are +//! implemented in the `os/{target_os}.rs` modules. + +#[macro_use] +mod error_macros; + +cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + #[path = "arch/x86.rs"] + #[macro_use] + mod arch; + } else if #[cfg(target_arch = "arm")] { + #[path = "arch/arm.rs"] + #[macro_use] + mod arch; + } else if #[cfg(target_arch = "aarch64")] { + #[path = "arch/aarch64.rs"] + #[macro_use] + mod arch; + } else if #[cfg(target_arch = "powerpc")] { + #[path = "arch/powerpc.rs"] + #[macro_use] + mod arch; + } else if #[cfg(target_arch = "powerpc64")] { + #[path = "arch/powerpc64.rs"] + #[macro_use] + mod arch; + } else if #[cfg(target_arch = "mips")] { + #[path = "arch/mips.rs"] + #[macro_use] + mod arch; + } else if #[cfg(target_arch = "mips64")] { + #[path = "arch/mips64.rs"] + #[macro_use] + mod arch; + } else { + // Unimplemented architecture: + mod arch { + pub enum Feature { + Null + } + } + } +} +pub use self::arch::Feature; + +mod bit; +mod cache; + +cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + // On x86/x86_64 no OS specific functionality is required. + #[path = "os/x86.rs"] + mod os; + } else if #[cfg(target_os = "linux")] { + #[path = "os/linux/mod.rs"] + mod os; + } else { + #[path = "os/other.rs"] + mod os; + } +} +pub use self::os::check_for;
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/linux/aarch64.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/aarch64.rs new file mode 100644 index 0000000..ebddf2d --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/aarch64.rs
@@ -0,0 +1,160 @@ +//! Run-time feature detection for Aarch64 on Linux. + +use arch::detect::Feature; +use arch::detect::cache; +use arch::detect::bit; +use super::auxvec; +use super::cpuinfo; + +/// Performs run-time feature detection. +#[inline] +pub fn check_for(x: Feature) -> bool { + cache::test(x as u32, detect_features) +} + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from /proc/cpuinfo. +fn detect_features() -> cache::Initializer { + if let Ok(auxv) = auxvec::auxv() { + let hwcap: AtHwcap = auxv.into(); + return hwcap.cache(); + } + if let Ok(c) = cpuinfo::CpuInfo::new() { + let hwcap: AtHwcap = c.into(); + return hwcap.cache(); + } + cache::Initializer::default() +} + +/// These values are part of the platform-specific [asm/hwcap.h][hwcap] . +/// +/// [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h +struct AtHwcap { + fp: bool, // 0 + asimd: bool, // 1 + // evtstrm: bool, // 2 + aes: bool, // 3 + pmull: bool, // 4 + sha1: bool, // 5 + sha2: bool, // 6 + crc32: bool, // 7 + atomics: bool, // 8 + fphp: bool, // 9 + asimdhp: bool, // 10 + // cpuid: bool, // 11 + asimdrdm: bool, // 12 + // jscvt: bool, // 13 + // fcma: bool, // 14 + lrcpc: bool, // 15 + // dcpop: bool, // 16 + // sha3: bool, // 17 + // sm3: bool, // 18 + // sm4: bool, // 19 + asimddp: bool, // 20 + // sha512: bool, // 21 + sve: bool, // 22 +} + +impl From<auxvec::AuxVec> for AtHwcap { + /// Reads AtHwcap from the auxiliary vector. + fn from(auxv: auxvec::AuxVec) -> Self { + AtHwcap { + fp: bit::test(auxv.hwcap, 0), + asimd: bit::test(auxv.hwcap, 1), + // evtstrm: bit::test(auxv.hwcap, 2), + aes: bit::test(auxv.hwcap, 3), + pmull: bit::test(auxv.hwcap, 4), + sha1: bit::test(auxv.hwcap, 5), + sha2: bit::test(auxv.hwcap, 6), + crc32: bit::test(auxv.hwcap, 7), + atomics: bit::test(auxv.hwcap, 8), + fphp: bit::test(auxv.hwcap, 9), + asimdhp: bit::test(auxv.hwcap, 10), + // cpuid: bit::test(auxv.hwcap, 11), + asimdrdm: bit::test(auxv.hwcap, 12), + // jscvt: bit::test(auxv.hwcap, 13), + // fcma: bit::test(auxv.hwcap, 14), + lrcpc: bit::test(auxv.hwcap, 15), + // dcpop: bit::test(auxv.hwcap, 16), + // sha3: bit::test(auxv.hwcap, 17), + // sm3: bit::test(auxv.hwcap, 18), + // sm4: bit::test(auxv.hwcap, 19), + asimddp: bit::test(auxv.hwcap, 20), + // sha512: bit::test(auxv.hwcap, 21), + sve: bit::test(auxv.hwcap, 22), + } + } +} + +impl From<cpuinfo::CpuInfo> for AtHwcap { + /// Reads AtHwcap from /proc/cpuinfo . + fn from(c: cpuinfo::CpuInfo) -> Self { + let f = &c.field("Features"); + AtHwcap { + // 64-bit names. FIXME: In 32-bit compatibility mode /proc/cpuinfo will + // map some of the 64-bit names to some 32-bit feature names. This does not + // cover that yet. + fp: f.has("fp"), + asimd: f.has("asimd"), + // evtstrm: f.has("evtstrm"), + aes: f.has("aes"), + pmull: f.has("pmull"), + sha1: f.has("sha1"), + sha2: f.has("sha2"), + crc32: f.has("crc32"), + atomics: f.has("atomics"), + fphp: f.has("fphp"), + asimdhp: f.has("asimdhp"), + // cpuid: f.has("cpuid"), + asimdrdm: f.has("asimdrdm"), + // jscvt: f.has("jscvt"), + // fcma: f.has("fcma"), + lrcpc: f.has("lrcpc"), + // dcpop: f.has("dcpop"), + // sha3: f.has("sha3"), + // sm3: f.has("sm3"), + // sm4: f.has("sm4"), + asimddp: f.has("asimddp"), + // sha512: f.has("sha512"), + sve: f.has("sve"), + } + } +} + +impl AtHwcap { + /// Initializes the cache from the feature -bits. + /// + /// The features are enabled approximately like in LLVM host feature detection: + /// https://github.com/llvm-mirror/llvm/blob/master/lib/Support/Host.cpp#L1273 + fn cache(self) -> cache::Initializer { + let mut value = cache::Initializer::default(); + { + let mut enable_feature = |f, enable| { + if enable { + value.set(f as u32); + } + }; + + enable_feature(Feature::fp, self.fp); + // Half-float support requires float support + enable_feature(Feature::fp16, self.fp && self.fphp); + enable_feature(Feature::pmull, self.pmull); + enable_feature(Feature::crc, self.crc32); + enable_feature(Feature::lse, self.atomics); + enable_feature(Feature::rcpc, self.lrcpc); + + // SIMD support requires float support - if half-floats are + // supported, it also requires half-float support: + let asimd = self.fp && self.asimd && (!self.fphp | self.asimdhp); + enable_feature(Feature::asimd, asimd); + // SIMD extensions require SIMD support: + enable_feature(Feature::rdm, self.asimdrdm && asimd); + enable_feature(Feature::dotprod, self.asimddp && asimd); + enable_feature(Feature::sve, self.sve && asimd); + + // Crypto is specified as AES + PMULL + SHA1 + SHA2 per LLVM/hosts.cpp + enable_feature(Feature::crypto, self.aes && self.pmull && self.sha1 && self.sha2); + } + value + } +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/linux/arm.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/arm.rs new file mode 100644 index 0000000..9d265e0 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/arm.rs
@@ -0,0 +1,52 @@ +//! Run-time feature detection for ARM on Linux. + +use arch::detect::Feature; +use arch::detect::cache; +use arch::detect::bit; +use super::auxvec; +use super::cpuinfo; + +/// Performs run-time feature detection. +#[inline] +pub fn check_for(x: Feature) -> bool { + cache::test(x as u32, detect_features) +} + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from /proc/cpuinfo. +fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + // The values are part of the platform-specific [asm/hwcap.h][hwcap] + // + // [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h + if let Ok(auxv) = auxvec::auxv() { + enable_feature(&mut value, Feature::neon, bit::test(auxv.hwcap, 12)); + enable_feature(&mut value, Feature::pmull, bit::test(auxv.hwcap2, 1)); + return value; + } + + if let Ok(c) = cpuinfo::CpuInfo::new() { + enable_feature(&mut value, Feature::neon, c.field("Features").has("neon") && + !has_broken_neon(&c)); + enable_feature(&mut value, Feature::pmull, c.field("Features").has("pmull")); + return value; + } + value +} + +/// Is the CPU known to have a broken NEON unit? +/// +/// See https://crbug.com/341598. +fn has_broken_neon(cpuinfo: &cpuinfo::CpuInfo) -> bool { + cpuinfo.field("CPU implementer") == "0x51" + && cpuinfo.field("CPU architecture") == "7" + && cpuinfo.field("CPU variant") == "0x1" + && cpuinfo.field("CPU part") == "0x04d" + && cpuinfo.field("CPU revision") == "0" +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/linux/auxvec.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/auxvec.rs new file mode 100644 index 0000000..20fbb5f --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/auxvec.rs
@@ -0,0 +1,270 @@ +//! Parses ELF auxiliary vectors. +#![cfg_attr(not(target_arch = "aarch64"), allow(dead_code))] + +use mem; +use prelude::v1::*; +use fs::File; +use io::Read; + +/// Key to access the CPU Hardware capabilities bitfield. +pub(crate) const AT_HWCAP: usize = 16; +/// Key to access the CPU Hardware capabilities 2 bitfield. +#[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] +pub(crate) const AT_HWCAP2: usize = 26; + +/// Cache HWCAP bitfields of the ELF Auxiliary Vector. +/// +/// If an entry cannot be read all the bits in the bitfield are set to zero. +/// This should be interpreted as all the features being disabled. +#[derive(Debug, Copy, Clone)] +pub(crate) struct AuxVec { + pub hwcap: usize, + #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + pub hwcap2: usize, +} + +/// ELF Auxiliary Vector +/// +/// The auxiliary vector is a memory region in a running ELF program's stack +/// composed of (key: usize, value: usize) pairs. +/// +/// The keys used in the aux vector are platform dependent. For Linux, they are +/// defined in [linux/auxvec.h][auxvec_h]. The hardware capabilities of a given +/// CPU can be queried with the `AT_HWCAP` and `AT_HWCAP2` keys. +/// +/// There is no perfect way of reading the auxiliary vector. +/// +/// - If the `getauxval` is dynamically linked to this binary, it will be used. +/// - Otherwise, try to read `/proc/self/auxv`. +/// - If that fails, this function returns an error. +/// +/// Note that run-time feature detection is not invoked for features that can +/// be detected at compile-time. Also note that if this function returns an +/// error, cpuinfo still can (and will) be used to try to perform run-time +/// feature detecton on some platforms. +/// +/// For more information about when `getauxval` is available check the great +/// [`auxv` crate documentation][auxv_docs]. +/// +/// [auxvec_h]: https://github.com/torvalds/linux/blob/master/include/uapi/linux/auxvec.h +/// [auxv_docs]: https://docs.rs/auxv/0.3.3/auxv/ +pub(crate) fn auxv() -> Result<AuxVec, ()> { + // Try to call a dynamically-linked getauxval function. + if let Ok(hwcap) = getauxval(AT_HWCAP) { + // Targets with only AT_HWCAP: + #[cfg(any(target_arch = "aarch64", target_arch = "mips", + target_arch = "mips64"))] + { + if hwcap != 0 { + return Ok(AuxVec { hwcap }); + } + } + + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + { + if let Ok(hwcap2) = getauxval(AT_HWCAP2) { + if hwcap != 0 && hwcap2 != 0 { + return Ok(AuxVec { hwcap, hwcap2 }); + } + } + } + drop(hwcap); + } + // If calling getauxval fails, try to read the auxiliary vector from + // its file: + auxv_from_file("/proc/self/auxv") +} + +/// Tries to read the `key` from the auxiliary vector by calling the +/// dynamically-linked `getauxval` function. If the function is not linked, +/// this function return `Err`. +fn getauxval(key: usize) -> Result<usize, ()> { + use libc; + pub type F = unsafe extern "C" fn(usize) -> usize; + unsafe { + let ptr = libc::dlsym( + libc::RTLD_DEFAULT, + "getauxval\0".as_ptr() as *const _, + ); + if ptr.is_null() { + return Err(()); + } + + let ffi_getauxval: F = mem::transmute(ptr); + Ok(ffi_getauxval(key)) + } +} + +/// Tries to read the auxiliary vector from the `file`. If this fails, this +/// function returns `Err`. +fn auxv_from_file(file: &str) -> Result<AuxVec, ()> { + let mut file = File::open(file).map_err(|_| ())?; + + // See https://github.com/torvalds/linux/blob/v3.19/include/uapi/linux/auxvec.h + // + // The auxiliary vector contains at most 32 (key,value) fields: from + // `AT_EXECFN = 31` to `AT_NULL = 0`. That is, a buffer of + // 2*32 `usize` elements is enough to read the whole vector. + let mut buf = [0_usize; 64]; + { + let raw: &mut [u8; 64 * mem::size_of::<usize>()] = + unsafe { mem::transmute(&mut buf) }; + file.read(raw).map_err(|_| ())?; + } + auxv_from_buf(&buf) +} + +/// Tries to interpret the `buffer` as an auxiliary vector. If that fails, this +/// function returns `Err`. +fn auxv_from_buf(buf: &[usize; 64]) -> Result<AuxVec, ()> { + // Targets with only AT_HWCAP: + #[cfg(any(target_arch = "aarch64", target_arch = "mips", + target_arch = "mips64"))] + { + for el in buf.chunks(2) { + match el[0] { + AT_HWCAP => return Ok(AuxVec { hwcap: el[1] }), + _ => (), + } + } + } + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + { + let mut hwcap = None; + let mut hwcap2 = None; + for el in buf.chunks(2) { + match el[0] { + AT_HWCAP => hwcap = Some(el[1]), + AT_HWCAP2 => hwcap2 = Some(el[1]), + _ => (), + } + } + + if let (Some(hwcap), Some(hwcap2)) = (hwcap, hwcap2) { + return Ok(AuxVec { hwcap, hwcap2 }); + } + } + drop(buf); + Err(()) +} + +#[cfg(test)] +mod tests { + extern crate auxv as auxv_crate; + use super::*; + + // Reads the Auxiliary Vector key from /proc/self/auxv + // using the auxv crate. + fn auxv_crate_getprocfs(key: usize) -> Option<usize> { + use self::auxv_crate::AuxvType; + use self::auxv_crate::procfs::search_procfs_auxv; + let k = key as AuxvType; + match search_procfs_auxv(&[k]) { + Ok(v) => Some(v[&k] as usize), + Err(_) => None, + } + } + + // Reads the Auxiliary Vector key from getauxval() + // using the auxv crate. + #[cfg(not(any(target_arch = "mips", target_arch = "mips64")))] + fn auxv_crate_getauxval(key: usize) -> Option<usize> { + use self::auxv_crate::AuxvType; + use self::auxv_crate::getauxval::Getauxval; + let q = auxv_crate::getauxval::NativeGetauxval {}; + match q.getauxval(key as AuxvType) { + Ok(v) => Some(v as usize), + Err(_) => None, + } + } + + // FIXME: on mips/mips64 getauxval returns 0, and /proc/self/auxv + // does not always contain the AT_HWCAP key under qemu. + #[cfg(not(any(target_arch = "mips", target_arch = "mips64", target_arch = "powerpc")))] + #[test] + fn auxv_crate() { + let v = auxv(); + if let Some(hwcap) = auxv_crate_getauxval(AT_HWCAP) { + let rt_hwcap = v.expect("failed to find hwcap key").hwcap; + assert_eq!(rt_hwcap, hwcap); + } + + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + { + if let Some(hwcap2) = auxv_crate_getauxval(AT_HWCAP2) { + let rt_hwcap2 = v.expect("failed to find hwcap2 key").hwcap2; + assert_eq!(rt_hwcap2, hwcap2); + } + } + } + + #[test] + fn auxv_dump() { + if let Ok(auxvec) = auxv() { + println!("{:?}", auxvec); + } else { + println!("both getauxval() and reading /proc/self/auxv failed!"); + } + } + + cfg_if! { + if #[cfg(target_arch = "arm")] { + #[test] + fn linux_rpi3() { + let v = auxv_from_file( + "../../stdsimd/arch/detect/test_data/linux-rpi3.auxv", + ).unwrap(); + assert_eq!(v.hwcap, 4174038); + assert_eq!(v.hwcap2, 16); + } + + #[test] + #[should_panic] + fn linux_macos_vb() { + let _ = auxv_from_file( + "../../stdsimd/arch/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv" + ).unwrap(); + // this file is incomplete (contains hwcap but not hwcap2), we + // want to fall back to /proc/cpuinfo in this case, so + // reading should fail. assert_eq!(v.hwcap, 126614527); + // assert_eq!(v.hwcap2, 0); + } + } else if #[cfg(target_arch = "aarch64")] { + #[test] + fn linux_x64() { + let v = auxv_from_file( + "../../stdsimd/arch/detect/test_data/linux-x64-i7-6850k.auxv", + ).unwrap(); + assert_eq!(v.hwcap, 3219913727); + } + } + } + + #[test] + fn auxv_dump_procfs() { + if let Ok(auxvec) = auxv_from_file("/proc/self/auxv") { + println!("{:?}", auxvec); + } else { + println!("reading /proc/self/auxv failed!"); + } + } + + #[test] + fn auxv_crate_procfs() { + let v = auxv(); + if let Some(hwcap) = auxv_crate_getprocfs(AT_HWCAP) { + assert_eq!(v.unwrap().hwcap, hwcap); + } + + // Targets with AT_HWCAP and AT_HWCAP2: + #[cfg(any(target_arch = "arm", target_arch = "powerpc64"))] + { + if let Some(hwcap2) = auxv_crate_getprocfs(AT_HWCAP2) { + assert_eq!(v.unwrap().hwcap2, hwcap2); + } + } + } +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/linux/cpuinfo.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/cpuinfo.rs new file mode 100644 index 0000000..fd7b587 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/cpuinfo.rs
@@ -0,0 +1,302 @@ +//! Parses /proc/cpuinfo +#![cfg_attr(not(target_arch = "arm"), allow(dead_code))] + +use prelude::v1::*; +use fs::File; +use io::{self, Read}; + +/// cpuinfo +pub(crate) struct CpuInfo { + raw: String, +} + +impl CpuInfo { + /// Reads /proc/cpuinfo into CpuInfo. + pub(crate) fn new() -> Result<Self, io::Error> { + let mut file = File::open("/proc/cpuinfo")?; + let mut cpui = Self { raw: String::new() }; + file.read_to_string(&mut cpui.raw)?; + Ok(cpui) + } + /// Returns the value of the cpuinfo `field`. + pub(crate) fn field(&self, field: &str) -> CpuInfoField { + for l in self.raw.lines() { + if l.trim().starts_with(field) { + return CpuInfoField::new(l.split(": ").nth(1)); + } + } + CpuInfoField(None) + } + + /// Returns the `raw` contents of `/proc/cpuinfo` + #[cfg(test)] + fn raw(&self) -> &String { + &self.raw + } + + #[cfg(test)] + fn from_str(other: &str) -> Result<Self, ::std::io::Error> { + Ok(Self { + raw: String::from(other), + }) + } +} + +/// Field of cpuinfo +#[derive(Debug)] +pub(crate) struct CpuInfoField<'a>(Option<&'a str>); + +impl<'a> PartialEq<&'a str> for CpuInfoField<'a> { + fn eq(&self, other: &&'a str) -> bool { + match self.0 { + None => other.is_empty(), + Some(f) => f == other.trim(), + } + } +} + +impl<'a> CpuInfoField<'a> { + pub(crate) fn new<'b>(v: Option<&'b str>) -> CpuInfoField<'b> { + match v { + None => CpuInfoField::<'b>(None), + Some(f) => CpuInfoField::<'b>(Some(f.trim())), + } + } + /// Does the field exist? + #[cfg(test)] + pub(crate) fn exists(&self) -> bool { + self.0.is_some() + } + /// Does the field contain `other`? + pub(crate) fn has(&self, other: &str) -> bool { + match self.0 { + None => other.is_empty(), + Some(f) => { + let other = other.trim(); + for v in f.split(' ') { + if v == other { + return true; + } + } + false + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn raw_dump() { + let cpuinfo = CpuInfo::new().unwrap(); + if cpuinfo.field("vendor_id") == "GenuineIntel" { + assert!(cpuinfo.field("flags").exists()); + assert!(!cpuinfo.field("vendor33_id").exists()); + assert!(cpuinfo.field("flags").has("sse")); + assert!(!cpuinfo.field("flags").has("avx314")); + } + println!("{}", cpuinfo.raw()); + } + + const CORE_DUO_T6500: &str = r"processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 23 +model name : Intel(R) Core(TM)2 Duo CPU T6500 @ 2.10GHz +stepping : 10 +microcode : 0xa0b +cpu MHz : 1600.000 +cache size : 2048 KB +physical id : 0 +siblings : 2 +core id : 0 +cpu cores : 2 +apicid : 0 +initial apicid : 0 +fdiv_bug : no +hlt_bug : no +f00f_bug : no +coma_bug : no +fpu : yes +fpu_exception : yes +cpuid level : 13 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe nx lm constant_tsc arch_perfmon pebs bts aperfmperf pni dtes64 monitor ds_cpl est tm2 ssse3 cx16 xtpr pdcm sse4_1 xsave lahf_lm dtherm +bogomips : 4190.43 +clflush size : 64 +cache_alignment : 64 +address sizes : 36 bits physical, 48 bits virtual +power management: +"; + + #[test] + fn core_duo_t6500() { + let cpuinfo = CpuInfo::from_str(CORE_DUO_T6500).unwrap(); + assert_eq!(cpuinfo.field("vendor_id"), "GenuineIntel"); + assert_eq!(cpuinfo.field("cpu family"), "6"); + assert_eq!(cpuinfo.field("model"), "23"); + assert_eq!( + cpuinfo.field("model name"), + "Intel(R) Core(TM)2 Duo CPU T6500 @ 2.10GHz" + ); + assert_eq!( + cpuinfo.field("flags"), + "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe nx lm constant_tsc arch_perfmon pebs bts aperfmperf pni dtes64 monitor ds_cpl est tm2 ssse3 cx16 xtpr pdcm sse4_1 xsave lahf_lm dtherm" + ); + assert!(cpuinfo.field("flags").has("fpu")); + assert!(cpuinfo.field("flags").has("dtherm")); + assert!(cpuinfo.field("flags").has("sse2")); + assert!(!cpuinfo.field("flags").has("avx")); + } + + const ARM_CORTEX_A53: &str = + r"Processor : AArch64 Processor rev 3 (aarch64) + processor : 0 + processor : 1 + processor : 2 + processor : 3 + processor : 4 + processor : 5 + processor : 6 + processor : 7 + Features : fp asimd evtstrm aes pmull sha1 sha2 crc32 + CPU implementer : 0x41 + CPU architecture: AArch64 + CPU variant : 0x0 + CPU part : 0xd03 + CPU revision : 3 + + Hardware : HiKey Development Board + "; + + #[test] + fn arm_cortex_a53() { + let cpuinfo = CpuInfo::from_str(ARM_CORTEX_A53).unwrap(); + assert_eq!( + cpuinfo.field("Processor"), + "AArch64 Processor rev 3 (aarch64)" + ); + assert_eq!( + cpuinfo.field("Features"), + "fp asimd evtstrm aes pmull sha1 sha2 crc32" + ); + assert!(cpuinfo.field("Features").has("pmull")); + assert!(!cpuinfo.field("Features").has("neon")); + assert!(cpuinfo.field("Features").has("asimd")); + } + + const ARM_CORTEX_A57: &str = r"Processor : Cortex A57 Processor rev 1 (aarch64) +processor : 0 +processor : 1 +processor : 2 +processor : 3 +Features : fp asimd aes pmull sha1 sha2 crc32 wp half thumb fastmult vfp edsp neon vfpv3 tlsi vfpv4 idiva idivt +CPU implementer : 0x41 +CPU architecture: 8 +CPU variant : 0x1 +CPU part : 0xd07 +CPU revision : 1"; + + #[test] + fn arm_cortex_a57() { + let cpuinfo = CpuInfo::from_str(ARM_CORTEX_A57).unwrap(); + assert_eq!( + cpuinfo.field("Processor"), + "Cortex A57 Processor rev 1 (aarch64)" + ); + assert_eq!( + cpuinfo.field("Features"), + "fp asimd aes pmull sha1 sha2 crc32 wp half thumb fastmult vfp edsp neon vfpv3 tlsi vfpv4 idiva idivt" + ); + assert!(cpuinfo.field("Features").has("pmull")); + assert!(cpuinfo.field("Features").has("neon")); + assert!(cpuinfo.field("Features").has("asimd")); + } + + const POWER8E_POWERKVM: &str = r"processor : 0 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +processor : 1 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +processor : 2 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +processor : 3 +cpu : POWER8E (raw), altivec supported +clock : 3425.000000MHz +revision : 2.1 (pvr 004b 0201) + +timebase : 512000000 +platform : pSeries +model : IBM pSeries (emulated by qemu) +machine : CHRP IBM pSeries (emulated by qemu)"; + + #[test] + fn power8_powerkvm() { + let cpuinfo = CpuInfo::from_str(POWER8E_POWERKVM).unwrap(); + assert_eq!(cpuinfo.field("cpu"), "POWER8E (raw), altivec supported"); + + assert!(cpuinfo.field("cpu").has("altivec")); + } + + const POWER5P: &str = r"processor : 0 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 1 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 2 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 3 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 4 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 5 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 6 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +processor : 7 +cpu : POWER5+ (gs) +clock : 1900.098000MHz +revision : 2.1 (pvr 003b 0201) + +timebase : 237331000 +platform : pSeries +machine : CHRP IBM,9133-55A"; + + #[test] + fn power5p() { + let cpuinfo = CpuInfo::from_str(POWER5P).unwrap(); + assert_eq!(cpuinfo.field("cpu"), "POWER5+ (gs)"); + + assert!(!cpuinfo.field("cpu").has("altivec")); + } +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/linux/mips.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/mips.rs new file mode 100644 index 0000000..92e95f0 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/mips.rs
@@ -0,0 +1,33 @@ +//! Run-time feature detection for MIPS on Linux. + +use arch::detect::Feature; +use arch::detect::cache; +use arch::detect::bit; +use super::auxvec; + +/// Performs run-time feature detection. +#[inline] +pub fn check_for(x: Feature) -> bool { + cache::test(x as u32, detect_features) +} + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from /proc/cpuinfo. +fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + // The values are part of the platform-specific [asm/hwcap.h][hwcap] + // + // [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h + if let Ok(auxv) = auxvec::auxv() { + enable_feature(&mut value, Feature::msa, bit::test(auxv.hwcap, 1)); + return value; + } + // TODO: fall back via cpuinfo + value +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/linux/mod.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/mod.rs new file mode 100644 index 0000000..11ac012 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/mod.rs
@@ -0,0 +1,26 @@ +//! Run-time feature detection on Linux + +mod auxvec; +mod cpuinfo; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + mod aarch64; + pub use self::aarch64::check_for; + } else if #[cfg(target_arch = "arm")] { + mod arm; + pub use self::arm::check_for; + } else if #[cfg(any(target_arch = "mips", target_arch = "mips64"))] { + mod mips; + pub use self::mips::check_for; + } else if #[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] { + mod powerpc; + pub use self::powerpc::check_for; + } else { + use arch::detect::Feature; + /// Performs run-time feature detection. + pub fn check_for(_x: Feature) -> bool { + false + } + } +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/linux/powerpc.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/powerpc.rs new file mode 100644 index 0000000..41f5850 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/linux/powerpc.rs
@@ -0,0 +1,43 @@ +//! Run-time feature detection for PowerPC on Linux. + +use arch::detect::Feature; +use arch::detect::cache; +use super::auxvec; +use super::cpuinfo; + +/// Performs run-time feature detection. +#[inline] +pub fn check_for(x: Feature) -> bool { + cache::test(x as u32, detect_features) +} + +/// Try to read the features from the auxiliary vector, and if that fails, try +/// to read them from /proc/cpuinfo. +fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + let enable_feature = |value: &mut cache::Initializer, f, enable| { + if enable { + value.set(f as u32); + } + }; + + // The values are part of the platform-specific [asm/cputable.h][cputable] + // + // [cputable]: https://github.com/torvalds/linux/blob/master/arch/powerpc/include/uapi/asm/cputable.h + if let Ok(auxv) = auxvec::auxv() { + // note: the PowerPC values are the mask to do the test (instead of the + // index of the bit to test like in ARM and Aarch64) + enable_feature(&mut value, Feature::altivec, auxv.hwcap & 0x10000000 != 0); + enable_feature(&mut value, Feature::vsx, auxv.hwcap & 0x00000080 != 0); + enable_feature(&mut value, Feature::power8, auxv.hwcap & 0x80000000 != 0); + return value; + } + + // PowerPC's /proc/cpuinfo lacks a proper Feature field, + // but `altivec` support is indicated in the `cpu` field. + if let Ok(c) = cpuinfo::CpuInfo::new() { + enable_feature(&mut value, Feature::altivec, c.field("cpu").has("altivec")); + return value; + } + value +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/other.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/other.rs new file mode 100644 index 0000000..562cbfe --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/other.rs
@@ -0,0 +1,9 @@ +//! Other operating systems + +use arch::detect::Feature; + +/// Performs run-time feature detection. +#[inline] +pub fn check_for(_x: Feature) -> bool { + false +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/os/x86.rs b/crates/stdsimd/src/stdsimd/arch/detect/os/x86.rs new file mode 100644 index 0000000..3ac009a --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/os/x86.rs
@@ -0,0 +1,348 @@ +//! x86 run-time feature detection is OS independent. + +use core::prelude::v1::*; +use core::mem; +#[cfg(target_arch = "x86")] +use coresimd::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use coresimd::arch::x86_64::*; + +use arch::detect::Feature; +use arch::detect::cache; +use arch::detect::bit; + +/// Performs run-time feature detection. +#[inline] +pub fn check_for(x: Feature) -> bool { + cache::test(x as u32, detect_features) +} + +/// Run-time feature detection on x86 works by using the CPUID instruction. +/// +/// The [CPUID Wikipedia page][wiki_cpuid] contains +/// all the information about which flags to set to query which values, and in +/// which registers these are reported. +/// +/// The definitive references are: +/// - [Intel 64 and IA-32 Architectures Software Developer's Manual Volume 2: +/// Instruction Set Reference, A-Z][intel64_ref]. +/// - [AMD64 Architecture Programmer's Manual, Volume 3: General-Purpose and +/// System Instructions][amd64_ref]. +/// +/// [wiki_cpuid]: https://en.wikipedia.org/wiki/CPUID +/// [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf +/// [amd64_ref]: http://support.amd.com/TechDocs/24594.pdf +#[cfg_attr(feature = "cargo-clippy", allow(similar_names))] +fn detect_features() -> cache::Initializer { + let mut value = cache::Initializer::default(); + + // If the x86 CPU does not support the CPUID instruction then it is too + // old to support any of the currently-detectable features. + if !has_cpuid() { + return value; + } + + // Calling `__cpuid`/`__cpuid_count` from here on is safe because the CPU + // has `cpuid` support. + + // 0. EAX = 0: Basic Information: + // - EAX returns the "Highest Function Parameter", that is, the maximum + // leaf value for subsequent calls of `cpuinfo` in range [0, + // 0x8000_0000]. - The vendor ID is stored in 12 u8 ascii chars, + // returned in EBX, EDX, and ECX (in that order): + let (max_basic_leaf, vendor_id) = unsafe { + let CpuidResult { + eax: max_basic_leaf, + ebx, + ecx, + edx, + } = __cpuid(0); + let vendor_id: [[u8; 4]; 3] = [ + mem::transmute(ebx), + mem::transmute(edx), + mem::transmute(ecx), + ]; + let vendor_id: [u8; 12] = mem::transmute(vendor_id); + (max_basic_leaf, vendor_id) + }; + + if max_basic_leaf < 1 { + // Earlier Intel 486, CPUID not implemented + return value; + } + + // EAX = 1, ECX = 0: Queries "Processor Info and Feature Bits"; + // Contains information about most x86 features. + let CpuidResult { + ecx: proc_info_ecx, + edx: proc_info_edx, + .. + } = unsafe { __cpuid(0x0000_0001_u32) }; + + // EAX = 7, ECX = 0: Queries "Extended Features"; + // Contains information about bmi,bmi2, and avx2 support. + let (extended_features_ebx, extended_features_ecx) = if max_basic_leaf >= 7 + { + let CpuidResult { ebx, ecx, .. } = unsafe { __cpuid(0x0000_0007_u32) }; + (ebx, ecx) + } else { + (0, 0) // CPUID does not support "Extended Features" + }; + + // EAX = 0x8000_0000, ECX = 0: Get Highest Extended Function Supported + // - EAX returns the max leaf value for extended information, that is, + // `cpuid` calls in range [0x8000_0000; u32::MAX]: + let CpuidResult { + eax: extended_max_basic_leaf, + .. + } = unsafe { __cpuid(0x8000_0000_u32) }; + + // EAX = 0x8000_0001, ECX=0: Queries "Extended Processor Info and Feature + // Bits" + let extended_proc_info_ecx = if extended_max_basic_leaf >= 1 { + let CpuidResult { ecx, .. } = unsafe { __cpuid(0x8000_0001_u32) }; + ecx + } else { + 0 + }; + + { + // borrows value till the end of this scope: + let mut enable = |r, rb, f| { + if bit::test(r as usize, rb) { + value.set(f as u32); + } + }; + + enable(proc_info_ecx, 0, Feature::sse3); + enable(proc_info_ecx, 9, Feature::ssse3); + enable(proc_info_ecx, 19, Feature::sse4_1); + enable(proc_info_ecx, 20, Feature::sse4_2); + enable(proc_info_ecx, 23, Feature::popcnt); + enable(proc_info_ecx, 25, Feature::aes); + enable(proc_info_ecx, 1, Feature::pclmulqdq); + enable(proc_info_ecx, 30, Feature::rdrand); + enable(extended_features_ebx, 18, Feature::rdseed); + enable(proc_info_edx, 4, Feature::tsc); + enable(proc_info_edx, 23, Feature::mmx); + enable(proc_info_edx, 24, Feature::fxsr); + enable(proc_info_edx, 25, Feature::sse); + enable(proc_info_edx, 26, Feature::sse2); + enable(extended_features_ebx, 29, Feature::sha); + + enable(extended_features_ebx, 3, Feature::bmi); + enable(extended_features_ebx, 8, Feature::bmi2); + + // `XSAVE` and `AVX` support: + let cpu_xsave = bit::test(proc_info_ecx as usize, 26); + if cpu_xsave { + // 0. Here the CPU supports `XSAVE`. + + // 1. Detect `OSXSAVE`, that is, whether the OS is AVX enabled and + // supports saving the state of the AVX/AVX2 vector registers on + // context-switches, see: + // + // - [intel: is avx enabled?][is_avx_enabled], + // - [mozilla: sse.cpp][mozilla_sse_cpp]. + // + // [is_avx_enabled]: https://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled + // [mozilla_sse_cpp]: https://hg.mozilla.org/mozilla-central/file/64bab5cbb9b6/mozglue/build/SSE.cpp#l190 + let cpu_osxsave = bit::test(proc_info_ecx as usize, 27); + + if cpu_osxsave { + // 2. The OS must have signaled the CPU that it supports saving and + // restoring the: + // + // * SSE -> `XCR0.SSE[1]` + // * AVX -> `XCR0.AVX[2]` + // * AVX-512 -> `XCR0.AVX-512[7:5]`. + // + // by setting the corresponding bits of `XCR0` to `1`. + // + // This is safe because the CPU supports `xsave` + // and the OS has set `osxsave`. + let xcr0 = unsafe { _xgetbv(0) }; + // Test `XCR0.SSE[1]` and `XCR0.AVX[2]` with the mask `0b110 == 6`: + let os_avx_support = xcr0 & 6 == 6; + // Test `XCR0.AVX-512[7:5]` with the mask `0b1110_0000 == 224`: + let os_avx512_support = xcr0 & 224 == 224; + + // Only if the OS and the CPU support saving/restoring the AVX + // registers we enable `xsave` support: + if os_avx_support { + // See "13.3 ENABLING THE XSAVE FEATURE SET AND XSAVE-ENABLED + // FEATURES" in the "Intel® 64 and IA-32 Architectures Software + // Developer’s Manual, Volume 1: Basic Architecture": + // + // "Software enables the XSAVE feature set by setting + // CR4.OSXSAVE[bit 18] to 1 (e.g., with the MOV to CR4 + // instruction). If this bit is 0, execution of any of XGETBV, + // XRSTOR, XRSTORS, XSAVE, XSAVEC, XSAVEOPT, XSAVES, and XSETBV + // causes an invalid-opcode exception (#UD)" + // + enable(proc_info_ecx, 26, Feature::xsave); + + // For `xsaveopt`, `xsavec`, and `xsaves` we need to query: + // Processor Extended State Enumeration Sub-leaf (EAX = 0DH, + // ECX = 1): + if max_basic_leaf >= 0xd { + let CpuidResult { + eax: proc_extended_state1_eax, + .. + } = unsafe { __cpuid_count(0xd_u32, 1) }; + enable(proc_extended_state1_eax, 0, Feature::xsaveopt); + enable(proc_extended_state1_eax, 1, Feature::xsavec); + enable(proc_extended_state1_eax, 3, Feature::xsaves); + } + + // FMA (uses 256-bit wide registers): + enable(proc_info_ecx, 12, Feature::fma); + + // And AVX/AVX2: + enable(proc_info_ecx, 28, Feature::avx); + enable(extended_features_ebx, 5, Feature::avx2); + + // For AVX-512 the OS also needs to support saving/restoring + // the extended state, only then we enable AVX-512 support: + if os_avx512_support { + enable(extended_features_ebx, 16, Feature::avx512f); + enable(extended_features_ebx, 17, Feature::avx512dq); + enable(extended_features_ebx, 21, Feature::avx512_ifma); + enable(extended_features_ebx, 26, Feature::avx512pf); + enable(extended_features_ebx, 27, Feature::avx512er); + enable(extended_features_ebx, 28, Feature::avx512cd); + enable(extended_features_ebx, 30, Feature::avx512bw); + enable(extended_features_ebx, 31, Feature::avx512vl); + enable(extended_features_ecx, 1, Feature::avx512_vbmi); + enable( + extended_features_ecx, + 14, + Feature::avx512_vpopcntdq, + ); + } + } + } + } + + // This detects ABM on AMD CPUs and LZCNT on Intel CPUs. + // On intel CPUs with popcnt, lzcnt implements the + // "missing part" of ABM, so we map both to the same + // internal feature. + // + // The `is_x86_feature_detected!("lzcnt")` macro then + // internally maps to Feature::abm. + enable(extended_proc_info_ecx, 5, Feature::abm); + if vendor_id == *b"AuthenticAMD" { + // These features are only available on AMD CPUs: + enable(extended_proc_info_ecx, 6, Feature::sse4a); + enable(extended_proc_info_ecx, 21, Feature::tbm); + } + } + + value +} + +#[cfg(test)] +mod tests { + extern crate cupid; + + #[test] + fn dump() { + println!("aes: {:?}", is_x86_feature_detected!("aes")); + println!("pclmulqdq: {:?}", is_x86_feature_detected!("pclmulqdq")); + println!("rdrand: {:?}", is_x86_feature_detected!("rdrand")); + println!("rdseed: {:?}", is_x86_feature_detected!("rdseed")); + println!("tsc: {:?}", is_x86_feature_detected!("tsc")); + println!("sse: {:?}", is_x86_feature_detected!("sse")); + println!("sse2: {:?}", is_x86_feature_detected!("sse2")); + println!("sse3: {:?}", is_x86_feature_detected!("sse3")); + println!("ssse3: {:?}", is_x86_feature_detected!("ssse3")); + println!("sse4.1: {:?}", is_x86_feature_detected!("sse4.1")); + println!("sse4.2: {:?}", is_x86_feature_detected!("sse4.2")); + println!("sse4a: {:?}", is_x86_feature_detected!("sse4a")); + println!("sha: {:?}", is_x86_feature_detected!("sha")); + println!("avx: {:?}", is_x86_feature_detected!("avx")); + println!("avx2: {:?}", is_x86_feature_detected!("avx2")); + println!("avx512f {:?}", is_x86_feature_detected!("avx512f")); + println!("avx512cd {:?}", is_x86_feature_detected!("avx512cd")); + println!("avx512er {:?}", is_x86_feature_detected!("avx512er")); + println!("avx512pf {:?}", is_x86_feature_detected!("avx512pf")); + println!("avx512bw {:?}", is_x86_feature_detected!("avx512bw")); + println!("avx512dq {:?}", is_x86_feature_detected!("avx512dq")); + println!("avx512vl {:?}", is_x86_feature_detected!("avx512vl")); + println!("avx512_ifma {:?}", is_x86_feature_detected!("avx512ifma")); + println!("avx512_vbmi {:?}", is_x86_feature_detected!("avx512vbmi")); + println!( + "avx512_vpopcntdq {:?}", + is_x86_feature_detected!("avx512vpopcntdq") + ); + println!("fma: {:?}", is_x86_feature_detected!("fma")); + println!("abm: {:?}", is_x86_feature_detected!("abm")); + println!("bmi: {:?}", is_x86_feature_detected!("bmi1")); + println!("bmi2: {:?}", is_x86_feature_detected!("bmi2")); + println!("tbm: {:?}", is_x86_feature_detected!("tbm")); + println!("popcnt: {:?}", is_x86_feature_detected!("popcnt")); + println!("lzcnt: {:?}", is_x86_feature_detected!("lzcnt")); + println!("fxsr: {:?}", is_x86_feature_detected!("fxsr")); + println!("xsave: {:?}", is_x86_feature_detected!("xsave")); + println!("xsaveopt: {:?}", is_x86_feature_detected!("xsaveopt")); + println!("xsaves: {:?}", is_x86_feature_detected!("xsaves")); + println!("xsavec: {:?}", is_x86_feature_detected!("xsavec")); + } + + #[test] + fn compare_with_cupid() { + let information = cupid::master().unwrap(); + assert_eq!(is_x86_feature_detected!("aes"), information.aesni()); + assert_eq!(is_x86_feature_detected!("pclmulqdq"), information.pclmulqdq()); + assert_eq!(is_x86_feature_detected!("rdrand"), information.rdrand()); + assert_eq!(is_x86_feature_detected!("rdseed"), information.rdseed()); + assert_eq!(is_x86_feature_detected!("tsc"), information.tsc()); + assert_eq!(is_x86_feature_detected!("sse"), information.sse()); + assert_eq!(is_x86_feature_detected!("sse2"), information.sse2()); + assert_eq!(is_x86_feature_detected!("sse3"), information.sse3()); + assert_eq!(is_x86_feature_detected!("ssse3"), information.ssse3()); + assert_eq!(is_x86_feature_detected!("sse4.1"), information.sse4_1()); + assert_eq!(is_x86_feature_detected!("sse4.2"), information.sse4_2()); + assert_eq!(is_x86_feature_detected!("sse4a"), information.sse4a()); + assert_eq!(is_x86_feature_detected!("sha"), information.sha()); + assert_eq!(is_x86_feature_detected!("avx"), information.avx()); + assert_eq!(is_x86_feature_detected!("avx2"), information.avx2()); + assert_eq!(is_x86_feature_detected!("avx512f"), information.avx512f()); + assert_eq!(is_x86_feature_detected!("avx512cd"), information.avx512cd()); + assert_eq!(is_x86_feature_detected!("avx512er"), information.avx512er()); + assert_eq!(is_x86_feature_detected!("avx512pf"), information.avx512pf()); + assert_eq!(is_x86_feature_detected!("avx512bw"), information.avx512bw()); + assert_eq!(is_x86_feature_detected!("avx512dq"), information.avx512dq()); + assert_eq!(is_x86_feature_detected!("avx512vl"), information.avx512vl()); + assert_eq!( + is_x86_feature_detected!("avx512ifma"), + information.avx512_ifma() + ); + assert_eq!( + is_x86_feature_detected!("avx512vbmi"), + information.avx512_vbmi() + ); + assert_eq!( + is_x86_feature_detected!("avx512vpopcntdq"), + information.avx512_vpopcntdq() + ); + assert_eq!(is_x86_feature_detected!("fma"), information.fma()); + assert_eq!(is_x86_feature_detected!("bmi1"), information.bmi1()); + assert_eq!(is_x86_feature_detected!("bmi2"), information.bmi2()); + assert_eq!(is_x86_feature_detected!("popcnt"), information.popcnt()); + assert_eq!(is_x86_feature_detected!("abm"), information.lzcnt()); + assert_eq!(is_x86_feature_detected!("tbm"), information.tbm()); + assert_eq!(is_x86_feature_detected!("lzcnt"), information.lzcnt()); + assert_eq!(is_x86_feature_detected!("xsave"), information.xsave()); + assert_eq!(is_x86_feature_detected!("xsaveopt"), information.xsaveopt()); + assert_eq!( + is_x86_feature_detected!("xsavec"), + information.xsavec_and_xrstor() + ); + assert_eq!( + is_x86_feature_detected!("xsaves"), + information.xsaves_xrstors_and_ia32_xss() + ); + } +}
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/test_data/linux-rpi3.auxv b/crates/stdsimd/src/stdsimd/arch/detect/test_data/linux-rpi3.auxv new file mode 100644 index 0000000..0538e66 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/test_data/linux-rpi3.auxv Binary files differ
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/test_data/linux-x64-i7-6850k.auxv b/crates/stdsimd/src/stdsimd/arch/detect/test_data/linux-x64-i7-6850k.auxv new file mode 100644 index 0000000..6afe1b3 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/test_data/linux-x64-i7-6850k.auxv Binary files differ
diff --git a/crates/stdsimd/src/stdsimd/arch/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv b/crates/stdsimd/src/stdsimd/arch/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv new file mode 100644 index 0000000..75abc02 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/arch/detect/test_data/macos-virtualbox-linux-x86-4850HQ.auxv Binary files differ
diff --git a/crates/stdsimd/src/stdsimd/mod.rs b/crates/stdsimd/src/stdsimd/mod.rs new file mode 100644 index 0000000..dd0a4b4 --- /dev/null +++ b/crates/stdsimd/src/stdsimd/mod.rs
@@ -0,0 +1,514 @@ +//! `stdsimd` + +/// SIMD and vendor intrinsics module. +/// +/// This module is intended to be the gateway to architecture-specific +/// intrinsic functions, typically related to SIMD (but not always!). Each +/// architecture that Rust compiles to may contain a submodule here, which +/// means that this is not a portable module! If you're writing a portable +/// library take care when using these APIs! +/// +/// Under this module you'll find an architecture-named module, such as +/// `x86_64`. Each `#[cfg(target_arch)]` that Rust can compile to may have a +/// module entry here, only present on that particular target. For example the +/// `i686-pc-windows-msvc` target will have an `x86` module here, whereas +/// `x86_64-pc-windows-msvc` has `x86_64`. +/// +/// [rfc]: https://github.com/rust-lang/rfcs/pull/2325 +/// [tracked]: https://github.com/rust-lang/rust/issues/48556 +/// +/// # Overview +/// +/// This module exposes vendor-specific intrinsics that typically correspond to +/// a single machine instruction. These intrinsics are not portable: their +/// availability is architecture-dependent, and not all machines of that +/// architecture might provide the intrinsic. +/// +/// The `arch` module is intended to be a low-level implementation detail for +/// higher-level APIs. Using it correctly can be quite tricky as you need to +/// ensure at least a few guarantees are upheld: +/// +/// * The correct architecture's module is used. For example the `arm` module +/// isn't available on the `x86_64-unknown-linux-gnu` target. This is +/// typically done by ensuring that `#[cfg]` is used appropriately when using +/// this module. +/// * The CPU the program is currently running on supports the function being +/// called. For example it is unsafe to call an AVX2 function on a CPU that +/// doesn't actually support AVX2. +/// +/// As a result of the latter of these guarantees all intrinsics in this module +/// are `unsafe` and extra care needs to be taken when calling them! +/// +/// # CPU Feature Detection +/// +/// In order to call these APIs in a safe fashion there's a number of +/// mechanisms available to ensure that the correct CPU feature is available +/// to call an intrinsic. Let's consider, for example, the `_mm256_add_epi64` +/// intrinsics on the `x86` and `x86_64` architectures. This function requires +/// the AVX2 feature as [documented by Intel][intel-dox] so to correctly call +/// this function we need to (a) guarantee we only call it on `x86`/`x86_64` +/// and (b) ensure that the CPU feature is available +/// +/// [intel-dox]: https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm256_add_epi64&expand=100 +/// +/// ## Static CPU Feature Detection +/// +/// The first option available to us is to conditionally compile code via the +/// `#[cfg]` attribute. CPU features correspond to the `target_feature` cfg +/// available, and can be used like so: +/// +/// ```ignore +/// #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), +/// target_feature = "avx2"))] +/// fn foo() { +/// #[cfg(target_arch = "x86")] +/// use std::arch::x86::_mm256_add_epi64; +/// #[cfg(target_arch = "x86_64")] +/// use std::arch::x86_64::_mm256_add_epi64; +/// +/// unsafe { +/// _mm256_add_epi64(...); +/// } +/// } +/// ``` +/// +/// Here we're using `#[cfg(target_feature = "avx2")]` to conditionally compile +/// this function into our module. This means that if the `avx2` feature is +/// *enabled statically* then we'll use the `_mm256_add_epi64` function at +/// runtime. The `unsafe` block here can be justified through the usage of +/// `#[cfg]` to only compile the code in situations where the safety guarantees +/// are upheld. +/// +/// Statically enabling a feature is typically done with the `-C +/// target-feature` or `-C target-cpu` flags to the compiler. For example if +/// your local CPU supports AVX2 then you can compile the above function with: +/// +/// ```sh +/// $ RUSTFLAGS='-C target-cpu=native' cargo build +/// ``` +/// +/// Or otherwise you can specifically enable just the AVX2 feature: +/// +/// ```sh +/// $ RUSTFLAGS='-C target-feature=+avx2' cargo build +/// ``` +/// +/// Note that when you compile a binary with a particular feature enabled it's +/// important to ensure that you only run the binary on systems which satisfy +/// the required feature set. +/// +/// ## Dynamic CPU Feature Detection +/// +/// Sometimes statically dispatching isn't quite what you want. Instead you +/// might want to build a portable binary that runs across a variety of CPUs, +/// but at runtime it selects the most optimized implementation available. This +/// allows you to build a "least common denominator" binary which has certain +/// sections more optimized for different CPUs. +/// +/// Taking our previous example from before, we're going to compile our binary +/// *without* AVX2 support, but we'd like to enable it for just one function. +/// We can do that in a manner like: +/// +/// ```ignore +/// fn foo() { +/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// { +/// if is_x86_feature_detected!("avx2") { +/// return unsafe { foo_avx2() }; +/// } +/// } +/// +/// // fallback implementation without using AVX2 +/// } +/// +/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// #[target_feature(enable = "avx2")] +/// unsafe fn foo_avx2() { +/// #[cfg(target_arch = "x86")] +/// use std::arch::x86::_mm256_add_epi64; +/// #[cfg(target_arch = "x86_64")] +/// use std::arch::x86_64::_mm256_add_epi64; +/// +/// _mm256_add_epi64(...); +/// } +/// ``` +/// +/// There's a couple of components in play here, so let's go through them in +/// detail! +/// +/// * First up we notice the `is_x86_feature_detected!` macro. Provided by +/// the standard library, this macro will perform necessary runtime detection +/// to determine whether the CPU the program is running on supports the +/// specified feature. In this case the macro will expand to a boolean +/// expression evaluating to whether the local CPU has the AVX2 feature or +/// not. +/// +/// Note that this macro, like the `arch` module, is platform-specific. For +/// example calling `is_x86_feature_detected!("avx2")` on ARM will be a +/// compile time error. To ensure we don't hit this error a statement level +/// `#[cfg]` is used to only compile usage of the macro on `x86`/`x86_64`. +/// +/// * Next up we see our AVX2-enabled function, `foo_avx2`. This function is +/// decorated with the `#[target_feature]` attribute which enables a CPU +/// feature for just this one function. Using a compiler flag like `-C +/// target-feature=+avx2` will enable AVX2 for the entire program, but using +/// an attribute will only enable it for the one function. Usage of the +/// `#[target_feature]` attribute currently requires the function to also be +/// `unsafe`, as we see here. This is because the function can only be +/// correctly called on systems which have the AVX2 (like the intrinsics +/// themselves). +/// +/// And with all that we should have a working program! This program will run +/// across all machines and it'll use the optimized AVX2 implementation on +/// machines where support is detected. +/// +/// # Ergonomics +/// +/// It's important to note that using the `arch` module is not the easiest +/// thing in the world, so if you're curious to try it out you may want to +/// brace yourself for some wordiness! +/// +/// The primary purpose of this module is to enable stable crates on crates.io +/// to build up much more ergonomic abstractions which end up using SIMD under +/// the hood. Over time these abstractions may also move into the standard +/// library itself, but for now this module is tasked with providing the bare +/// minimum necessary to use vendor intrinsics on stable Rust. +/// +/// # Other architectures +/// +/// This documentation is only for one particular architecture, you can find +/// others at: +/// +/// * [`x86`] +/// * [`x86_64`] +/// * [`arm`] +/// * [`aarch64`] +/// * [`mips`] +/// * [`mips64`] +/// * [`powerpc`] +/// * [`powerpc64`] +/// * [`nvptx`] +/// * [`wasm32`] +/// +/// [`x86`]: x86/index.html +/// [`x86_64`]: x86_64/index.html +/// [`arm`]: arm/index.html +/// [`aarch64`]: aarch64/index.html +/// [`mips`]: mips/index.html +/// [`mips64`]: mips64/index.html +/// [`powerpc`]: powerpc/index.html +/// [`powerpc64`]: powerpc64/index.html +/// [`nvptx`]: nvptx/index.html +/// [`wasm32`]: wasm32/index.html +/// +/// # Examples +/// +/// First let's take a look at not actually using any intrinsics but instead +/// using LLVM's auto-vectorization to produce optimized vectorized code for +/// AVX2 and also for the default platform. +/// +/// ```rust +/// # #![cfg_attr(not(dox), feature(cfg_target_feature, target_feature, stdsimd))] +/// +/// # #[cfg(not(dox))] +/// # #[macro_use] +/// # extern crate stdsimd; +/// +/// fn main() { +/// let mut dst = [0]; +/// add_quickly(&[1], &[2], &mut dst); +/// assert_eq!(dst[0], 3); +/// } +/// +/// fn add_quickly(a: &[u8], b: &[u8], c: &mut [u8]) { +/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// { +/// // Note that this `unsafe` block is safe because we're testing +/// // that the `avx2` feature is indeed available on our CPU. +/// if is_x86_feature_detected!("avx2") { +/// return unsafe { add_quickly_avx2(a, b, c) } +/// } +/// } +/// +/// add_quickly_fallback(a, b, c) +/// } +/// +/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// #[target_feature(enable = "avx2")] +/// unsafe fn add_quickly_avx2(a: &[u8], b: &[u8], c: &mut [u8]) { +/// add_quickly_fallback(a, b, c) // the function below is inlined here +/// } +/// +/// fn add_quickly_fallback(a: &[u8], b: &[u8], c: &mut [u8]) { +/// for ((a, b), c) in a.iter().zip(b).zip(c) { +/// *c = *a + *b; +/// } +/// } +/// ``` +/// +/// Next up let's take a look at an example of manually using intrinsics. Here +/// we'll be using SSE4.1 features to implement hex encoding. +/// +/// ``` +/// # #![cfg_attr(not(dox), feature(cfg_target_feature, target_feature, stdsimd))] +/// # #![cfg_attr(not(dox), no_std)] +/// # #[cfg(not(dox))] +/// # extern crate std as real_std; +/// # #[cfg(not(dox))] +/// # #[macro_use] +/// # extern crate stdsimd as std; +/// +/// fn main() { +/// let mut dst = [0; 32]; +/// hex_encode(b"\x01\x02\x03", &mut dst); +/// assert_eq!(&dst[..6], b"010203"); +/// +/// let mut src = [0; 16]; +/// for i in 0..16 { +/// src[i] = (i + 1) as u8; +/// } +/// hex_encode(&src, &mut dst); +/// assert_eq!(&dst, b"0102030405060708090a0b0c0d0e0f10"); +/// } +/// +/// pub fn hex_encode(src: &[u8], dst: &mut [u8]) { +/// let len = src.len().checked_mul(2).unwrap(); +/// assert!(dst.len() >= len); +/// +/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// { +/// if is_x86_feature_detected!("sse4.1") { +/// return unsafe { hex_encode_sse41(src, dst) }; +/// } +/// } +/// +/// hex_encode_fallback(src, dst) +/// } +/// +/// // translated from https://github.com/Matherunner/bin2hex-sse/blob/master/base16_sse4.cpp +/// #[target_feature(enable = "sse4.1")] +/// #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +/// unsafe fn hex_encode_sse41(mut src: &[u8], dst: &mut [u8]) { +/// #[cfg(target_arch = "x86")] +/// use std::arch::x86::*; +/// #[cfg(target_arch = "x86_64")] +/// use std::arch::x86_64::*; +/// +/// let ascii_zero = _mm_set1_epi8(b'0' as i8); +/// let nines = _mm_set1_epi8(9); +/// let ascii_a = _mm_set1_epi8((b'a' - 9 - 1) as i8); +/// let and4bits = _mm_set1_epi8(0xf); +/// +/// let mut i = 0_isize; +/// while src.len() >= 16 { +/// let invec = _mm_loadu_si128(src.as_ptr() as *const _); +/// +/// let masked1 = _mm_and_si128(invec, and4bits); +/// let masked2 = _mm_and_si128(_mm_srli_epi64(invec, 4), and4bits); +/// +/// // return 0xff corresponding to the elements > 9, or 0x00 otherwise +/// let cmpmask1 = _mm_cmpgt_epi8(masked1, nines); +/// let cmpmask2 = _mm_cmpgt_epi8(masked2, nines); +/// +/// // add '0' or the offset depending on the masks +/// let masked1 = _mm_add_epi8( +/// masked1, +/// _mm_blendv_epi8(ascii_zero, ascii_a, cmpmask1), +/// ); +/// let masked2 = _mm_add_epi8( +/// masked2, +/// _mm_blendv_epi8(ascii_zero, ascii_a, cmpmask2), +/// ); +/// +/// // interleave masked1 and masked2 bytes +/// let res1 = _mm_unpacklo_epi8(masked2, masked1); +/// let res2 = _mm_unpackhi_epi8(masked2, masked1); +/// +/// _mm_storeu_si128(dst.as_mut_ptr().offset(i * 2) as *mut _, res1); +/// _mm_storeu_si128(dst.as_mut_ptr().offset(i * 2 + 16) as *mut _, +/// res2); src = &src[16..]; +/// i += 16; +/// } +/// +/// let i = i as usize; +/// hex_encode_fallback(src, &mut dst[i * 2..]); +/// } +/// +/// fn hex_encode_fallback(src: &[u8], dst: &mut [u8]) { +/// fn hex(byte: u8) -> u8 { +/// static TABLE: &[u8] = b"0123456789abcdef"; +/// TABLE[byte as usize] +/// } +/// +/// for (byte, slots) in src.iter().zip(dst.chunks_mut(2)) { +/// slots[0] = hex((*byte >> 4) & 0xf); +/// slots[1] = hex(*byte & 0xf); +/// } +/// } +/// ``` +#[stable(feature = "simd_arch", since = "1.27.0")] +pub mod arch { + #[cfg(all(not(dox), target_arch = "x86"))] + #[stable(feature = "simd_x86", since = "1.27.0")] + pub use coresimd::arch::x86; + + #[cfg(all(not(dox), target_arch = "x86_64"))] + #[stable(feature = "simd_x86", since = "1.27.0")] + pub use coresimd::arch::x86_64; + + #[cfg(all(not(dox), target_arch = "arm"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::arm; + + #[cfg(all(not(dox), target_arch = "aarch64"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::aarch64; + + #[cfg(all(not(dox), target_arch = "mips"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::mips; + + #[cfg(all(not(dox), target_arch = "mips64"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::mips64; + + #[cfg(all(not(dox), target_arch = "powerpc"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::powerpc; + + #[cfg(all(not(dox), target_arch = "powerpc64"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::powerpc64; + + #[cfg(all(not(dox), any(target_arch = "nvptx", target_arch = "nvptx64")))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::nvptx; + + #[cfg(all(not(dox), target_arch = "wasm32"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub use coresimd::arch::wasm32; + + #[doc(hidden)] // unstable implementation detail + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod detect; + + /// Platform-specific intrinsics for the `x86` platform. + /// + /// The documentation with the full listing of `x86` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/x86/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "x86"))] + #[stable(feature = "simd_x86", since = "1.27.0")] + pub mod x86 {} + + /// Platform-specific intrinsics for the `x86_64` platform. + /// + /// The documentation with the full listing of `x86_64` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/x86_64/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "x86_64"))] + #[stable(feature = "simd_x86", since = "1.27.0")] + pub mod x86_64 {} + + /// Platform-specific intrinsics for the `arm` platform. + /// + /// The documentation with the full listing of `arm` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/arm/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "arm"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod arm {} + + /// Platform-specific intrinsics for the `aarch64` platform. + /// + /// The documentation with the full listing of `aarch64` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/aarch64/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "aarch64"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod aarch64 {} + + /// Platform-specific intrinsics for the `mips` platform. + /// + /// The documentation with the full listing of `mips` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/mips/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "mips"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod mips {} + + /// Platform-specific intrinsics for the `mips64` platform. + /// + /// The documentation with the full listing of `mips64` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/mips64/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "mips64"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod mips64 {} + + /// Platform-specific intrinsics for the `powerpc` platform. + /// + /// The documentation with the full listing of `powerpc` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/powerpc/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "powerpc"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod powerpc {} + + /// Platform-specific intrinsics for the `powerpc64` platform. + /// + /// The documentation with the full listing of `powerpc64` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/powerpc64/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "powerpc64"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod powerpc64 {} + + /// Platform-specific intrinsics for the `nvptx` platform. + /// + /// The documentation with the full listing of `nvptx` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/nvptx/index.html + #[cfg(dox)] + #[doc(cfg(any(target_arch = "nvptx", target_arch = "nvptx64")))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod nvptx {} + + /// Platform-specific intrinsics for the `wasm32` platform. + /// + /// The documentation with the full listing of `wasm32` intrinsics is + /// available in [libcore], but the module is re-exported here in std + /// as well. + /// + /// [libcore]: ../../../core/arch/wasm32/index.html + #[cfg(dox)] + #[doc(cfg(target_arch = "wasm32"))] + #[unstable(feature = "stdsimd", issue = "27731")] + pub mod wasm32 {} +}