| //! Run-time feature detection for Aarch64 on Linux. |
| |
| use super::auxvec; |
| use crate::detect::{Feature, bit, cache}; |
| |
| /// Try to read the features from the auxiliary vector. |
| pub(crate) fn detect_features() -> cache::Initializer { |
| #[cfg(target_os = "android")] |
| let is_exynos9810 = { |
| // Samsung Exynos 9810 has a bug that big and little cores have different |
| // ISAs. And on older Android (pre-9), the kernel incorrectly reports |
| // that features available only on some cores are available on all cores. |
| // https://reviews.llvm.org/D114523 |
| let mut arch = [0_u8; libc::PROP_VALUE_MAX as usize]; |
| let len = unsafe { |
| libc::__system_property_get(c"ro.arch".as_ptr(), arch.as_mut_ptr() as *mut libc::c_char) |
| }; |
| // On Exynos, ro.arch is not available on Android 12+, but it is fine |
| // because Android 9+ includes the fix. |
| len > 0 && arch.starts_with(b"exynos9810") |
| }; |
| #[cfg(not(target_os = "android"))] |
| let is_exynos9810 = false; |
| |
| if let Ok(auxv) = auxvec::auxv() { |
| let hwcap: AtHwcap = auxv.into(); |
| return hwcap.cache(is_exynos9810); |
| } |
| cache::Initializer::default() |
| } |
| |
| /// These values are part of the platform-specific [asm/hwcap.h][hwcap] . |
| /// |
| /// The names match those used for cpuinfo. |
| /// |
| /// [hwcap]: https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h |
| #[derive(Debug, Default, PartialEq)] |
| struct AtHwcap { |
| // AT_HWCAP |
| fp: bool, |
| asimd: bool, |
| // evtstrm: No LLVM support. |
| aes: bool, |
| pmull: bool, |
| sha1: bool, |
| sha2: bool, |
| crc32: bool, |
| atomics: bool, |
| fphp: bool, |
| asimdhp: bool, |
| // cpuid: No LLVM support. |
| asimdrdm: bool, |
| jscvt: bool, |
| fcma: bool, |
| lrcpc: bool, |
| dcpop: bool, |
| sha3: bool, |
| sm3: bool, |
| sm4: bool, |
| asimddp: bool, |
| sha512: bool, |
| sve: bool, |
| fhm: bool, |
| dit: bool, |
| uscat: bool, |
| ilrcpc: bool, |
| flagm: bool, |
| ssbs: bool, |
| sb: bool, |
| paca: bool, |
| pacg: bool, |
| |
| // AT_HWCAP2 |
| dcpodp: bool, |
| sve2: bool, |
| sveaes: bool, |
| svepmull: bool, |
| svebitperm: bool, |
| svesha3: bool, |
| svesm4: bool, |
| flagm2: bool, |
| frint: bool, |
| // svei8mm: See i8mm feature. |
| svef32mm: bool, |
| svef64mm: bool, |
| // svebf16: See bf16 feature. |
| i8mm: bool, |
| bf16: bool, |
| // dgh: No LLVM support. |
| rng: bool, |
| bti: bool, |
| mte: bool, |
| ecv: bool, |
| // afp: bool, |
| // rpres: bool, |
| // mte3: bool, |
| sme: bool, |
| smei16i64: bool, |
| smef64f64: bool, |
| // smei8i32: bool, |
| // smef16f32: bool, |
| // smeb16f32: bool, |
| // smef32f32: bool, |
| smefa64: bool, |
| wfxt: bool, |
| // ebf16: bool, |
| // sveebf16: bool, |
| cssc: bool, |
| // rprfm: bool, |
| sve2p1: bool, |
| sme2: bool, |
| sme2p1: bool, |
| // smei16i32: bool, |
| // smebi32i32: bool, |
| smeb16b16: bool, |
| smef16f16: bool, |
| mops: bool, |
| hbc: bool, |
| sveb16b16: bool, |
| lrcpc3: bool, |
| lse128: bool, |
| fpmr: bool, |
| lut: bool, |
| faminmax: bool, |
| f8cvt: bool, |
| f8fma: bool, |
| f8dp4: bool, |
| f8dp2: bool, |
| f8e4m3: bool, |
| f8e5m2: bool, |
| smelutv2: bool, |
| smef8f16: bool, |
| smef8f32: bool, |
| smesf8fma: bool, |
| smesf8dp4: bool, |
| smesf8dp2: bool, |
| // pauthlr: bool, |
| } |
| |
| impl From<auxvec::AuxVec> for AtHwcap { |
| /// Reads AtHwcap from the auxiliary vector. |
| fn from(auxv: auxvec::AuxVec) -> Self { |
| let mut cap = AtHwcap { |
| fp: bit::test(auxv.hwcap, 0), |
| asimd: bit::test(auxv.hwcap, 1), |
| // evtstrm: bit::test(auxv.hwcap, 2), |
| aes: bit::test(auxv.hwcap, 3), |
| pmull: bit::test(auxv.hwcap, 4), |
| sha1: bit::test(auxv.hwcap, 5), |
| sha2: bit::test(auxv.hwcap, 6), |
| crc32: bit::test(auxv.hwcap, 7), |
| atomics: bit::test(auxv.hwcap, 8), |
| fphp: bit::test(auxv.hwcap, 9), |
| asimdhp: bit::test(auxv.hwcap, 10), |
| // cpuid: bit::test(auxv.hwcap, 11), |
| asimdrdm: bit::test(auxv.hwcap, 12), |
| jscvt: bit::test(auxv.hwcap, 13), |
| fcma: bit::test(auxv.hwcap, 14), |
| lrcpc: bit::test(auxv.hwcap, 15), |
| dcpop: bit::test(auxv.hwcap, 16), |
| sha3: bit::test(auxv.hwcap, 17), |
| sm3: bit::test(auxv.hwcap, 18), |
| sm4: bit::test(auxv.hwcap, 19), |
| asimddp: bit::test(auxv.hwcap, 20), |
| sha512: bit::test(auxv.hwcap, 21), |
| sve: bit::test(auxv.hwcap, 22), |
| fhm: bit::test(auxv.hwcap, 23), |
| dit: bit::test(auxv.hwcap, 24), |
| uscat: bit::test(auxv.hwcap, 25), |
| ilrcpc: bit::test(auxv.hwcap, 26), |
| flagm: bit::test(auxv.hwcap, 27), |
| ssbs: bit::test(auxv.hwcap, 28), |
| sb: bit::test(auxv.hwcap, 29), |
| paca: bit::test(auxv.hwcap, 30), |
| pacg: bit::test(auxv.hwcap, 31), |
| |
| // AT_HWCAP2 |
| dcpodp: bit::test(auxv.hwcap2, 0), |
| sve2: bit::test(auxv.hwcap2, 1), |
| sveaes: bit::test(auxv.hwcap2, 2), |
| svepmull: bit::test(auxv.hwcap2, 3), |
| svebitperm: bit::test(auxv.hwcap2, 4), |
| svesha3: bit::test(auxv.hwcap2, 5), |
| svesm4: bit::test(auxv.hwcap2, 6), |
| flagm2: bit::test(auxv.hwcap2, 7), |
| frint: bit::test(auxv.hwcap2, 8), |
| // svei8mm: bit::test(auxv.hwcap2, 9), |
| svef32mm: bit::test(auxv.hwcap2, 10), |
| svef64mm: bit::test(auxv.hwcap2, 11), |
| // svebf16: bit::test(auxv.hwcap2, 12), |
| i8mm: bit::test(auxv.hwcap2, 13), |
| bf16: bit::test(auxv.hwcap2, 14), |
| // dgh: bit::test(auxv.hwcap2, 15), |
| rng: bit::test(auxv.hwcap2, 16), |
| bti: bit::test(auxv.hwcap2, 17), |
| mte: bit::test(auxv.hwcap2, 18), |
| ecv: bit::test(auxv.hwcap2, 19), |
| // afp: bit::test(auxv.hwcap2, 20), |
| // rpres: bit::test(auxv.hwcap2, 21), |
| // mte3: bit::test(auxv.hwcap2, 22), |
| sme: bit::test(auxv.hwcap2, 23), |
| smei16i64: bit::test(auxv.hwcap2, 24), |
| smef64f64: bit::test(auxv.hwcap2, 25), |
| // smei8i32: bit::test(auxv.hwcap2, 26), |
| // smef16f32: bit::test(auxv.hwcap2, 27), |
| // smeb16f32: bit::test(auxv.hwcap2, 28), |
| // smef32f32: bit::test(auxv.hwcap2, 29), |
| smefa64: bit::test(auxv.hwcap2, 30), |
| wfxt: bit::test(auxv.hwcap2, 31), |
| ..Default::default() |
| }; |
| |
| // Hardware capabilities from bits 32 to 63 should only |
| // be tested on LP64 targets with 64 bits `usize`. |
| // On ILP32 targets like `aarch64-unknown-linux-gnu_ilp32`, |
| // these hardware capabilities will default to `false`. |
| // https://github.com/rust-lang/rust/issues/146230 |
| #[cfg(target_pointer_width = "64")] |
| { |
| // cap.ebf16: bit::test(auxv.hwcap2, 32); |
| // cap.sveebf16: bit::test(auxv.hwcap2, 33); |
| cap.cssc = bit::test(auxv.hwcap2, 34); |
| // cap.rprfm: bit::test(auxv.hwcap2, 35); |
| cap.sve2p1 = bit::test(auxv.hwcap2, 36); |
| cap.sme2 = bit::test(auxv.hwcap2, 37); |
| cap.sme2p1 = bit::test(auxv.hwcap2, 38); |
| // cap.smei16i32 = bit::test(auxv.hwcap2, 39); |
| // cap.smebi32i32 = bit::test(auxv.hwcap2, 40); |
| cap.smeb16b16 = bit::test(auxv.hwcap2, 41); |
| cap.smef16f16 = bit::test(auxv.hwcap2, 42); |
| cap.mops = bit::test(auxv.hwcap2, 43); |
| cap.hbc = bit::test(auxv.hwcap2, 44); |
| cap.sveb16b16 = bit::test(auxv.hwcap2, 45); |
| cap.lrcpc3 = bit::test(auxv.hwcap2, 46); |
| cap.lse128 = bit::test(auxv.hwcap2, 47); |
| cap.fpmr = bit::test(auxv.hwcap2, 48); |
| cap.lut = bit::test(auxv.hwcap2, 49); |
| cap.faminmax = bit::test(auxv.hwcap2, 50); |
| cap.f8cvt = bit::test(auxv.hwcap2, 51); |
| cap.f8fma = bit::test(auxv.hwcap2, 52); |
| cap.f8dp4 = bit::test(auxv.hwcap2, 53); |
| cap.f8dp2 = bit::test(auxv.hwcap2, 54); |
| cap.f8e4m3 = bit::test(auxv.hwcap2, 55); |
| cap.f8e5m2 = bit::test(auxv.hwcap2, 56); |
| cap.smelutv2 = bit::test(auxv.hwcap2, 57); |
| cap.smef8f16 = bit::test(auxv.hwcap2, 58); |
| cap.smef8f32 = bit::test(auxv.hwcap2, 59); |
| cap.smesf8fma = bit::test(auxv.hwcap2, 60); |
| cap.smesf8dp4 = bit::test(auxv.hwcap2, 61); |
| cap.smesf8dp2 = bit::test(auxv.hwcap2, 62); |
| // cap.pauthlr = bit::test(auxv.hwcap2, ??); |
| } |
| cap |
| } |
| } |
| |
| impl AtHwcap { |
| /// Initializes the cache from the feature -bits. |
| /// |
| /// The feature dependencies here come directly from LLVM's feature definitions: |
| /// https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64.td |
| fn cache(self, is_exynos9810: bool) -> cache::Initializer { |
| let mut value = cache::Initializer::default(); |
| { |
| let mut enable_feature = |f, enable| { |
| if enable { |
| value.set(f as u32); |
| } |
| }; |
| |
| // Samsung Exynos 9810 has a bug that big and little cores have different |
| // ISAs. And on older Android (pre-9), the kernel incorrectly reports |
| // that features available only on some cores are available on all cores. |
| // So, only check features that are known to be available on exynos-m3: |
| // $ rustc --print cfg --target aarch64-linux-android -C target-cpu=exynos-m3 | grep target_feature |
| // See also https://github.com/rust-lang/stdarch/pull/1378#discussion_r1103748342. |
| if is_exynos9810 { |
| enable_feature(Feature::fp, self.fp); |
| enable_feature(Feature::crc, self.crc32); |
| // ASIMD support requires float support - if half-floats are |
| // supported, it also requires half-float support: |
| let asimd = self.fp && self.asimd && (!self.fphp | self.asimdhp); |
| enable_feature(Feature::asimd, asimd); |
| // Cryptographic extensions require ASIMD |
| // AES also covers FEAT_PMULL |
| enable_feature(Feature::aes, self.aes && self.pmull && asimd); |
| enable_feature(Feature::sha2, self.sha1 && self.sha2 && asimd); |
| return value; |
| } |
| |
| enable_feature(Feature::fp, self.fp); |
| // Half-float support requires float support |
| enable_feature(Feature::fp16, self.fp && self.fphp); |
| // FHM (fp16fml in LLVM) requires half float support |
| enable_feature(Feature::fhm, self.fphp && self.fhm); |
| enable_feature(Feature::pmull, self.pmull); |
| enable_feature(Feature::crc, self.crc32); |
| enable_feature(Feature::lse, self.atomics); |
| enable_feature(Feature::lse2, self.uscat); |
| enable_feature(Feature::lse128, self.lse128 && self.atomics); |
| enable_feature(Feature::rcpc, self.lrcpc); |
| // RCPC2 (rcpc-immo in LLVM) requires RCPC support |
| let rcpc2 = self.ilrcpc && self.lrcpc; |
| enable_feature(Feature::rcpc2, rcpc2); |
| enable_feature(Feature::rcpc3, self.lrcpc3 && rcpc2); |
| enable_feature(Feature::dit, self.dit); |
| enable_feature(Feature::flagm, self.flagm); |
| enable_feature(Feature::flagm2, self.flagm2); |
| enable_feature(Feature::ssbs, self.ssbs); |
| enable_feature(Feature::sb, self.sb); |
| enable_feature(Feature::paca, self.paca); |
| enable_feature(Feature::pacg, self.pacg); |
| // enable_feature(Feature::pauth_lr, self.pauthlr); |
| enable_feature(Feature::dpb, self.dcpop); |
| enable_feature(Feature::dpb2, self.dcpodp); |
| enable_feature(Feature::rand, self.rng); |
| enable_feature(Feature::bti, self.bti); |
| enable_feature(Feature::mte, self.mte); |
| // jsconv requires float support |
| enable_feature(Feature::jsconv, self.jscvt && self.fp); |
| enable_feature(Feature::rdm, self.asimdrdm); |
| enable_feature(Feature::dotprod, self.asimddp); |
| enable_feature(Feature::frintts, self.frint); |
| |
| // FEAT_I8MM & FEAT_BF16 also include optional SVE components which linux exposes |
| // separately. We ignore that distinction here. |
| enable_feature(Feature::i8mm, self.i8mm); |
| enable_feature(Feature::bf16, self.bf16); |
| |
| // ASIMD support requires float support - if half-floats are |
| // supported, it also requires half-float support: |
| let asimd = self.fp && self.asimd && (!self.fphp | self.asimdhp); |
| enable_feature(Feature::asimd, asimd); |
| // ASIMD extensions require ASIMD support: |
| enable_feature(Feature::fcma, self.fcma && asimd); |
| enable_feature(Feature::sve, self.sve && asimd); |
| |
| // SVE extensions require SVE & ASIMD |
| enable_feature(Feature::f32mm, self.svef32mm && self.sve && asimd); |
| enable_feature(Feature::f64mm, self.svef64mm && self.sve && asimd); |
| |
| // Cryptographic extensions require ASIMD |
| enable_feature(Feature::aes, self.aes && asimd); |
| enable_feature(Feature::sha2, self.sha1 && self.sha2 && asimd); |
| // SHA512/SHA3 require SHA1 & SHA256 |
| enable_feature( |
| Feature::sha3, |
| self.sha512 && self.sha3 && self.sha1 && self.sha2 && asimd, |
| ); |
| enable_feature(Feature::sm4, self.sm3 && self.sm4 && asimd); |
| |
| // SVE2 requires SVE |
| let sve2 = self.sve2 && self.sve && asimd; |
| enable_feature(Feature::sve2, sve2); |
| enable_feature(Feature::sve2p1, self.sve2p1 && sve2); |
| // SVE2 extensions require SVE2 and crypto features |
| enable_feature(Feature::sve2_aes, self.sveaes && self.svepmull && sve2 && self.aes); |
| enable_feature(Feature::sve2_sm4, self.svesm4 && sve2 && self.sm3 && self.sm4); |
| enable_feature( |
| Feature::sve2_sha3, |
| self.svesha3 && sve2 && self.sha512 && self.sha3 && self.sha1 && self.sha2, |
| ); |
| enable_feature(Feature::sve2_bitperm, self.svebitperm && self.sve2); |
| enable_feature(Feature::sve_b16b16, self.bf16 && self.sveb16b16); |
| enable_feature(Feature::hbc, self.hbc); |
| enable_feature(Feature::mops, self.mops); |
| enable_feature(Feature::ecv, self.ecv); |
| enable_feature(Feature::lut, self.lut); |
| enable_feature(Feature::cssc, self.cssc); |
| enable_feature(Feature::fpmr, self.fpmr); |
| enable_feature(Feature::faminmax, self.faminmax); |
| let fp8 = self.f8cvt && self.faminmax && self.lut && self.bf16; |
| enable_feature(Feature::fp8, fp8); |
| let fp8fma = self.f8fma && fp8; |
| enable_feature(Feature::fp8fma, fp8fma); |
| let fp8dot4 = self.f8dp4 && fp8fma; |
| enable_feature(Feature::fp8dot4, fp8dot4); |
| enable_feature(Feature::fp8dot2, self.f8dp2 && fp8dot4); |
| enable_feature(Feature::wfxt, self.wfxt); |
| let sme = self.sme && self.bf16; |
| enable_feature(Feature::sme, sme); |
| enable_feature(Feature::sme_i16i64, self.smei16i64 && sme); |
| enable_feature(Feature::sme_f64f64, self.smef64f64 && sme); |
| enable_feature(Feature::sme_fa64, self.smefa64 && sme && sve2); |
| let sme2 = self.sme2 && sme; |
| enable_feature(Feature::sme2, sme2); |
| enable_feature(Feature::sme2p1, self.sme2p1 && sme2); |
| enable_feature( |
| Feature::sme_b16b16, |
| sme2 && self.bf16 && self.sveb16b16 && self.smeb16b16, |
| ); |
| enable_feature(Feature::sme_f16f16, self.smef16f16 && sme2); |
| enable_feature(Feature::sme_lutv2, self.smelutv2); |
| let sme_f8f32 = self.smef8f32 && sme2 && fp8; |
| enable_feature(Feature::sme_f8f32, sme_f8f32); |
| enable_feature(Feature::sme_f8f16, self.smef8f16 && sme_f8f32); |
| let ssve_fp8fma = self.smesf8fma && sme2 && fp8; |
| enable_feature(Feature::ssve_fp8fma, ssve_fp8fma); |
| let ssve_fp8dot4 = self.smesf8dp4 && ssve_fp8fma; |
| enable_feature(Feature::ssve_fp8dot4, ssve_fp8dot4); |
| enable_feature(Feature::ssve_fp8dot2, self.smesf8dp2 && ssve_fp8dot4); |
| } |
| value |
| } |
| } |
| |
| #[cfg(target_endian = "little")] |
| #[cfg(test)] |
| mod tests; |
