#
std_detect/detect/os/
x86.rs

    1//! x86 run-time feature detection is OS independent.
    2
    3#[cfg(target_arch = "x86")]
    4use core::arch::x86::*;
    5#[cfg(target_arch = "x86_64")]
    6use core::arch::x86_64::*;
    7use core::mem;
    8
    9use crate::detect::{Feature, bit, cache};
    10
    11/// Run-time feature detection on x86 works by using the CPUID instruction.
    12///
    13/// The [CPUID Wikipedia page][wiki_cpuid] contains
    14/// all the information about which flags to set to query which values, and in
    15/// which registers these are reported.
    16///
    17/// The definitive references are:
    18/// - [Intel 64 and IA-32 Architectures Software Developer's Manual Volume 2:
    19///   Instruction Set Reference, A-Z][intel64_ref].
    20/// - [AMD64 Architecture Programmer's Manual, Volume 3: General-Purpose and
    21///   System Instructions][amd64_ref].
    22///
    23/// [wiki_cpuid]: https://en.wikipedia.org/wiki/CPUID
    24/// [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
    25/// [amd64_ref]: http://support.amd.com/TechDocs/24594.pdf
    26#[allow(clippy::similar_names)]
    27pub(crate) fn detect_features() -> cache::Initializer {
    28    let mut value = cache::Initializer::default();
    29
    30    if cfg!(target_env = "sgx") {
    31        // doesn't support this because it is untrusted data
    32        return value;
    33    }
    34
    35    // Calling `__cpuid`/`__cpuid_count` from here on is safe because the CPU
    36    // has `cpuid` support.
    37
    38    // 0. EAX = 0: Basic Information:
    39    // - EAX returns the "Highest Function Parameter", that is, the maximum
    40    // leaf value for subsequent calls of `cpuinfo` in range [0,
    41    // 0x8000_0000]. - The vendor ID is stored in 12 u8 ascii chars,
    42    // returned in EBX, EDX, and   ECX (in that order):
    43    let (max_basic_leaf, vendor_id) = unsafe {
    44        let CpuidResult { eax: max_basic_leaf, ebx, ecx, edx } = __cpuid(0);
    45        let vendor_id: [[u8; 4]; 3] = [ebx.to_ne_bytes(), edx.to_ne_bytes(), ecx.to_ne_bytes()];
    46        let vendor_id: [u8; 12] = mem::transmute(vendor_id);
    47        (max_basic_leaf, vendor_id)
    48    };
    49
    50    if max_basic_leaf < 1 {
    51        // Earlier Intel 486, CPUID not implemented
    52        return value;
    53    }
    54
    55    // EAX = 1, ECX = 0: Queries "Processor Info and Feature Bits";
    56    // Contains information about most x86 features.
    57    let CpuidResult { ecx: proc_info_ecx, edx: proc_info_edx, .. } =
    58        unsafe { __cpuid(0x0000_0001_u32) };
    59
    60    // EAX = 7: Queries "Extended Features";
    61    // Contains information about bmi,bmi2, and avx2 support.
    62    let (
    63        extended_features_ebx,
    64        extended_features_ecx,
    65        extended_features_edx,
    66        extended_features_eax_leaf_1,
    67        extended_features_edx_leaf_1,
    68    ) = if max_basic_leaf >= 7 {
    69        let CpuidResult { ebx, ecx, edx, .. } = unsafe { __cpuid(0x0000_0007_u32) };
    70        let CpuidResult { eax: eax_1, edx: edx_1, .. } =
    71            unsafe { __cpuid_count(0x0000_0007_u32, 0x0000_0001_u32) };
    72        (ebx, ecx, edx, eax_1, edx_1)
    73    } else {
    74        (0, 0, 0, 0, 0) // CPUID does not support "Extended Features"
    75    };
    76
    77    // EAX = 0x8000_0000, ECX = 0: Get Highest Extended Function Supported
    78    // - EAX returns the max leaf value for extended information, that is,
    79    // `cpuid` calls in range [0x8000_0000; u32::MAX]:
    80    let CpuidResult { eax: extended_max_basic_leaf, .. } = unsafe { __cpuid(0x8000_0000_u32) };
    81
    82    // EAX = 0x8000_0001, ECX=0: Queries "Extended Processor Info and Feature
    83    // Bits"
    84    let extended_proc_info_ecx = if extended_max_basic_leaf >= 1 {
    85        let CpuidResult { ecx, .. } = unsafe { __cpuid(0x8000_0001_u32) };
    86        ecx
    87    } else {
    88        0
    89    };
    90
    91    {
    92        // borrows value till the end of this scope:
    93        let mut enable = |r, rb, f| {
    94            let present = bit::test(r as usize, rb);
    95            if present {
    96                value.set(f as u32);
    97            }
    98            present
    99        };
    100
    101        enable(proc_info_ecx, 0, Feature::sse3);
    102        enable(proc_info_ecx, 1, Feature::pclmulqdq);
    103        enable(proc_info_ecx, 9, Feature::ssse3);
    104        enable(proc_info_ecx, 13, Feature::cmpxchg16b);
    105        enable(proc_info_ecx, 19, Feature::sse4_1);
    106        enable(proc_info_ecx, 20, Feature::sse4_2);
    107        enable(proc_info_ecx, 22, Feature::movbe);
    108        enable(proc_info_ecx, 23, Feature::popcnt);
    109        enable(proc_info_ecx, 25, Feature::aes);
    110        let f16c = enable(proc_info_ecx, 29, Feature::f16c);
    111        enable(proc_info_ecx, 30, Feature::rdrand);
    112        enable(extended_features_ebx, 18, Feature::rdseed);
    113        enable(extended_features_ebx, 19, Feature::adx);
    114        enable(extended_features_ebx, 11, Feature::rtm);
    115        enable(proc_info_edx, 4, Feature::tsc);
    116        enable(proc_info_edx, 23, Feature::mmx);
    117        enable(proc_info_edx, 24, Feature::fxsr);
    118        enable(proc_info_edx, 25, Feature::sse);
    119        enable(proc_info_edx, 26, Feature::sse2);
    120        enable(extended_features_ebx, 29, Feature::sha);
    121
    122        enable(extended_features_ecx, 8, Feature::gfni);
    123        enable(extended_features_ecx, 9, Feature::vaes);
    124        enable(extended_features_ecx, 10, Feature::vpclmulqdq);
    125
    126        enable(extended_features_ebx, 3, Feature::bmi1);
    127        enable(extended_features_ebx, 8, Feature::bmi2);
    128
    129        enable(extended_features_ebx, 9, Feature::ermsb);
    130
    131        enable(extended_features_eax_leaf_1, 31, Feature::movrs);
    132
    133        // Detect if CPUID.19h available
    134        if bit::test(extended_features_ecx as usize, 23) {
    135            let CpuidResult { ebx, .. } = unsafe { __cpuid(0x19) };
    136            enable(ebx, 0, Feature::kl);
    137            enable(ebx, 2, Feature::widekl);
    138        }
    139
    140        // `XSAVE` and `AVX` support:
    141        let cpu_xsave = bit::test(proc_info_ecx as usize, 26);
    142        if cpu_xsave {
    143            // 0. Here the CPU supports `XSAVE`.
    144
    145            // 1. Detect `OSXSAVE`, that is, whether the OS is AVX enabled and
    146            // supports saving the state of the AVX/AVX2 vector registers on
    147            // context-switches, see:
    148            //
    149            // - [intel: is avx enabled?][is_avx_enabled],
    150            // - [mozilla: sse.cpp][mozilla_sse_cpp].
    151            //
    152            // [is_avx_enabled]: https://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled
    153            // [mozilla_sse_cpp]: https://hg.mozilla.org/mozilla-central/file/64bab5cbb9b6/mozglue/build/SSE.cpp#l190
    154            let cpu_osxsave = bit::test(proc_info_ecx as usize, 27);
    155
    156            if cpu_osxsave {
    157                // 2. The OS must have signaled the CPU that it supports saving and
    158                // restoring the:
    159                //
    160                // * SSE -> `XCR0.SSE[1]`
    161                // * AVX -> `XCR0.AVX[2]`
    162                // * AVX-512 -> `XCR0.AVX-512[7:5]`.
    163                // * AMX -> `XCR0.AMX[18:17]`
    164                //
    165                // by setting the corresponding bits of `XCR0` to `1`.
    166                //
    167                // This is safe because the CPU supports `xsave`
    168                // and the OS has set `osxsave`.
    169                let xcr0 = unsafe { _xgetbv(0) };
    170                // Test `XCR0.SSE[1]` and `XCR0.AVX[2]` with the mask `0b110 == 6`:
    171                let os_avx_support = xcr0 & 6 == 6;
    172                // Test `XCR0.AVX-512[7:5]` with the mask `0b1110_0000 == 0xe0`:
    173                let os_avx512_support = xcr0 & 0xe0 == 0xe0;
    174                // Test `XCR0.AMX[18:17]` with the mask `0b110_0000_0000_0000_0000 == 0x60000`
    175                let os_amx_support = xcr0 & 0x60000 == 0x60000;
    176
    177                // Only if the OS and the CPU support saving/restoring the AVX
    178                // registers we enable `xsave` support:
    179                if os_avx_support {
    180                    // See "13.3 ENABLING THE XSAVE FEATURE SET AND XSAVE-ENABLED
    181                    // FEATURES" in the "Intel® 64 and IA-32 Architectures Software
    182                    // Developer’s Manual, Volume 1: Basic Architecture":
    183                    //
    184                    // "Software enables the XSAVE feature set by setting
    185                    // CR4.OSXSAVE[bit 18] to 1 (e.g., with the MOV to CR4
    186                    // instruction). If this bit is 0, execution of any of XGETBV,
    187                    // XRSTOR, XRSTORS, XSAVE, XSAVEC, XSAVEOPT, XSAVES, and XSETBV
    188                    // causes an invalid-opcode exception (#UD)"
    189                    //
    190                    enable(proc_info_ecx, 26, Feature::xsave);
    191
    192                    // For `xsaveopt`, `xsavec`, and `xsaves` we need to query:
    193                    // Processor Extended State Enumeration Sub-leaf (EAX = 0DH,
    194                    // ECX = 1):
    195                    if max_basic_leaf >= 0xd {
    196                        let CpuidResult { eax: proc_extended_state1_eax, .. } =
    197                            unsafe { __cpuid_count(0xd_u32, 1) };
    198                        enable(proc_extended_state1_eax, 0, Feature::xsaveopt);
    199                        enable(proc_extended_state1_eax, 1, Feature::xsavec);
    200                        enable(proc_extended_state1_eax, 3, Feature::xsaves);
    201                    }
    202
    203                    // FMA (uses 256-bit wide registers):
    204                    let fma = enable(proc_info_ecx, 12, Feature::fma);
    205
    206                    // And AVX/AVX2:
    207                    enable(proc_info_ecx, 28, Feature::avx);
    208                    enable(extended_features_ebx, 5, Feature::avx2);
    209
    210                    // "Short" versions of AVX512 instructions
    211                    enable(extended_features_eax_leaf_1, 4, Feature::avxvnni);
    212                    enable(extended_features_eax_leaf_1, 23, Feature::avxifma);
    213                    enable(extended_features_edx_leaf_1, 4, Feature::avxvnniint8);
    214                    enable(extended_features_edx_leaf_1, 5, Feature::avxneconvert);
    215                    enable(extended_features_edx_leaf_1, 10, Feature::avxvnniint16);
    216
    217                    enable(extended_features_eax_leaf_1, 0, Feature::sha512);
    218                    enable(extended_features_eax_leaf_1, 1, Feature::sm3);
    219                    enable(extended_features_eax_leaf_1, 2, Feature::sm4);
    220
    221                    // For AVX-512 the OS also needs to support saving/restoring
    222                    // the extended state, only then we enable AVX-512 support:
    223                    // Also, Rust makes `avx512f` imply `fma` and `f16c`, because
    224                    // otherwise the assembler is broken. But Intel doesn't guarantee
    225                    // that `fma` and `f16c` are available with `avx512f`, so we
    226                    // need to check for them separately.
    227                    if os_avx512_support && f16c && fma {
    228                        enable(extended_features_ebx, 16, Feature::avx512f);
    229                        enable(extended_features_ebx, 17, Feature::avx512dq);
    230                        enable(extended_features_ebx, 21, Feature::avx512ifma);
    231                        enable(extended_features_ebx, 26, Feature::avx512pf);
    232                        enable(extended_features_ebx, 27, Feature::avx512er);
    233                        enable(extended_features_ebx, 28, Feature::avx512cd);
    234                        enable(extended_features_ebx, 30, Feature::avx512bw);
    235                        enable(extended_features_ebx, 31, Feature::avx512vl);
    236                        enable(extended_features_ecx, 1, Feature::avx512vbmi);
    237                        enable(extended_features_ecx, 6, Feature::avx512vbmi2);
    238                        enable(extended_features_ecx, 11, Feature::avx512vnni);
    239                        enable(extended_features_ecx, 12, Feature::avx512bitalg);
    240                        enable(extended_features_ecx, 14, Feature::avx512vpopcntdq);
    241                        enable(extended_features_edx, 8, Feature::avx512vp2intersect);
    242                        enable(extended_features_edx, 23, Feature::avx512fp16);
    243                        enable(extended_features_eax_leaf_1, 5, Feature::avx512bf16);
    244                    }
    245                }
    246
    247                if os_amx_support {
    248                    enable(extended_features_edx, 24, Feature::amx_tile);
    249                    enable(extended_features_edx, 25, Feature::amx_int8);
    250                    enable(extended_features_edx, 22, Feature::amx_bf16);
    251                    enable(extended_features_eax_leaf_1, 21, Feature::amx_fp16);
    252                    enable(extended_features_edx_leaf_1, 8, Feature::amx_complex);
    253
    254                    if max_basic_leaf >= 0x1e {
    255                        let CpuidResult { eax: amx_feature_flags_eax, .. } =
    256                            unsafe { __cpuid_count(0x1e_u32, 1) };
    257
    258                        enable(amx_feature_flags_eax, 4, Feature::amx_fp8);
    259                        enable(amx_feature_flags_eax, 5, Feature::amx_transpose);
    260                        enable(amx_feature_flags_eax, 6, Feature::amx_tf32);
    261                        enable(amx_feature_flags_eax, 7, Feature::amx_avx512);
    262                        enable(amx_feature_flags_eax, 8, Feature::amx_movrs);
    263                    }
    264                }
    265            }
    266        }
    267
    268        // This detects ABM on AMD CPUs and LZCNT on Intel CPUs.
    269        // On intel CPUs with popcnt, lzcnt implements the
    270        // "missing part" of ABM, so we map both to the same
    271        // internal feature.
    272        //
    273        // The `is_x86_feature_detected!("lzcnt")` macro then
    274        // internally maps to Feature::abm.
    275        enable(extended_proc_info_ecx, 5, Feature::lzcnt);
    276
    277        // As Hygon Dhyana originates from AMD technology and shares most of the architecture with
    278        // AMD's family 17h, but with different CPU Vendor ID("HygonGenuine")/Family series
    279        // number(Family 18h).
    280        //
    281        // For CPUID feature bits, Hygon Dhyana(family 18h) share the same definition with AMD
    282        // family 17h.
    283        //
    284        // Related AMD CPUID specification is https://www.amd.com/system/files/TechDocs/25481.pdf.
    285        // Related Hygon kernel patch can be found on
    286        // http://lkml.kernel.org/r/5ce86123a7b9dad925ac583d88d2f921040e859b.1538583282.git.puwen@hygon.cn
    287        if vendor_id == *b"AuthenticAMD" || vendor_id == *b"HygonGenuine" {
    288            // These features are available on AMD arch CPUs:
    289            enable(extended_proc_info_ecx, 6, Feature::sse4a);
    290            enable(extended_proc_info_ecx, 21, Feature::tbm);
    291            enable(extended_proc_info_ecx, 11, Feature::xop);
    292        }
    293    }
    294
    295    // Unfortunately, some Skylake chips erroneously report support for BMI1 and
    296    // BMI2 without actual support. These chips don't support AVX, and it seems
    297    // that all Intel chips with non-erroneous support BMI do (I didn't check
    298    // other vendors), so we can disable these flags for chips that don't also
    299    // report support for AVX.
    300    //
    301    // It's possible this will pessimize future chips that do support BMI and
    302    // not AVX, but this seems minor compared to a hard crash you get when
    303    // executing an unsupported instruction (to put it another way, it's safe
    304    // for us to under-report CPU features, but not to over-report them). Still,
    305    // to limit any impact this may have in the future, we only do this for
    306    // Intel chips, as it's a bug only present in their chips.
    307    //
    308    // This bug is documented as `SKL052` in the errata section of this document:
    309    // http://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/desktop-6th-gen-core-family-spec-update.pdf
    310    if vendor_id == *b"GenuineIntel" && !value.test(Feature::avx as u32) {
    311        value.unset(Feature::bmi1 as u32);
    312        value.unset(Feature::bmi2 as u32);
    313    }
    314
    315    value
    316}