From c31527558aee941acc3a42107668bf71d47e06ff Mon Sep 17 00:00:00 2001 From: JacobBarthelmeh Date: Wed, 23 Sep 2026 15:33:56 -0600 Subject: [PATCH] remove the now redundent cpuid checks, reduce sha256 alignment without movbe changes --- wolfcrypt/src/cpuid.c | 52 ------------------ wolfcrypt/src/sha256.c | 121 +++++++++++++++++++---------------------- 2 files changed, 55 insertions(+), 118 deletions(-) diff --git a/wolfcrypt/src/cpuid.c b/wolfcrypt/src/cpuid.c index 7ccdd57cfe..39b45818b3 100644 --- a/wolfcrypt/src/cpuid.c +++ b/wolfcrypt/src/cpuid.c @@ -168,57 +168,6 @@ return 0; } - /* XGETBV #UD unless CPUID.1:ECX.OSXSAVE is set. */ - static WC_INLINE word64 cpuid_xgetbv0(void) - { - word32 eax, edx; - #ifndef _MSC_VER - /* 0f 01 d0 = xgetbv */ - __asm__ __volatile__ ( - ".byte 0x0f, 0x01, 0xd0" - : "=a" (eax), "=d" (edx) - : "c" (0) - ); - #else - unsigned long long xcr0 = _xgetbv(0); - eax = (word32)xcr0; - edx = (word32)(xcr0 >> 32); - #endif - return ((word64)edx << 32) | eax; - } - - /* AVX needs OS XSAVE + XCR0 YMM/ZMM. CPUID alone can still SIGILL. */ - #define CPUID_AVX_YMM_FLAGS \ - (CPUID_AVX1 | CPUID_AVX2 | CPUID_VAES) - #define CPUID_AVX_ZMM_FLAGS \ - (CPUID_AVX512 | CPUID_AVX512_VBMI | CPUID_AVX512_VBMI2 | \ - CPUID_AVX512_IFMA | CPUID_AVX512_VL | CPUID_AVX512_DQ | \ - CPUID_AVX512_BW) - /* XCR0: bit1 XMM, bit2 YMM */ - #define CPUID_XCR0_SSE_AVX 0x6ULL - /* + bit5 opmask, bit6 ZMM_Hi256, bit7 Hi16_ZMM */ - #define CPUID_XCR0_AVX512 0xe6ULL - - static void cpuid_mask_avx_without_os_support(cpuid_flags_t* flags) - { - word64 xcr0; - - /* CPUID.1:ECX.OSXSAVE */ - if (!cpuid_flag(1, 0, ECX, 27)) { - *flags &= ~(CPUID_AVX_YMM_FLAGS | CPUID_AVX_ZMM_FLAGS); - return; - } - - xcr0 = cpuid_xgetbv0(); - if ((xcr0 & CPUID_XCR0_SSE_AVX) != CPUID_XCR0_SSE_AVX) { - *flags &= ~(CPUID_AVX_YMM_FLAGS | CPUID_AVX_ZMM_FLAGS); - return; - } - if ((xcr0 & CPUID_XCR0_AVX512) != CPUID_XCR0_AVX512) { - *flags &= ~CPUID_AVX_ZMM_FLAGS; - } - } - static WC_INLINE void cpuid_set_flags(void) { @@ -268,7 +217,6 @@ if (cpuid_is_intel()) { new_cpuid_flags |= CPUID_INTEL ; } if (cpuid_is_amd()) { new_cpuid_flags |= CPUID_AMD ; } if (cpuid_flag(1, 0, ECX, 9)) { new_cpuid_flags |= CPUID_SSSE3 ; } - cpuid_mask_avx_without_os_support(&new_cpuid_flags); (void)wolfSSL_Atomic_Uint_CompareExchange (&cpuid_flags, &old_cpuid_flags, new_cpuid_flags); } diff --git a/wolfcrypt/src/sha256.c b/wolfcrypt/src/sha256.c index 04dc69bbe6..6f9612af0d 100644 --- a/wolfcrypt/src/sha256.c +++ b/wolfcrypt/src/sha256.c @@ -2417,23 +2417,11 @@ static WC_INLINE int Transform_Sha256_Len(wc_Sha256* sha256, const byte* data, unsigned char* hash) { int ret; - #if defined(WOLFSSL_X86_64_BUILD) && defined(USE_INTEL_SPEEDUP) && \ - defined(HAVE_CPUID_INTEL) - int use_movbe; - #endif if ((sha256 == NULL) || (data == NULL)) { return BAD_FUNC_ARG; } - /* Align 16 for SHA-NI / AVX loads. Skip when data is already - * sha256->buffer (LMS); that buffer is not ALIGN16 on 32-bit. */ - if ((data != (const unsigned char*)sha256->buffer) && - (((wc_ptr_t)data & 0xfU) != 0)) { - XMEMCPY(sha256->buffer, data, WC_SHA256_BLOCK_SIZE); - data = (const unsigned char*)sha256->buffer; - } - if (SHA256_UPDATE_REV_BYTES(&sha256->ctx)) { #ifdef WOLFSSL_WIDE_BYTE /* CHAR_BIT != 8: pack 16 big-endian schedule words octet-wise */ @@ -2457,65 +2445,66 @@ static WC_INLINE int Transform_Sha256_Len(wc_Sha256* sha256, const byte* data, if (!SHA256_REV_BYTES(&sha256->ctx)) { XMEMCPY(hash, sha256->digest, WC_SHA256_DIGEST_SIZE); } - else { - #if defined(WOLFSSL_X86_64_BUILD) && defined(USE_INTEL_SPEEDUP) && \ - defined(HAVE_CPUID_INTEL) - use_movbe = (IS_INTEL_MOVBE(cpuid_get_flags()) != 0); - if (use_movbe) { - __asm__ __volatile__ ( - "mov 0x00(%[d]), %%esi\n\t" - "movbe %%esi, 0x00(%[h])\n\t" - "mov 0x04(%[d]), %%esi\n\t" - "movbe %%esi, 0x04(%[h])\n\t" - "mov 0x08(%[d]), %%esi\n\t" - "movbe %%esi, 0x08(%[h])\n\t" - "mov 0x0c(%[d]), %%esi\n\t" - "movbe %%esi, 0x0c(%[h])\n\t" - "mov 0x10(%[d]), %%esi\n\t" - "movbe %%esi, 0x10(%[h])\n\t" - "mov 0x14(%[d]), %%esi\n\t" - "movbe %%esi, 0x14(%[h])\n\t" - "mov 0x18(%[d]), %%esi\n\t" - "movbe %%esi, 0x18(%[h])\n\t" - "mov 0x1c(%[d]), %%esi\n\t" - "movbe %%esi, 0x1c(%[h])\n\t" - : - : [d] "r" (sha256->digest), [h] "r" (hash) - : "memory", "esi" - ); - } - else + #if defined(WOLFSSL_X86_64_BUILD) && defined(USE_INTEL_SPEEDUP) + else if (!IS_INTEL_MOVBE(cpuid_get_flags())) { + /* No MOVBE: reverse into aligned buffer, hash may be + * unaligned. */ + word32 buf[WC_SHA256_DIGEST_SIZE / sizeof(word32)]; + ByteReverseWords(buf, sha256->digest, WC_SHA256_DIGEST_SIZE); + XMEMCPY(hash, buf, WC_SHA256_DIGEST_SIZE); + } #endif - { - /* C fallback, now reachable on x86_64 when the CPU has - * no MOVBE. LMS passes an unaligned interior pointer as - * hash, so the alignment staging must be unconditional: - * WOLFSSL_GENERAL_ALIGNMENT >= 4 does not imply hash is - * aligned here. */ - word32* hash32 = (word32*)hash; - word32* digest = (word32*)sha256->digest; - ALIGN16 word32 buf[WC_SHA256_DIGEST_SIZE / sizeof(word32)]; + else { + #if defined(WOLFSSL_X86_64_BUILD) && defined(USE_INTEL_SPEEDUP) + __asm__ __volatile__ ( + "mov 0x00(%[d]), %%esi\n\t" + "movbe %%esi, 0x00(%[h])\n\t" + "mov 0x04(%[d]), %%esi\n\t" + "movbe %%esi, 0x04(%[h])\n\t" + "mov 0x08(%[d]), %%esi\n\t" + "movbe %%esi, 0x08(%[h])\n\t" + "mov 0x0c(%[d]), %%esi\n\t" + "movbe %%esi, 0x0c(%[h])\n\t" + "mov 0x10(%[d]), %%esi\n\t" + "movbe %%esi, 0x10(%[h])\n\t" + "mov 0x14(%[d]), %%esi\n\t" + "movbe %%esi, 0x14(%[h])\n\t" + "mov 0x18(%[d]), %%esi\n\t" + "movbe %%esi, 0x18(%[h])\n\t" + "mov 0x1c(%[d]), %%esi\n\t" + "movbe %%esi, 0x1c(%[h])\n\t" + : + : [d] "r" (sha256->digest), [h] "r" (hash) + : "memory", "esi" + ); + #else + word32* hash32 = (word32*)hash; + word32* digest = (word32*)sha256->digest; + #if WOLFSSL_GENERAL_ALIGNMENT < 4 + ALIGN16 word32 buf[WC_SHA256_DIGEST_SIZE / sizeof(word32)]; - if (((size_t)digest & 0x3) != 0) { - if (((size_t)hash32 & 0x3) != 0) { - XMEMCPY(buf, digest, WC_SHA256_DIGEST_SIZE); - hash32 = buf; - digest = buf; - } - else { - XMEMCPY(hash, digest, WC_SHA256_DIGEST_SIZE); - digest = hash32; - } + if (((size_t)digest & 0x3) != 0) { + if (((size_t)hash32 & 0x3) != 0) { + XMEMCPY(buf, digest, WC_SHA256_DIGEST_SIZE); + hash32 = buf; + digest = buf; } - else if (((size_t)hash32 & 0x3) != 0) { - hash32 = digest; - } - ByteReverseWords(hash32, digest, - (word32)(sizeof(word32) * 8)); - if (hash != (byte*)hash32) { - XMEMCPY(hash, hash32, WC_SHA256_DIGEST_SIZE); + else { + XMEMCPY(hash, digest, WC_SHA256_DIGEST_SIZE); + digest = hash32; } } + else if (((size_t)hash32 & 0x3) != 0) { + hash32 = digest; + } + #endif + ByteReverseWords(hash32, digest, (word32)(sizeof(word32) * 8)); + #if WOLFSSL_GENERAL_ALIGNMENT < 4 + if (hash != (byte*)hash32) { + XMEMCPY(hash, hash32, WC_SHA256_DIGEST_SIZE); + } + #endif + #endif /* WOLFSSL_X86_64_BUILD && USE_INTEL_SPEEDUP */ } sha256->digest[0] = 0x6A09E667L; sha256->digest[1] = 0xBB67AE85L;