crypt/wolfssl
1
0
Fork 0
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-10-24 04:09:07 +03:00
Code Activity

linuxkm/x86_vector_register_glue.c: add WC_SVR_USE_NATIVE_REG_BUFS: per-CPU

Browse Source 
native vector register save buffers, serving hardirq and softirq contexts
where kernel_fpu_begin() is unusable.

previously these contexts were refused (WC_ACCEL_INHIBIT_E): hardirq
outright, softirq whenever !may_use_simd() (pre-6.15, a softirq
interrupting a foreign kernel_fpu section).  an exact save/compute/restore
cycle is invisible to the interrupted context and to kernel FPU
bookkeeping, and run-to-completion freezes the blocking condition beneath
the section, so both classes are natively serviceable.

- per-CPU {softirq, hardirq} state, each {depth, pin_preempt, save_area};
  xsave64/xrstor64 with RFBM 0xe7 & XCR0 (fxsave64 fallback), plain XSAVE
  only; feature detection by direct CPUID/XGETBV, independent of kernel
  cpufeature machinery.

- recursion is a per-(CPU, class) depth count -- save on the 0->1 edge,
  restore on 1->0; softirq sections hold preempt_disable() (load-bearing
  on PREEMPT_RT).

- hardirq serves plain saves; softirq serves plain and _MAYBE_INHIBIT
  (its legacy short-circuit is bypassed when native is ready); _INHIBIT,
  hardirq _MAYBE, and NMI keep refusal semantics; task context is
  untouched.

- wc_svr_native_init() activates in place from
  wc_linuxkm_allocate_svr_states() in any context but NMI -- the FIPS
  pre-Init lazy allocation arrives inside the outermost save's own
  bracket, and wolfCrypt_Init() has been observed with irqs disabled.  a
  self-bracketed register round-trip selftest gates wc_svr_native_ready;
  on any failure the long-standing refusal semantics remain fully in
  force.

- wc_linuxkm_svr_native_is_ready(): runtime readiness accessor for policy
  decisions elsewhere (e.g. the FIPS<v7 default-bank NO_VECTOR_OPS pin).

- WOLFSSL_LINUXKM_VERBOSE_DEBUG: split the softirq WC_ACCEL_INHIBIT_E
  counter into per-class counters (softirq, hardirq, NMI, other),
  reported at module shutdown.

in non-VERBOSE builds with WC_SVR_DONT_USE_NATIVE_REG_BUFS (or in non-SVR
builds), generated object code is byte-identical to the previous revision.
This commit is contained in:
Daniel Pouzzner
2026-08-21 15:23:11 -05:00
parent da7186c45d
commit 8c9b2bb45b
5 changed files with 1308 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
linuxkm/linuxkm_wc_port.h
View File

@@ -259,6 +259,12 @@
#endif
#endif
#if defined(WC_SVR_DONT_USE_NATIVE_REG_BUFS)
#undef WC_SVR_USE_NATIVE_REG_BUFS
#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && !defined(WC_SVR_USE_NATIVE_REG_BUFS)
#define WC_SVR_USE_NATIVE_REG_BUFS
#endif
#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(7, 0, 0)
#if defined(HAVE_HASHDRBG) && \
defined(HAVE_ENTROPY_MEMUSE) && \
@@ -740,8 +746,13 @@
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \
defined(CONFIG_X86)
extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void);
extern void free_wolfcrypt_linuxkm_fpu_states(void);
extern __must_check int wc_linuxkm_allocate_svr_states(void);
extern void wc_linuxkm_free_svr_states(void);
WOLFSSL_API void wc_svr_disallowed_count_reset(void);
#ifdef WC_SVR_USE_NATIVE_REG_BUFS
WOLFSSL_LOCAL __must_check int wc_linuxkm_svr_native_is_ready(void);
#endif
WOLFSSL_API __must_check unsigned long long int wc_svr_disallowed_count_current(void);
WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
@@ -1247,9 +1258,9 @@
#ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
#ifdef CONFIG_X86
typeof(allocate_wolfcrypt_linuxkm_fpu_states) *allocate_wolfcrypt_linuxkm_fpu_states;
typeof(wc_linuxkm_allocate_svr_states) *wc_linuxkm_allocate_svr_states;
typeof(wc_can_save_vector_registers_x86) *wc_can_save_vector_registers_x86;
typeof(free_wolfcrypt_linuxkm_fpu_states) *free_wolfcrypt_linuxkm_fpu_states;
typeof(wc_linuxkm_free_svr_states) *wc_linuxkm_free_svr_states;
typeof(wc_restore_vector_registers_x86) *wc_restore_vector_registers_x86;
typeof(wc_save_vector_registers_x86) *wc_save_vector_registers_x86;
#else /* !CONFIG_X86 */
@@ -1606,9 +1617,9 @@
#define get_current WC_PIE_INDIRECT_SYM(get_current)
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
#define allocate_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(allocate_wolfcrypt_linuxkm_fpu_states)
#define wc_linuxkm_allocate_svr_states WC_PIE_INDIRECT_SYM(wc_linuxkm_allocate_svr_states)
#define wc_can_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_can_save_vector_registers_x86)
#define free_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(free_wolfcrypt_linuxkm_fpu_states)
#define wc_linuxkm_free_svr_states WC_PIE_INDIRECT_SYM(wc_linuxkm_free_svr_states)
#define wc_restore_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_restore_vector_registers_x86)
#define wc_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_save_vector_registers_x86)
#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)

9
linuxkm/lkcapi_sha_glue.c
View File

@@ -2123,7 +2123,9 @@ static int wc_linuxkm_rng_bank_init(struct wc_rng_bank *ctx)
int ret;
word32 flags = WC_RNG_BANK_FLAG_CAN_WAIT;
#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(7,0,0)
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \
!defined(WC_SVR_USE_NATIVE_REG_BUFS) && \
defined(HAVE_FIPS) && FIPS_VERSION3_LT(7,0,0)
/* before v7, the SHA-2 implementations couldn't dynamically switch between
* C and asm in a given wc_Sha256 instance.
*/
@@ -2205,6 +2207,11 @@ static struct wc_rng_bank_inst *linuxkm_get_drbg(struct wc_rng_bank *ctx) {
WC_RNG_BANK_FLAG_CAN_WAIT |
WC_RNG_BANK_FLAG_PREFER_AFFINITY_INST;
#ifdef WC_SVR_USE_NATIVE_REG_BUFS
if (wc_linuxkm_svr_native_is_ready())
flags |= WC_RNG_BANK_FLAG_AFFINITY_LOCK;
else
#endif
if (wc_linuxkm_can_block())
flags |= WC_RNG_BANK_FLAG_AFFINITY_LOCK;
else

10
linuxkm/module_hooks.c
View File

@@ -192,10 +192,14 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m)
m->irq_flags = irq_flags;
return 0;
}
if (irq_count() != 0) {
if (! wc_linuxkm_can_block()) {
/* Note, this catches calls while SAVE_VECTOR_REGISTERS()ed as
* required, because in_softirq() is always true while saved,
* even for WC_FPU_INHIBITED_FLAG contexts.
*
* It also catches non-interrupt atomic callers -- tasks holding a
* spinlock or running with IRQs off -- which must not reach the
* cond_resched() retry loop below.
*/
spin_lock_irqsave(&m->lock, irq_flags);
m->irq_flags = irq_flags;
@@ -1522,9 +1526,9 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread;
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
wolfssl_linuxkm_pie_redirect_table.allocate_wolfcrypt_linuxkm_fpu_states = allocate_wolfcrypt_linuxkm_fpu_states;
wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_allocate_svr_states = wc_linuxkm_allocate_svr_states;
wolfssl_linuxkm_pie_redirect_table.wc_can_save_vector_registers_x86 = wc_can_save_vector_registers_x86;
wolfssl_linuxkm_pie_redirect_table.free_wolfcrypt_linuxkm_fpu_states = free_wolfcrypt_linuxkm_fpu_states;
wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_free_svr_states = wc_linuxkm_free_svr_states;
wolfssl_linuxkm_pie_redirect_table.wc_restore_vector_registers_x86 = wc_restore_vector_registers_x86;
wolfssl_linuxkm_pie_redirect_table.wc_save_vector_registers_x86 = wc_save_vector_registers_x86;
#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)

1434
linuxkm/x86_vector_register_glue.c
View File

File diff suppressed because it is too large Load Diff

6
wolfcrypt/src/wc_port.c
View File

@@ -530,9 +530,9 @@ int wolfCrypt_Init(void)
#endif
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(WOLFSSL_LINUXKM)
ret = allocate_wolfcrypt_linuxkm_fpu_states();
ret = wc_linuxkm_allocate_svr_states();
if (ret != 0) {
WOLFSSL_MSG("allocate_wolfcrypt_linuxkm_fpu_states failed");
WOLFSSL_MSG("wc_linuxkm_allocate_svr_states failed");
WOLFCRYPT_INIT_RAISE_BAD_STATE();
}
#endif
@@ -892,7 +892,7 @@ int wolfCrypt_Cleanup(void)
wolfSSL_CleanupHandle();
#endif
#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(WOLFSSL_LINUXKM)
free_wolfcrypt_linuxkm_fpu_states();
wc_linuxkm_free_svr_states();
#endif
#ifdef HAVE_ENTROPY_MEMUSE