crypt/wolfssl
1
0
Fork 0
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-10-24 04:09:07 +03:00
Code Activity

linuxkm: RNG invalidation recovery, kernel mutex behavior, error-trace fixes.

Browse Source 
Fix error-trace machinery and add missing WC_NO_ERR_TRACE()s: errno-domain
comparisons in the PK and SHA glue and module_hooks (ENOMEM, EAGAIN, EINTR,
EBADMSG, and wolfCrypt INTERRUPTED_E/NOT_READY_E sites) now use the
-WC_NO_ERR_TRACE(E*) idiom, with .wolfssl_known_macro_extras updated to match.

Change default kernel "mutex" (spinlock) behavior: atomic-context callers now
never spin -- they trylock and fail fast -- and task-context lockers re-enable
preemption (and with it, migration) immediately after acquisition, rebalancing
the preempt count just before release.  The legacy behavior -- task-context
holders promoted to non-preemptible for the duration of the hold, permitting
atomic-context callers (except NMI) to spin-wait -- remains available via a new
WC_LINUXKM_SPIN_IN_ATOMIC macro, which is #error-incompatible with
CONFIG_PREEMPT_RT; the mutex state field (.irq_flags vs .preempt_reenabled)
follows the selected regime, and wc_lkm_UnlockMutex() moves out of line (with a
pie redirect table entry).

Add pre- and post-checks bracketing DRBG operations against the invalidation
recovery window (keyed off last_invalidation_at, stamped at invalidation).  The
pre-check works as a waiting room, keeping callers from generating a CAS scan
storm during the invalidation recovery interval (during which there are no
usable RNGs system-wide).  In the post-check, -EAGAIN passes through for another
lap, a window overrun returns -ETIMEDOUT, atomic-context callers get one-shot
-EBUSY (blockability is checked before signal state, because in hardirq context
`current` is an unrelated task), pending signals return -EINTR, and otherwise
the caller relaxes and retries.  Both checks have stubs so reduced
configurations build, and in-window messaging is ratelimited.

Add emergency reinitialization of the entropy daemon's root_rng: when the
recovery sweep finds the root invalidated, the daemon retries
wc_RNG_DRBG_Reseed_Now() with an uncredited timestamp nonce while the failure
remains retryable (BUSY_E, NOT_READY_E, ENTROPY_RT_E, ENTROPY_APT_E), relaxing
between attempts; for the default bank the reseed runs with preemption disabled
to maximize its chances.  If the reseed fails terminally, the daemon makes one
last-ditch recovery via wc_FreeRng() and wc_rng_bank_root_rng_init() (or
wc_InitRng() for a standalone root); a successful reinit always clears the
invalidated state.  If that too fails, the daemon exits, leaving bank instances
to self-reseed.  Outcomes are reported at severities differentiated by default
vs non-default bank (pr_notice/pr_info success, pr_emerg/pr_err failure).  The
daemon also now supports a standalone malloc'd root_rng where no bank features
are built, instantiated at startup and freed on exit.

Also:
* wc_linuxkm_drbg_generate_tfm(): retry loop around the invalidation window
  replaces the single-shot call; output buffer is zeroized on terminal error.
* wc__get_random_bytes(): honors the recovery window, and zeroizes its
  staging buffer both before retry re-entry and on the error exit.  An
  invalidation may be peculiar to this VM and leave the bytes live in another
  VM, so they are wiped when abandoned, as possible CSP elsewhere.
* wc_linuxkm_entropy_daemon(): the registered-leaf recovery pass is now
  additionally gated on WC_LINUXKM_HAVE_RNG_REGISTRY, matching its actual
  dependency; seed health test failures (ENTROPY_RT_E/ENTROPY_APT_E) on bank
  instances are now reported unconditionally at pr_warn_ratelimited severity
  ("entropy source suspect"), rather than pr_err only under WC_VERBOSE_RNG;
  the WC_RNG_DEBUG_STATS dump at daemon exit is removed (was redundant).
* wolfcrypt/test/test.c: SVR access-counter checks now additionally gated on
  new WC_SVR_HAVE_FLAGS, matching the configurations that actually maintain
  the counters.
This commit is contained in:
Daniel Pouzzner
2026-09-21 01:18:43 -05:00
parent 075b3341e4
commit 3d683a35ec
9 changed files with 515 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.wolfssl_known_macro_extras
View File

@@ -808,6 +808,7 @@ WC_HASH_CUSTOM_MIN_DIGEST_SIZE
WC_INIT_ERROR_WHEN_CONTENDED
WC_LINUXKM_NO_ENTROPY_DAEMON
WC_LINUXKM_NO_USE_HEAP_WRAPPERS
WC_LINUXKM_SPIN_IN_ATOMIC
WC_LINUXKM_SVR_NO_BATCHING
WC_MLDSA_NO_ASM
WC_MLKEM_KERNEL_ASM
@@ -1197,8 +1198,8 @@ WOLFSSL_TLS13_IGNORE_AEAD_LIMITS
WOLFSSL_TLS13_IGNORE_PT_ALERT_ON_ENC
WOLFSSL_TLS13_NULL_CIPHER_IN_DEFAULT
WOLFSSL_TLS13_SHA512
WOLFSSL_TLS13_TICKET_CHECK_PSK_MODES
WOLFSSL_TLS13_TICKET_BEFORE_FINISHED
WOLFSSL_TLS13_TICKET_CHECK_PSK_MODES
WOLFSSL_TLSX_PQC_MLKEM_STORE_PRIV_KEY
WOLFSSL_TRACK_MEMORY_FULL
WOLFSSL_TRAP_MALLOC_SZ

41
linuxkm/linuxkm_wc_port.h
View File

@@ -204,6 +204,7 @@
#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
struct wolfSSL_Mutex;
extern int wc_lkm_LockMutex(struct wolfSSL_Mutex* m);
extern int wc_lkm_UnlockMutex(struct wolfSSL_Mutex* m);
#endif
#ifndef WC_LINUXKM_INTR_SIGNALS
@@ -237,6 +238,7 @@
WC_SVR_FLAG_MAYBE_INHIBIT = 2,
WC_SVR_FLAG_FUZZ = 4
};
#define WC_SVR_HAVE_FLAGS
#if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || \
defined(WOLFSSL_SP_X86_64_ASM)
@@ -1519,6 +1521,7 @@
typeof(_cond_resched) *_cond_resched;
#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
typeof(wc_lkm_LockMutex) *wc_lkm_LockMutex;
typeof(wc_lkm_UnlockMutex) *wc_lkm_UnlockMutex;
#endif
typeof(wc_linuxkm_can_block) *wc_linuxkm_can_block;
@@ -2146,20 +2149,33 @@
*/
#include <linux/spinlock.h>
#if IS_ENABLED(CONFIG_PREEMPT_RT) && defined(WC_LINUXKM_SPIN_IN_ATOMIC)
#error WC_LINUXKM_SPIN_IN_ATOMIC is incompatible with CONFIG_PREEMPT_RT.
#endif
typedef struct wolfSSL_Mutex {
spinlock_t lock;
#ifdef WC_LINUXKM_SPIN_IN_ATOMIC
unsigned long irq_flags;
#else
int preempt_reenabled;
#endif
#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
unsigned int magic;
#endif
} wolfSSL_Mutex;
#ifdef WC_LINUXKM_SPIN_IN_ATOMIC
#define WC_LINUXKM_MUTEX_STATE_FIELD .irq_flags
#else
#define WC_LINUXKM_MUTEX_STATE_FIELD .preempt_reenabled
#endif
#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
#define WC_LINUXKM_SPINLOCK_MAGIC 1702166717U
#define WOLFSSL_MUTEX_INITIALIZER(lockname) { \
.lock =__SPIN_LOCK_UNLOCKED(lockname), \
.irq_flags = 0, \
WC_LINUXKM_MUTEX_STATE_FIELD = 0, \
.magic = WC_LINUXKM_SPINLOCK_MAGIC \
}
@@ -2167,7 +2183,7 @@
#define WOLFSSL_MUTEX_INITIALIZER(lockname) { \
.lock =__SPIN_LOCK_UNLOCKED(lockname), \
.irq_flags = 0 \
WC_LINUXKM_MUTEX_STATE_FIELD = 0 \
}
#endif
@@ -2180,7 +2196,11 @@
# else
spin_lock_init(&m->lock);
#endif
#ifdef WC_LINUXKM_SPIN_IN_ATOMIC
m->irq_flags = 0;
#else
m->preempt_reenabled = 0;
#endif
#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
m->magic = WC_LINUXKM_SPINLOCK_MAGIC;
#endif
@@ -2212,6 +2232,11 @@
return WC_PIE_INDIRECT_SYM(wc_lkm_LockMutex)(m);
}
static __always_inline int wc_UnLockMutex(wolfSSL_Mutex *m)
{
return WC_PIE_INDIRECT_SYM(wc_lkm_UnlockMutex)(m);
}
#else /* !WC_CONTAINERIZE_THIS */
static __must_check __always_inline int wc_LockMutex(wolfSSL_Mutex *m)
@@ -2219,18 +2244,12 @@
return wc_lkm_LockMutex(m);
}
#endif /* !WC_CONTAINERIZE_THIS */
static __always_inline int wc_UnLockMutex(wolfSSL_Mutex* m)
static __always_inline int wc_UnLockMutex(wolfSSL_Mutex *m)
{
#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
if ((m == NULL) || (m->magic != WC_LINUXKM_SPINLOCK_MAGIC))
return -1;
#endif
spin_unlock_irqrestore(&m->lock, m->irq_flags);
return 0;
return wc_lkm_UnlockMutex(m);
}
#endif /* !WC_CONTAINERIZE_THIS */
#endif
#ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT

2
linuxkm/lkcapi_dh_glue.c
View File

@@ -2903,7 +2903,7 @@ static int linuxkm_test_kpp_driver(const char * driver,
if (IS_ERR(tfm)) {
pr_err("error: allocating kpp algorithm %s failed: %d\n",
driver, (int)PTR_ERR(tfm));
if (PTR_ERR(tfm) == -ENOMEM)
if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM))
test_rc = MEMORY_E;
else
test_rc = BAD_FUNC_ARG;

2
linuxkm/lkcapi_ecdh_glue.c
View File

@@ -920,7 +920,7 @@ static int linuxkm_test_ecdh_nist_driver(const char * driver,
{
pr_err("error: allocating kpp algorithm %s failed: %d\n",
driver, (int)PTR_ERR(tfm));
if (PTR_ERR(tfm) == -ENOMEM)
if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM))
test_rc = MEMORY_E;
else
test_rc = BAD_FUNC_ARG;

8
linuxkm/lkcapi_ecdsa_glue.c
View File

@@ -1048,7 +1048,7 @@ static int linuxkm_test_ecdsa_nist_driver(const char * driver,
if (IS_ERR(tfm)) {
pr_err("error: allocating sig algorithm %s failed: %d\n",
driver, (int)PTR_ERR(tfm));
if (PTR_ERR(tfm) == -ENOMEM)
if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM))
test_rc = MEMORY_E;
else
test_rc = BAD_FUNC_ARG;
@@ -1123,7 +1123,7 @@ static int linuxkm_test_ecdsa_nist_driver(const char * driver,
ret = crypto_sig_verify(tfm, raw_sig, (unsigned int)sizeof(*raw_sig),
hash, hash_len);
if ((ret != -EBADMSG) && (ret != -EKEYREJECTED)) {
if ((ret != -WC_NO_ERR_TRACE(EBADMSG)) && (ret != -WC_NO_ERR_TRACE(EKEYREJECTED))) {
pr_err("error: crypto_sig_verify returned %d, expected %d or %d\n",
ret, -EBADMSG, -EKEYREJECTED);
test_rc = BAD_FUNC_ARG;
@@ -1193,7 +1193,7 @@ static int linuxkm_test_ecdsa_nist_driver(const char * driver,
{
pr_err("error: allocating akcipher algorithm %s failed: %d\n",
driver, (int)PTR_ERR(tfm));
if (PTR_ERR(tfm) == -ENOMEM)
if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM))
test_rc = MEMORY_E;
else
test_rc = BAD_FUNC_ARG;
@@ -1269,7 +1269,7 @@ static int linuxkm_test_ecdsa_nist_driver(const char * driver,
/* it should fail */
ret = crypto_akcipher_verify(req);
if (ret != -EBADMSG) {
if (ret != -WC_NO_ERR_TRACE(EBADMSG)) {
pr_err("error: crypto_akcipher_verify returned %d, expected %d\n",
ret, -EBADMSG);
test_rc = BAD_FUNC_ARG;

4
linuxkm/lkcapi_rsa_glue.c
View File

@@ -2736,7 +2736,7 @@ static int linuxkm_test_pkcs1pad_driver(const char * driver, int nbits,
else {
pr_err("error: allocating akcipher algorithm %s failed: %d\n",
driver, (int)PTR_ERR(tfm));
if (PTR_ERR(tfm) == -ENOMEM) {
if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM)) {
test_rc = MEMORY_E;
}
else {
@@ -3243,7 +3243,7 @@ static int linuxkm_test_pkcs1_driver(const char * driver, int nbits,
else {
pr_err("error: allocating sig algorithm %s failed: %d\n",
driver, (int)PTR_ERR(tfm));
if (PTR_ERR(tfm) == -ENOMEM) {
if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM)) {
test_rc = MEMORY_E;
}
else {

429
linuxkm/lkcapi_sha_glue.c
View File

@@ -2192,6 +2192,7 @@ static int linuxkm_affinity_unlock(void *arg) {
defined(WC_RNG_HAVE_FREE_HOOK) && defined(WC_RNG_HAVE_LOCK)
#define WC_LINUXKM_HAVE_RNG_REGISTRY
#define WC_LINUXKM_HAVE_RNG_INVALIDATION
/* Registry of every kernel-module RNG object needing state-invalidation
* coverage: banks (default and tfm-private) and long-lived process-context
@@ -2311,10 +2312,15 @@ static WARN_UNUSED_RESULT int wc_linuxkm_rng_registry_add_bank(struct wc_rng_ban
* and wake the daemon -- its loop-head check recovers the root first, and
* consumers recover per-instance through the NEEDS_RECOVERY_E protocol
* and the daemon's recovery pass. */
static volatile unsigned long last_invalidation_at;
static int wc_linuxkm_rng_state_invalidate(void) {
struct linuxkm_rng_object *obj;
int ret = 0;
last_invalidation_at = jiffies;
/* Process context (vmfork notifier / pm notifier); the registry mutex
* is sleepable and never taken from atomic context. */
mutex_lock(&wc_linuxkm_rng_registry_mutex);
@@ -2364,6 +2370,59 @@ static int wc_linuxkm_rng_state_invalidate(void) {
return 0;
}
static WC_INLINE int rng_invalidation_pre_check(WC_RNG *rng, size_t n) {
while (jiffies_to_msecs(jiffies - last_invalidation_at) < 1000) {
WC_RNG_lock_arg_t lock_state;
if (wc_RNG_lock_read(rng, &lock_state) != 0)
{
/* something's wrong, but it's not ours to fix. */
return 0;
}
if (! (lock_state & WC_RNG_LOCK_ENTROPY_INVALIDATED)) {
/* all is well, at least in terms of _ENTROPY_INVALIDATED. */
return 0;
}
if (! wc_linuxkm_can_block()) {
pr_notice_ratelimited(
"libwolfssl: RNG call (%zu B) from non-blockable "
"context with root RNG in invalidation recovery "
"(CPU %d, preempt_count 0x%x).\n",
n, raw_smp_processor_id(), preempt_count());
return -EBUSY;
}
if (wc_linuxkm_check_for_intr_signals() == WC_NO_ERR_TRACE(INTERRUPTED_E)) {
return -EINTR;
}
wc_linuxkm_relax_long_loop();
}
/* either not in invalidation/recovery at all, or it's been more than a
* second since the most recent invalidation. proceed as usual either
* way. */
return 0;
}
static WC_INLINE int rng_invalidation_post_check(const char *caller, int genret) {
if (genret != -WC_NO_ERR_TRACE(EAGAIN))
return genret;
if (jiffies_to_msecs(jiffies - last_invalidation_at) >= 1000) {
pr_err_ratelimited("%s: RNG invalidation recovery window exceeded; "
"failing RNG request.\n", caller);
return -ETIMEDOUT;
}
if (! wc_linuxkm_can_block()) {
pr_notice_ratelimited(
"libwolfssl: RNG call from non-blockable "
"context with root RNG in invalidation recovery "
"(CPU %d, preempt_count 0x%x).\n",
raw_smp_processor_id(), preempt_count());
return -EBUSY;
}
if (wc_linuxkm_check_for_intr_signals() == WC_NO_ERR_TRACE(INTERRUPTED_E))
return -EINTR;
wc_linuxkm_relax_long_loop();
return -EAGAIN;
}
/* Stock-kernel event coverage for the invalidation machinery: the kernel
* already broadcasts the two state-duplication events publicly -- VM fork
* (vmgenid, via the random_vmfork notifier chain, kernels >= 5.18) and
@@ -2525,23 +2584,38 @@ static struct module_attribute wc_linuxkm_rng_state_invalidate_attr =
#endif /* !HAVE_FIPS || FIPS_VERSION3_GE(7,0,0) */
#ifndef WC_LINUXKM_HAVE_RNG_INVALIDATION
static WC_INLINE int rng_invalidation_pre_check(WC_RNG *rng, size_t n) {
(void)rng;
(void)n;
return 0;
}
static WC_INLINE int rng_invalidation_post_check(const char *caller, int genret) {
(void)caller;
return genret;
}
#endif
#ifndef WC_LINUXKM_NO_ENTROPY_DAEMON
#ifdef WC_LINUXKM_HAVE_RNG_REGISTRY
#if defined(WC_LINUXKM_HAVE_RNG_REGISTRY) && defined(WC_LINUXKM_HAVE_RNG_INVALIDATION)
#if defined(WC_LINUXKM_VMGENID_POLL) || \
(defined(CONFIG_ACPI) && !IS_ENABLED(CONFIG_VMGENID))
/* Without CONFIG_VMGENID, we can only detect VM fork events by polling.
* Mainline gained vmgenid and the random_vmfork notifier chain together in
* kernel 5.18, so on older kernels and kernels with CONFIG_VMGENID configured
* off, there is no event to subscribe to -- but the ACPI VM Generation ID
* device (Microsoft spec; exposed by QEMU, Hyper-V, VMware) is still present,
* and its 16-byte counter changes exactly when the hypervisor
* forks/clones/restores the VM. The daemon polls it each iteration (a 16-byte
* compare of a memremap'd page -- effectively free) and, on change, invalidates
* all module RNG state and recovers its own root immediately, folding the new
* generation id into the credited recovery reseed as nonce. Detection latency
* is bounded by the daemon nap.
#if defined(CONFIG_ACPI) && \
(defined(WC_LINUXKM_VMGENID_POLL) || !IS_BUILTIN(CONFIG_VMGENID))
/* Without CONFIG_VMGENID, or with it modularized and possibly unloaded, we can
* only detect VM fork events by polling. Mainline gained vmgenid and the
* random_vmfork notifier chain together in kernel 5.18, so on older kernels and
* kernels with CONFIG_VMGENID configured off, there is no event to subscribe to
* -- but the ACPI VM Generation ID device (Microsoft spec; exposed by QEMU,
* Hyper-V, VMware) is still present, and its 16-byte counter changes exactly
* when the hypervisor forks/clones/restores the VM. The daemon polls it each
* iteration (a 16-byte compare of a memremap'd page -- effectively free) and,
* on change, invalidates all module RNG state and recovers its own root
* immediately, folding the new generation id into the credited recovery reseed
* as nonce. Detection latency is bounded by the daemon nap.
*
* All state is per-daemon, on the daemon's stack: wc_linuxkm_entropy_daemon()
* is threadsafe, and concurrent daemons discover, map, and poll
@@ -2648,9 +2722,9 @@ static void wc_linuxkm_vmgenid_poll_teardown(
}
st->state = 0;
}
#endif /* CONFIG_ACPI && !CONFIG_VMGENID */
#endif /* CONFIG_ACPI && !IS_BUILTIN(CONFIG_VMGENID) */
#endif /* WC_LINUXKM_HAVE_RNG_REGISTRY */
#endif /* WC_LINUXKM_HAVE_RNG_REGISTRY && WC_LINUXKM_HAVE_RNG_INVALIDATION */
/* Entropy-banking daemon for the default rng bank: cycles the bank's
* instances, keeping each DRBG's nextSeed aperture full so that
@@ -2736,9 +2810,30 @@ static int wc_linuxkm_entropy_daemon(void *arg)
if (WOLFSSL_ATOMIC_LOAD(bank->daemon_magic) != WC_LINUXKM_ENTROPY_DAEMON_MAGIC)
return -EINVAL;
struct WC_RNG *root_rng = wc_rng_bank_root_rng_get(bank);
struct WC_RNG *root_rng;
int root_rng_reseed_countdown = 0;
#if defined(WC_RNG_HAVE_RBGC) || defined(WC_RNG_HAVE_NEXT_SEED) || \
defined(WC_RNG_HAVE_POOL)
root_rng = wc_rng_bank_root_rng_get(bank);
#else
root_rng = (struct WC_RNG *)malloc(sizeof(*root_rng));
if (root_rng == NULL) {
ret = MEMORY_E;
}
else {
ret = wc_InitRng(root_rng);
if (ret != 0) {
free(root_rng);
root_rng = NULL;
}
}
if (ret != 0) {
pr_err("wc_linuxkm_entropy_daemon: could not instantiate root_rng "
"(code %d) -- continuing without.\n", ret);
}
#endif
#ifdef WC_RNG_HAVE_POOL
if (root_rng != NULL) {
/* One-time pool allocation for every instance, before any
@@ -2771,31 +2866,124 @@ static int wc_linuxkm_entropy_daemon(void *arg)
* event: the saturated reseedCtr from wc_RNG_invalidate_entropy()
* also forces this, but that write races our own generates (the
* root is unleased by design), so the flag is the authoritative
* signal and this the authoritative response. */
* signal and this is the authoritative response. */
if (root_rng != NULL) {
WC_RNG_lock_arg_t root_lock_state;
if ((wc_RNG_lock_read(root_rng, &root_lock_state) == 0) &&
(root_lock_state & WC_RNG_LOCK_ENTROPY_INVALIDATED))
WC_RNG_lock_arg_t root_lock_state = 0;
int inv_ret;
while (((inv_ret = wc_RNG_lock_read(root_rng, &root_lock_state)) == 0) &&
(root_lock_state & WC_RNG_LOCK_ENTROPY_INVALIDATED))
{
unsigned long uncredited_nonce = random_get_entropy();
int inv_ret;
#ifdef WC_VERBOSE_RNG
pr_info("wc_linuxkm_entropy_daemon: starting root recovery reseed.\n");
#endif
#if !defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) || defined(WC_SVR_USE_NATIVE_REG_BUFS)
if (bank->flags & WC_RNG_BANK_FLAG_DEFAULT_BANK) {
/* all RNGs depend on the default root reseed -- inhibit
* preemption to maximize our chances. */
preempt_disable();
}
#endif
inv_ret = wc_RNG_DRBG_Reseed_Now(
root_rng,
(byte *)&uncredited_nonce, (word32)sizeof uncredited_nonce);
#if !defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) || defined(WC_SVR_USE_NATIVE_REG_BUFS)
if (bank->flags & WC_RNG_BANK_FLAG_DEFAULT_BANK)
preempt_enable();
#endif
ForceZero(&uncredited_nonce, (word32)sizeof uncredited_nonce);
if (inv_ret == 0) {
#ifdef WC_VERBOSE_RNG
pr_info("wc_linuxkm_entropy_daemon: finished root recovery reseed.\n");
#endif
progress = 1;
break;
}
else
else {
pr_err_ratelimited("wc_entropyd: post-invalidation "
"root_rng reseed failed: %d\n", inv_ret);
"root_rng reseed failed: %d\n", inv_ret);
if (kthread_should_stop())
break;
if ((inv_ret != WC_NO_ERR_TRACE(BUSY_E)) &&
(inv_ret != WC_NO_ERR_TRACE(NOT_READY_E)) &&
(inv_ret != WC_NO_ERR_TRACE(ENTROPY_RT_E)) &&
(inv_ret != WC_NO_ERR_TRACE(ENTROPY_APT_E)))
{
break;
}
wc_linuxkm_relax_long_loop();
}
}
if (inv_ret != 0) {
if (kthread_should_stop()) {
/* prompt shutdown; skip sweeps; the pr_err above already
* notified of error condition. */
break;
}
else {
/* One last ditch effort to recover. */
unsigned long uncredited_nonce = random_get_entropy();
if (bank->flags & WC_RNG_BANK_FLAG_DEFAULT_BANK) {
pr_notice(
"wc_entropyd: default root_rng recovery via reseed "
"failed (%d) -- attempting reinit\n", inv_ret);
}
else {
pr_notice(
"wc_entropyd: root_rng recovery via reseed "
"failed (%d) -- attempting reinit\n", inv_ret);
}
inv_ret = wc_FreeRng(root_rng);
if (inv_ret == 0) {
#if defined(WC_RNG_HAVE_RBGC) || defined(WC_RNG_HAVE_NEXT_SEED) || \
defined(WC_RNG_HAVE_POOL)
inv_ret = wc_rng_bank_root_rng_init(
bank,
(byte *)&uncredited_nonce, (word32)sizeof uncredited_nonce,
NULL, 0, 0);
#else
inv_ret = wc_InitRng(root_rng);
#endif
}
ForceZero(&uncredited_nonce, (word32)sizeof uncredited_nonce);
if (inv_ret == 0) {
progress = 1;
if (bank->flags & WC_RNG_BANK_FLAG_DEFAULT_BANK) {
pr_notice("wc_entropyd: default root_rng reinitialization "
"succeeded.\n");
}
else {
pr_info("wc_entropyd: root_rng reinitialization "
"succeeded.\n");
}
/* Valid recovery: successful wc_InitRng() always clears
* WC_RNG_LOCK_ENTROPY_INVALIDATED and (of course) only
* succeeds if seeding succeeded -- fall through to the
* sweeps.
*/
}
else {
if (bank->flags & WC_RNG_BANK_FLAG_DEFAULT_BANK) {
pr_emerg(
"wc_entropyd: default root_rng reinitialization "
"failed (%d) -- daemon exiting; bank instances "
"will self-reseed\n", inv_ret);
}
else {
pr_err(
"wc_entropyd: root_rng reinitialization "
"failed (%d) -- daemon exiting; bank instances "
"will self-reseed\n", inv_ret);
}
break;
}
}
}
}
#endif
@@ -2972,7 +3160,8 @@ static int wc_linuxkm_entropy_daemon(void *arg)
}
#endif /* WC_RNG_HAVE_POOL */
#if defined(WC_RNG_HAVE_NEXT_SEED) && defined(WC_RNG_HAVE_RBGC)
#if defined(WC_LINUXKM_HAVE_RNG_REGISTRY) && \
defined(WC_RNG_HAVE_NEXT_SEED) && defined(WC_RNG_HAVE_RBGC)
/* registered-leaf pass: bank RBGC seeds from root_rng into
* long-lived leaves that are invalidated or chain-backed, so their
* next generate recovers/promotes in place
@@ -3009,7 +3198,7 @@ static int wc_linuxkm_entropy_daemon(void *arg)
&needs_recovery_snapshot, 0);
}
}
#endif /* WC_RNG_HAVE_NEXT_SEED && WC_RNG_HAVE_RBGC */
#endif /* WC_LINUXKM_HAVE_RNG_REGISTRY && WC_RNG_HAVE_NEXT_SEED && WC_RNG_HAVE_RBGC */
/* if we're coping with congestion hits, continue here, don't bog down
* in primary seed ops. */
@@ -3034,11 +3223,9 @@ static int wc_linuxkm_entropy_daemon(void *arg)
else if ((ret == WC_NO_ERR_TRACE(ENTROPY_RT_E)) ||
(ret == WC_NO_ERR_TRACE(ENTROPY_APT_E)))
{
#ifdef WC_VERBOSE_RNG
pr_err_ratelimited(
pr_warn_ratelimited(
"WARNING: wc_entropyd: seed health test failed on DRBG inst "
"%d: %d -- entropy source suspect\n", i, ret);
#endif
}
else {
pr_err_ratelimited(
@@ -3048,7 +3235,7 @@ static int wc_linuxkm_entropy_daemon(void *arg)
}
#endif /* WC_RNG_HAVE_NEXT_SEED */
next_pass:
next_pass:
if (progress) {
cond_resched();
@@ -3060,39 +3247,19 @@ static int wc_linuxkm_entropy_daemon(void *arg)
}
if (root_rng != NULL) {
#ifdef WC_RNG_DEBUG_STATS
struct wc_rng_debug_stats_snapshot s;
if (wc_rng_debug_stats_snap(&s, root_rng) == 0) {
pr_info("RNG INFO: wc_entropyd root total_bytes_requested=" WC_RNG_STAT_FMT "\n"
" total_bytes_produced=" WC_RNG_STAT_FMT
" total_requests=" WC_RNG_STAT_FMT "\n"
" reseeds=" WC_RNG_STAT_FMT
" stirs=" WC_RNG_STAT_FMT
" seed_failures=" WC_RNG_STAT_FMT "\n"
#ifdef WC_RNG_HAVE_NEXT_SEED
" nextstirs_banked=" WC_RNG_STAT_FMT
" nextstirs_redeemed=" WC_RNG_STAT_FMT "\n"
#endif
,
s._stats_total_bytes_requested,
s._stats_total_bytes_produced,
s._stats_total_requests,
s._stats_reseeds,
s._stats_stirs,
s._stats_seed_failures
#ifdef WC_RNG_HAVE_NEXT_SEED
,
s._stats_nextstirs_banked,
s._stats_nextstirs_redeemed
#endif
);
}
#endif /* WC_RNG_DEBUG_STATS */
#ifdef WC_LINUXKM_VMGENID_POLL
wc_linuxkm_vmgenid_poll_teardown(&vmgenid_poll_state);
#endif
}
#if !(defined(WC_RNG_HAVE_RBGC) || defined(WC_RNG_HAVE_NEXT_SEED) || \
defined(WC_RNG_HAVE_POOL))
if (root_rng != NULL) {
wc_FreeRng(root_rng);
free(root_rng);
}
#endif
return 0;
}
@@ -3450,19 +3617,24 @@ static struct wc_rng_bank_inst *linuxkm_get_drbg(struct wc_rng_bank *ctx) {
return ret;
}
static void linuxkm_put_drbg(struct wc_rng_bank_inst **drbg) {
static int linuxkm_put_drbg(struct wc_rng_bank_inst **drbg) {
int ret = wc_rng_bank_inst_checkin(drbg);
if (ret == WC_NO_ERR_TRACE(NEEDS_RECOVERY_E)) {
/* informational: checked in successfully; the instance is
* entropy-invalidated (e.g. a state-invalidation event landed
* mid-lease) and recovers via the checkout admissions or the
* patrol. */
* mid-lease) and recovers via the checkout admissions or the patrol.
* The caller is notified to give it the option of regenerating the
* random bytes generated under the previous lease. */
return -EAGAIN;
}
else if (ret != 0) {
pr_err("ERROR: wc_rng_bank_inst_checkin() in linuxkm_put_drbg() "
"returned err %d.\n", ret);
WC_DUMP_BACKTRACE_NONDEBUG;
return -EFAULT;
}
else
return 0;
}
#if defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT) && defined(HAVE_HASHDRBG)
@@ -3563,8 +3735,18 @@ static int wc_linuxkm_drbg_generate(struct wc_rng_bank *ctx,
wc_drbg_reseed_ctr_t cur_counter = 0;
struct wc_rng_bank_inst *drbg;
if (pr && !can_wait)
return -EAGAIN;
if (pr && !can_wait) {
/* -EBUSY is the module's meaning for "non-blockable context, service
* requires waiting" (rng_invalidation_{pre,post}_check()), and
* PR-from-atomic is exactly that -- prediction resistance mandates a
* fresh gather, and the current procedure for PR gathering arranges for
* preemptibility, which is incompatible with calls from atomic
* context. Returning -EBUSY here keeps the kernel layer's error
* language two-valued and clean: -EBUSY = your context can't be served,
* -EAGAIN = regenerate protocol.
*/
return -EBUSY;
}
drbg = linuxkm_get_drbg(ctx);
@@ -3867,7 +4049,11 @@ static int wc_linuxkm_drbg_generate(struct wc_rng_bank *ctx,
out:
linuxkm_put_drbg(&drbg);
{
int checkin_ret = linuxkm_put_drbg(&drbg);
if (ret == 0)
ret = checkin_ret;
}
return ret;
}
@@ -3876,14 +4062,28 @@ static int wc_linuxkm_drbg_generate_tfm(struct crypto_rng *tfm,
const u8 *src, unsigned int slen,
u8 *dst, unsigned int dlen)
{
int ret;
if (tfm->base.__crt_alg->cra_init != wc_linuxkm_drbg_init_tfm)
{
pr_err_once("BUG: mismatched tfm.\n");
return -EFAULT;
}
return wc_linuxkm_drbg_generate((struct wc_rng_bank *)crypto_rng_ctx(tfm),
src, slen, dst, dlen, 0 /* pr */);
for (;;) {
ret = wc_linuxkm_drbg_generate((struct wc_rng_bank *)crypto_rng_ctx(tfm),
src, slen, dst, dlen, 0 /* pr */);
ret = rng_invalidation_post_check(__func__, ret);
if (ret != -WC_NO_ERR_TRACE(EAGAIN))
break;
}
/* Failure may be due to an invalidation, which may be unique to this VM,
* leaving these bytes live in another VM -- wipe them here. */
if (ret != 0)
ForceZero(dst, dlen);
return ret;
}
static int wc_linuxkm_drbg_seed(struct wc_rng_bank *ctx,
@@ -3991,10 +4191,38 @@ static int wc__get_random_bytes(void *buf, size_t len)
return -ECANCELED;
}
else {
#ifdef WC_LINUXKM_HAVE_RNG_INVALIDATION
/* if in a recovery window, defer the request until done, if possible. */
try_again:
ret = rng_invalidation_pre_check(&current_default_wc_rng_bank->root_rng, len);
if (ret != 0) {
(void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
return ret;
}
#endif /* WC_LINUXKM_HAVE_RNG_INVALIDATION */
ret = wc_linuxkm_drbg_generate(current_default_wc_rng_bank,
NULL, 0, buf, (unsigned int)len, 0 /* pr */);
#ifdef WC_LINUXKM_HAVE_RNG_INVALIDATION
if ((ret == -WC_NO_ERR_TRACE(EAGAIN)) &&
(jiffies_to_msecs(jiffies - last_invalidation_at) < 1000))
{
/* An invalidation may be unique to this VM, leaving these bytes
* live in another VM -- wipe them here. */
ForceZero(buf, (word32)len);
goto try_again;
}
#endif /* WC_LINUXKM_HAVE_RNG_INVALIDATION */
(void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
if (ret) {
if (ret != 0) {
/* An invalidation may be unique to this VM, leaving these bytes
* live in another VM -- wipe them here. */
ForceZero(buf, (word32)len);
#ifdef WOLFSSL_LINUXKM_GET_RANDOM_NO_FALLTHROUGH
pr_emerg_ratelimited("ERROR: FIPS RNG source failed: "
"wc__get_random_bytes(): wc_linuxkm_drbg_generate() "
@@ -4024,7 +4252,14 @@ static ssize_t wc_get_random_bytes_user(struct iov_iter *iter) {
"wc_get_random_bytes_user() returned %ld.\n", ret);
return -EIO; /* no fallthrough to native randomness */
}
else {
ret = rng_invalidation_pre_check(&current_default_wc_rng_bank->root_rng, iov_iter_count(iter));
if (ret != 0) {
(void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
return ret;
}
{
size_t this_copied, total_copied = 0;
byte *block;
byte block_small[WC_SHA256_BLOCK_SIZE];
@@ -4045,9 +4280,15 @@ static ssize_t wc_get_random_bytes_user(struct iov_iter *iter) {
size_t n = min_t(size_t, iov_iter_count(iter), block_size);
ret = wc_linuxkm_drbg_generate(current_default_wc_rng_bank,
NULL, 0, block, n, 0 /* pr */);
ret = rng_invalidation_post_check(__func__, ret);
if (ret == -WC_NO_ERR_TRACE(EAGAIN))
continue;
if (unlikely(ret != 0)) {
pr_emerg_ratelimited("ERROR: wc_get_random_bytes_user() "
"wc_linuxkm_drbg_generate() returned %ld.\n", ret);
if (ret != -WC_NO_ERR_TRACE(EINTR)) {
pr_emerg_ratelimited(
"ERROR: %s: wc_linuxkm_drbg_generate() returned %ld.\n",
__func__, ret);
}
break;
}
@@ -4065,6 +4306,10 @@ static ssize_t wc_get_random_bytes_user(struct iov_iter *iter) {
cond_resched();
}
/* NEEDS_RECOVERY_E here is a duplicate of the epoch flag: daemon is
* already woken, gates already armed, and this caller's only remaining
* duty is honest accounting of total_copied. Next read parks at
* pre_check. */
(void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
ForceZero(block, block_size);
@@ -4074,16 +4319,14 @@ static ssize_t wc_get_random_bytes_user(struct iov_iter *iter) {
if (total_copied == 0) {
if (ret == 0)
ret = -EFAULT;
else {
ret = -EIO;
}
return -EFAULT;
else if (ret == -WC_NO_ERR_TRACE(EINTR))
return -ERESTARTSYS;
else
return ret;
}
if (total_copied != 0)
ret = (ssize_t)total_copied; /* partial success wins */
return ret;
return (ssize_t)total_copied;
}
__builtin_unreachable();
}
@@ -4102,7 +4345,14 @@ static ssize_t wc_extract_crng_user(void __user *buf, size_t nbytes) {
"wc_extract_crng_user() returned %ld.\n", ret);
return -EIO; /* no fallthrough to native randomness */
}
else {
ret = rng_invalidation_pre_check(&current_default_wc_rng_bank->root_rng, nbytes);
if (ret != 0) {
(void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
return ret;
}
{
size_t this_copied, total_copied = 0;
byte *block;
byte block_small[WC_SHA256_BLOCK_SIZE];
@@ -4123,9 +4373,15 @@ static ssize_t wc_extract_crng_user(void __user *buf, size_t nbytes) {
size_t n = min_t(size_t, nbytes - total_copied, block_size);
ret = wc_linuxkm_drbg_generate(current_default_wc_rng_bank,
NULL, 0, block, n, 0 /* pr */);
ret = rng_invalidation_post_check(__func__, ret);
if (ret == -WC_NO_ERR_TRACE(EAGAIN))
continue;
if (unlikely(ret != 0)) {
pr_emerg_ratelimited("ERROR: wc_extract_crng_user() "
"wc_linuxkm_drbg_generate() returned %ld.\n", ret);
if (ret != -WC_NO_ERR_TRACE(EINTR)) {
pr_emerg_ratelimited(
"ERROR: %s: wc_linuxkm_drbg_generate() returned %ld.\n",
__func__, ret);
}
break;
}
@@ -4153,15 +4409,14 @@ static ssize_t wc_extract_crng_user(void __user *buf, size_t nbytes) {
if (total_copied == 0) {
if (ret == 0)
ret = -EFAULT;
return -EFAULT;
else if (ret == -WC_NO_ERR_TRACE(EINTR))
return -ERESTARTSYS;
else
ret = -EIO;
return ret;
}
if (total_copied != 0)
ret = (ssize_t)total_copied; /* partial success wins */
return ret;
return (ssize_t)total_copied;
}
__builtin_unreachable();
}

174
linuxkm/module_hooks.c
View File

@@ -61,26 +61,31 @@
#ifdef WOLFSSL_DEBUG_TRACE_ERROR_CODES
enum linux_errcodes {
my_EINVAL = EINVAL,
my_ENOMEM = ENOMEM,
my_EBADMSG = EBADMSG,
my_ENOKEY = ENOKEY,
my_EFAULT = EFAULT,
my_EAFNOSUPPORT = EAFNOSUPPORT,
my_EOVERFLOW = EOVERFLOW,
my_EOPNOTSUPP = EOPNOTSUPP,
my_EDEADLK = EDEADLK,
my_EAGAIN = EAGAIN,
my_EBUSY = EBUSY,
my_ECANCELED = ECANCELED,
my_EINTR = EINTR,
my_ELIBBAD = ELIBBAD,
my_ENODATA = ENODATA,
my_ENODEV = ENODEV,
my_EPERM = EPERM,
my_EFBIG = EFBIG
CONST_NUM_ERR_EINVAL = EINVAL,
CONST_NUM_ERR_ENOMEM = ENOMEM,
CONST_NUM_ERR_EBADMSG = EBADMSG,
CONST_NUM_ERR_ENOKEY = ENOKEY,
CONST_NUM_ERR_EFAULT = EFAULT,
CONST_NUM_ERR_EAFNOSUPPORT = EAFNOSUPPORT,
CONST_NUM_ERR_EOVERFLOW = EOVERFLOW,
CONST_NUM_ERR_EOPNOTSUPP = EOPNOTSUPP,
CONST_NUM_ERR_EDEADLK = EDEADLK,
CONST_NUM_ERR_EAGAIN = EAGAIN,
CONST_NUM_ERR_EBUSY = EBUSY,
CONST_NUM_ERR_ECANCELED = ECANCELED,
CONST_NUM_ERR_EINTR = EINTR,
CONST_NUM_ERR_ELIBBAD = ELIBBAD,
CONST_NUM_ERR_ENODATA = ENODATA,
CONST_NUM_ERR_ENODEV = ENODEV,
CONST_NUM_ERR_EPERM = EPERM,
CONST_NUM_ERR_EFBIG = EFBIG,
CONST_NUM_ERR_ETIMEDOUT = ETIMEDOUT,
CONST_NUM_ERR_ERESTARTSYS = ERESTARTSYS,
CONST_NUM_ERR_EKEYREJECTED = EKEYREJECTED
};
/* transform the Linux error codes into enums like the wolfCrypt error
* codes. */
#undef EINVAL
#undef ENOMEM
#undef EBADMSG
@@ -99,26 +104,57 @@
#undef ENODEV
#undef EPERM
#undef EFBIG
#undef ETIMEDOUT
#undef ERESTARTSYS
#undef EKEYREJECTED
#define EINVAL WC_ERR_TRACE(my_EINVAL)
#define ENOMEM WC_ERR_TRACE(my_ENOMEM)
#define EBADMSG WC_ERR_TRACE(my_EBADMSG)
#define ENOKEY WC_ERR_TRACE(my_ENOKEY)
#define EFAULT WC_ERR_TRACE(my_EFAULT)
#define EAFNOSUPPORT WC_ERR_TRACE(my_EAFNOSUPPORT)
#define EOVERFLOW WC_ERR_TRACE(my_EOVERFLOW)
#define EOPNOTSUPP WC_ERR_TRACE(my_EOPNOTSUPP)
#define EDEADLK WC_ERR_TRACE(my_EDEADLK)
#define EAGAIN WC_ERR_TRACE(my_EAGAIN)
#define EBUSY WC_ERR_TRACE(my_EBUSY)
#define ECANCELED WC_ERR_TRACE(my_ECANCELED)
#define EINTR WC_ERR_TRACE(my_EINTR)
#define ELIBBAD WC_ERR_TRACE(my_ELIBBAD)
#define ENODATA WC_ERR_TRACE(my_ENODATA)
#define ENODEV WC_ERR_TRACE(my_ENODEV)
#define EPERM WC_ERR_TRACE(my_EPERM)
#define EFBIG WC_ERR_TRACE(my_EFBIG)
#endif
enum {
EINVAL = CONST_NUM_ERR_EINVAL,
ENOMEM = CONST_NUM_ERR_ENOMEM,
EBADMSG = CONST_NUM_ERR_EBADMSG,
ENOKEY = CONST_NUM_ERR_ENOKEY,
EFAULT = CONST_NUM_ERR_EFAULT,
EAFNOSUPPORT = CONST_NUM_ERR_EAFNOSUPPORT,
EOVERFLOW = CONST_NUM_ERR_EOVERFLOW,
EOPNOTSUPP = CONST_NUM_ERR_EOPNOTSUPP,
EDEADLK = CONST_NUM_ERR_EDEADLK,
EAGAIN = CONST_NUM_ERR_EAGAIN,
EBUSY = CONST_NUM_ERR_EBUSY,
ECANCELED = CONST_NUM_ERR_ECANCELED,
EINTR = CONST_NUM_ERR_EINTR,
ELIBBAD = CONST_NUM_ERR_ELIBBAD,
ENODATA = CONST_NUM_ERR_ENODATA,
ENODEV = CONST_NUM_ERR_ENODEV,
EPERM = CONST_NUM_ERR_EPERM,
EFBIG = CONST_NUM_ERR_EFBIG,
ETIMEDOUT = CONST_NUM_ERR_ETIMEDOUT,
ERESTARTSYS = CONST_NUM_ERR_ERESTARTSYS,
EKEYREJECTED = CONST_NUM_ERR_EKEYREJECTED
};
#define EINVAL WC_ERR_TRACE(EINVAL)
#define ENOMEM WC_ERR_TRACE(ENOMEM)
#define EBADMSG WC_ERR_TRACE(EBADMSG)
#define ENOKEY WC_ERR_TRACE(ENOKEY)
#define EFAULT WC_ERR_TRACE(EFAULT)
#define EAFNOSUPPORT WC_ERR_TRACE(EAFNOSUPPORT)
#define EOVERFLOW WC_ERR_TRACE(EOVERFLOW)
#define EOPNOTSUPP WC_ERR_TRACE(EOPNOTSUPP)
#define EDEADLK WC_ERR_TRACE(EDEADLK)
#define EAGAIN WC_ERR_TRACE(EAGAIN)
#define EBUSY WC_ERR_TRACE(EBUSY)
#define ECANCELED WC_ERR_TRACE(ECANCELED)
#define EINTR WC_ERR_TRACE(EINTR)
#define ELIBBAD WC_ERR_TRACE(ELIBBAD)
#define ENODATA WC_ERR_TRACE(ENODATA)
#define ENODEV WC_ERR_TRACE(ENODEV)
#define EPERM WC_ERR_TRACE(EPERM)
#define EFBIG WC_ERR_TRACE(EFBIG)
#define ETIMEDOUT WC_ERR_TRACE(ETIMEDOUT)
#define ERESTARTSYS WC_ERR_TRACE(ERESTARTSYS)
#define EKEYREJECTED WC_ERR_TRACE(EKEYREJECTED)
#endif /* WOLFSSL_DEBUG_TRACE_ERROR_CODES */
#ifdef CONFIG_HAVE_KPROBES
static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name);
@@ -256,27 +292,41 @@ extern int wolfcrypt_benchmark_main(int argc, char** argv);
#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
int wc_lkm_LockMutex(wolfSSL_Mutex* m)
{
#ifdef WC_LINUXKM_SPIN_IN_ATOMIC
unsigned long irq_flags;
#endif
#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
if ((m == NULL) || (m->magic != WC_LINUXKM_SPINLOCK_MAGIC))
return BAD_FUNC_ARG;
#endif
/* first, try the cheap way. */
#ifdef WC_LINUXKM_SPIN_IN_ATOMIC
if (spin_trylock_irqsave(&m->lock, irq_flags)) {
m->irq_flags = irq_flags;
return 0;
}
#else
if (spin_trylock(&m->lock)) {
if (wc_linuxkm_can_block()) {
preempt_enable();
m->preempt_reenabled = 1;
}
return 0;
}
#endif
if (in_nmi())
return BUSY_E;
if (! wc_linuxkm_can_block()) {
#if IS_ENABLED(CONFIG_PREEMPT_RT)
#ifndef WC_LINUXKM_SPIN_IN_ATOMIC
/* RT spinlock_t is a sleeping rtmutex; an atomic caller has no legal
* wait -- the entry trylock was its one shot. */
return BUSY_E;
#else /* !CONFIG_PREEMPT_RT */
/* Note, this catches calls while SAVE_VECTOR_REGISTERS()ed as
* required, because in_softirq() is always true while saved,
* even for WC_FPU_INHIBITED_FLAG contexts.
#else /* WC_LINUXKM_SPIN_IN_ATOMIC */
/* Note, this catches calls while SAVE_VECTOR_REGISTERS()ed, because
* in_softirq() is always true while saved, even for
* WC_FPU_INHIBITED_FLAG contexts.
*
* It also catches non-interrupt atomic callers -- tasks holding a
* spinlock or running with IRQs off -- which must not reach the
@@ -285,7 +335,7 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m)
spin_lock_irqsave(&m->lock, irq_flags);
m->irq_flags = irq_flags;
return 0;
#endif /* !CONFIG_PREEMPT_RT */
#endif /* WC_LINUXKM_SPIN_IN_ATOMIC */
}
else {
for (;;) {
@@ -298,14 +348,49 @@ int wc_lkm_LockMutex(wolfSSL_Mutex* m)
* but a low-priority holder under a high-priority RT poller
* inherits nothing and the loop degrades to prioritized
* busy-wait. */
#ifdef WC_LINUXKM_SPIN_IN_ATOMIC
/* When this path is configured, task context lockers are promoted
* to non-preemptible context for the duration of the mutex hold,
* allowing atomic context callers (except NMI) to spin-wait without
* risk of deadlock. */
if (spin_trylock_irqsave(&m->lock, irq_flags)) {
m->irq_flags = irq_flags;
return 0;
}
#else
/* When this path is configured, we re-enable preemption and
* migration for regular tasks while they hold the mutex. Atomic
* context callers must correspondingly use one-try as above,
* otherwise deadlocks are inevitable.
*/
if (spin_trylock(&m->lock)) {
preempt_enable();
m->preempt_reenabled = 1;
return 0;
}
#endif
}
}
__builtin_unreachable();
}
int wc_lkm_UnlockMutex(wolfSSL_Mutex* m) {
#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
if ((m == NULL) || (m->magic != WC_LINUXKM_SPINLOCK_MAGIC))
return BAD_FUNC_ARG;
#endif
#ifdef WC_LINUXKM_SPIN_IN_ATOMIC
spin_unlock_irqrestore(&m->lock, m->irq_flags);
#else
if (m->preempt_reenabled) {
m->preempt_reenabled = 0;
preempt_disable();
}
spin_unlock(&m->lock);
#endif
return 0;
}
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0)
@@ -2034,6 +2119,7 @@ static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
wolfssl_linuxkm_pie_redirect_table.wc_lkm_LockMutex = wc_lkm_LockMutex;
wolfssl_linuxkm_pie_redirect_table.wc_lkm_UnlockMutex = wc_lkm_UnlockMutex;
#endif
#ifdef CONFIG_ARM64
@@ -2187,7 +2273,7 @@ static int updateFipsHash(void)
}
if (IS_ERR(tfm)) {
if (PTR_ERR(tfm) == -ENOMEM) {
if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM)) {
pr_err("ERROR: crypto_alloc_shash failed: out of memory\n");
ret = MEMORY_E;
} else if (PTR_ERR(tfm) == -ENOENT) {

5
wolfcrypt/test/test.c
View File

@@ -29125,7 +29125,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t random_bank_test(void)
#endif
ret = wc_rng_bank_init(bank, WC_RNG_BANK_STATIC_SIZE,
#ifndef DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON
#if !defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) && defined(WC_SVR_HAVE_FLAGS)
WC_RNG_BANK_FLAG_NO_VECTOR_OPS |
#endif
WC_RNG_BANK_FLAG_QUIET |
@@ -29168,7 +29168,8 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t random_bank_test(void)
defined(WC_NO_INTERNAL_FUNCTION_POINTERS) && \
defined(HAVE_FIPS) && \
FIPS_VERSION3_LT(7,0,0) && \
!defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON)
!defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) && \
defined(WC_SVR_HAVE_FLAGS)
#ifdef WOLFSSL_DRBG_SHA512
if (rng_inst->rng.drbgType == WC_DRBG_SHA512) {