/* module_hooks.c -- module load/unload hooks for libwolfssl.ko * * Copyright (C) 2006-2026 wolfSSL Inc. * * This file is part of wolfSSL. * * wolfSSL is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * * wolfSSL is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ #define WOLFSSL_LINUXKM_NEED_LINUX_CURRENT #include #include #ifndef WOLFSSL_LICENSE #define WOLFSSL_LICENSE "GPL" #endif #ifdef WOLFCRYPT_ONLY #include #else #include #endif #ifdef HAVE_FIPS #ifdef USE_CONTESTMUTEX #error USE_CONTESTMUTEX is incompatible with WOLFSSL_LINUXKM #endif #ifdef WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE #include #endif #include #if defined(WOLFSSL_FIPS_DEV_NO_POST) && !defined(WC_USE_PIE_FENCEPOSTS_FOR_FIPS) #define WC_USE_PIE_FENCEPOSTS_FOR_FIPS #endif #endif #if !defined(NO_CRYPT_TEST) || defined(LINUXKM_LKCAPI_REGISTER) #include #endif #ifdef HAVE_ENTROPY_MEMUSE #include #endif #include #include #ifdef NO_INLINE #include #else #define WOLFSSL_MISC_INCLUDED #include #endif #ifdef WOLFSSL_DEBUG_TRACE_ERROR_CODES enum linux_errcodes { CONST_NUM_ERR_EINVAL = EINVAL, CONST_NUM_ERR_ENOMEM = ENOMEM, CONST_NUM_ERR_EBADMSG = EBADMSG, CONST_NUM_ERR_ENOKEY = ENOKEY, CONST_NUM_ERR_EFAULT = EFAULT, CONST_NUM_ERR_EAFNOSUPPORT = EAFNOSUPPORT, CONST_NUM_ERR_EOVERFLOW = EOVERFLOW, CONST_NUM_ERR_EOPNOTSUPP = EOPNOTSUPP, CONST_NUM_ERR_EDEADLK = EDEADLK, CONST_NUM_ERR_EAGAIN = EAGAIN, CONST_NUM_ERR_EBUSY = EBUSY, CONST_NUM_ERR_ECANCELED = ECANCELED, CONST_NUM_ERR_EINTR = EINTR, CONST_NUM_ERR_ELIBBAD = ELIBBAD, CONST_NUM_ERR_ENODATA = ENODATA, CONST_NUM_ERR_ENODEV = ENODEV, CONST_NUM_ERR_EPERM = EPERM, CONST_NUM_ERR_EFBIG = EFBIG, CONST_NUM_ERR_ETIMEDOUT = ETIMEDOUT, CONST_NUM_ERR_ERESTARTSYS = ERESTARTSYS, CONST_NUM_ERR_EKEYREJECTED = EKEYREJECTED }; /* transform the Linux error codes into enums like the wolfCrypt error * codes. */ #undef EINVAL #undef ENOMEM #undef EBADMSG #undef ENOKEY #undef EFAULT #undef EAFNOSUPPORT #undef EOVERFLOW #undef EOPNOTSUPP #undef EDEADLK #undef EAGAIN #undef EBUSY #undef ECANCELED #undef EINTR #undef ELIBBAD #undef ENODATA #undef ENODEV #undef EPERM #undef EFBIG #undef ETIMEDOUT #undef ERESTARTSYS #undef EKEYREJECTED enum { EINVAL = CONST_NUM_ERR_EINVAL, ENOMEM = CONST_NUM_ERR_ENOMEM, EBADMSG = CONST_NUM_ERR_EBADMSG, ENOKEY = CONST_NUM_ERR_ENOKEY, EFAULT = CONST_NUM_ERR_EFAULT, EAFNOSUPPORT = CONST_NUM_ERR_EAFNOSUPPORT, EOVERFLOW = CONST_NUM_ERR_EOVERFLOW, EOPNOTSUPP = CONST_NUM_ERR_EOPNOTSUPP, EDEADLK = CONST_NUM_ERR_EDEADLK, EAGAIN = CONST_NUM_ERR_EAGAIN, EBUSY = CONST_NUM_ERR_EBUSY, ECANCELED = CONST_NUM_ERR_ECANCELED, EINTR = CONST_NUM_ERR_EINTR, ELIBBAD = CONST_NUM_ERR_ELIBBAD, ENODATA = CONST_NUM_ERR_ENODATA, ENODEV = CONST_NUM_ERR_ENODEV, EPERM = CONST_NUM_ERR_EPERM, EFBIG = CONST_NUM_ERR_EFBIG, ETIMEDOUT = CONST_NUM_ERR_ETIMEDOUT, ERESTARTSYS = CONST_NUM_ERR_ERESTARTSYS, EKEYREJECTED = CONST_NUM_ERR_EKEYREJECTED }; #define EINVAL WC_ERR_TRACE(EINVAL) #define ENOMEM WC_ERR_TRACE(ENOMEM) #define EBADMSG WC_ERR_TRACE(EBADMSG) #define ENOKEY WC_ERR_TRACE(ENOKEY) #define EFAULT WC_ERR_TRACE(EFAULT) #define EAFNOSUPPORT WC_ERR_TRACE(EAFNOSUPPORT) #define EOVERFLOW WC_ERR_TRACE(EOVERFLOW) #define EOPNOTSUPP WC_ERR_TRACE(EOPNOTSUPP) #define EDEADLK WC_ERR_TRACE(EDEADLK) #define EAGAIN WC_ERR_TRACE(EAGAIN) #define EBUSY WC_ERR_TRACE(EBUSY) #define ECANCELED WC_ERR_TRACE(ECANCELED) #define EINTR WC_ERR_TRACE(EINTR) #define ELIBBAD WC_ERR_TRACE(ELIBBAD) #define ENODATA WC_ERR_TRACE(ENODATA) #define ENODEV WC_ERR_TRACE(ENODEV) #define EPERM WC_ERR_TRACE(EPERM) #define EFBIG WC_ERR_TRACE(EFBIG) #define ETIMEDOUT WC_ERR_TRACE(ETIMEDOUT) #define ERESTARTSYS WC_ERR_TRACE(ERESTARTSYS) #define EKEYREJECTED WC_ERR_TRACE(EKEYREJECTED) #endif /* WOLFSSL_DEBUG_TRACE_ERROR_CODES */ #ifdef CONFIG_HAVE_KPROBES static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name); #define WC_LINUXKM_HAVE_MY_KALLSYMS_LOOKUP_NAME #endif static int wolfcrypt_inited; #ifndef WOLFCRYPT_ONLY static int wolfssl_inited; #endif static void wc_linuxkm_deinstall(void); static int libwolfssl_cleanup(void) { int ret; wc_linuxkm_deinstall(); #ifndef WOLFCRYPT_ONLY if (wolfssl_inited) { wolfssl_inited = 0; ret = wolfSSL_Cleanup(); if (ret != WOLFSSL_SUCCESS) pr_err("ERROR: wolfSSL_Cleanup() failed: %s\n", wc_GetErrorString(ret)); else pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n"); } #endif if (wolfcrypt_inited) { wolfcrypt_inited = 0; ret = wolfCrypt_Cleanup(); } else ret = 0; if (ret != 0) pr_err("ERROR: wolfCrypt_Cleanup() failed: %s\n", wc_GetErrorString(ret)); else pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n"); #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(HAVE_FIPS) wc_linuxkm_free_svr_states(); #endif return ret; } #if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) /* failsafe definitions for FIPS <5.3 */ #ifndef FIPS_IN_CORE_DIGEST_SIZE #ifndef NO_SHA256 #define FIPS_IN_CORE_DIGEST_SIZE WC_SHA256_DIGEST_SIZE #define FIPS_IN_CORE_HASH_TYPE WC_SHA256 #elif defined(WOLFSSL_SHA384) #define FIPS_IN_CORE_DIGEST_SIZE WC_SHA384_DIGEST_SIZE #define FIPS_IN_CORE_HASH_TYPE WC_SHA384 #else #error Unsupported FIPS hash alg. #endif #endif #ifndef FIPS_IN_CORE_KEY_SZ #define FIPS_IN_CORE_KEY_SZ FIPS_IN_CORE_DIGEST_SIZE #endif #ifndef FIPS_IN_CORE_VERIFY_SZ #define FIPS_IN_CORE_VERIFY_SZ FIPS_IN_CORE_DIGEST_SIZE #endif #if FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE) extern char verifyCore[FIPS_IN_CORE_DIGEST_SIZE*2 + 1]; #endif #endif #ifdef WC_SYM_RELOC_TABLES #ifdef DEBUG_LINUXKM_PIE_SUPPORT /* cheap portable ad-hoc hash function to confirm bitwise stability of the PIE * binary image. */ static unsigned int hash_span(const u8 *start, const u8 *end, unsigned int sum) { WC_SANITIZE_DISABLE(); while (start < end) { unsigned int rotate_by; sum ^= *start++; rotate_by = (sum ^ (sum >> 5)) & 31; sum = (sum << rotate_by) | (sum >> (32 - rotate_by)); } WC_SANITIZE_ENABLE(); return sum; } struct wc_reloc_counts reloc_counts = {}; #endif /* DEBUG_LINUXKM_PIE_SUPPORT */ static int set_up_wolfssl_linuxkm_pie_redirect_table(void); #ifdef HAVE_FIPS extern const unsigned int wolfCrypt_FIPS_ro_start[]; extern const unsigned int wolfCrypt_FIPS_ro_end[]; #endif #endif /* WC_SYM_RELOC_TABLES */ #if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) static void lkmFipsCb(int ok, int err, const char* hash) { if ((! ok) || (err != 0)) pr_err("ERROR: libwolfssl FIPS error: %s\n", wc_GetErrorString(err)); if (err == WC_NO_ERR_TRACE(IN_CORE_FIPS_E)) { if (hash) { pr_err("In-core integrity hash check failure.\n" "Update FIPS hash with \"make module-update-fips-hash FIPS_HASH=%s\".\n", hash); } else { pr_err("In-core integrity hash check failure.\n"); pr_err("ERROR: could not compute new hash. Contact customer support.\n"); } } } #ifdef WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE static int updateFipsHash(void); #endif #endif /* HAVE_FIPS && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef WOLFSSL_LINUXKM_BENCHMARKS extern int wolfcrypt_benchmark_main(int argc, char** argv); #endif /* WOLFSSL_LINUXKM_BENCHMARKS */ #ifndef WOLFSSL_LINUXKM_USE_MUTEXES int wc_lkm_LockMutex(wolfSSL_Mutex* m) { int can_block = wc_linuxkm_can_block(); #ifdef WC_LINUXKM_SPIN_IN_ATOMIC unsigned long irq_flags; #endif #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG if ((m == NULL) || (m->magic != WC_LINUXKM_SPINLOCK_MAGIC)) return BAD_FUNC_ARG; #endif #if IS_ENABLED(CONFIG_PREEMPT_RT) /* On RT, spinlock_t is retyped to a sleeping rtmutex, and spin_trylock * becomes rt_spin_trylock, illegal from NMI. */ if (in_nmi()) return BUSY_E; #endif /* first, try the cheap way. */ #ifdef WC_LINUXKM_SPIN_IN_ATOMIC if (spin_trylock_irqsave(&m->lock, irq_flags)) { m->irq_flags = irq_flags; return 0; } #else if (spin_trylock(&m->lock)) { #if !IS_ENABLED(CONFIG_PREEMPT_RT) /* On CONFIG_PREEMPT_RT kernels, spin_trylock() does not disable * preemption, so don't re-enable it. */ if (can_block) { preempt_enable(); m->preempt_reenabled = 1; } #endif return 0; } #endif #if !IS_ENABLED(CONFIG_PREEMPT_RT) if (in_nmi()) return BUSY_E; #endif if (! can_block) { #ifndef WC_LINUXKM_SPIN_IN_ATOMIC /* RT spinlock_t is a sleeping rtmutex; an atomic caller has no legal * wait -- the entry trylock was its one shot. */ return BUSY_E; #else /* WC_LINUXKM_SPIN_IN_ATOMIC */ /* Note, this catches calls while SAVE_VECTOR_REGISTERS()ed, because * in_softirq() is always true while saved, even for * WC_FPU_INHIBITED_FLAG contexts. * * It also catches non-interrupt atomic callers -- tasks holding a * spinlock or running with IRQs off -- which must not reach the * cond_resched() retry loop below. */ spin_lock_irqsave(&m->lock, irq_flags); m->irq_flags = irq_flags; return 0; #endif /* WC_LINUXKM_SPIN_IN_ATOMIC */ } else { for (;;) { int sig_ret = wc_linuxkm_check_for_intr_signals(); if (sig_ret) return sig_ret; cond_resched(); /* FWIW, on PREEMPT_RT kernels, this polls a sleeping lock instead * of blocking on it, which bypasses priority inheritance: legal, * but a low-priority holder under a high-priority RT poller * inherits nothing and the loop degrades to prioritized * busy-wait. */ #ifdef WC_LINUXKM_SPIN_IN_ATOMIC /* When this path is configured, task context lockers are promoted * to non-preemptible context for the duration of the mutex hold, * allowing atomic context callers (except NMI) to spin-wait without * risk of deadlock. */ if (spin_trylock_irqsave(&m->lock, irq_flags)) { m->irq_flags = irq_flags; return 0; } #else /* When this path is configured, we re-enable preemption and * migration for regular tasks while they hold the mutex. Atomic * context callers must correspondingly use one-try as above, * otherwise deadlocks are inevitable. */ if (spin_trylock(&m->lock)) { #if !IS_ENABLED(CONFIG_PREEMPT_RT) /* On CONFIG_PREEMPT_RT kernels, spin_trylock() does not disable * preemption, so don't re-enable it. */ preempt_enable(); m->preempt_reenabled = 1; #endif return 0; } #endif } } __builtin_unreachable(); } int wc_lkm_UnlockMutex(wolfSSL_Mutex* m) { #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG if ((m == NULL) || (m->magic != WC_LINUXKM_SPINLOCK_MAGIC)) return BAD_FUNC_ARG; #endif #ifdef WC_LINUXKM_SPIN_IN_ATOMIC spin_unlock_irqrestore(&m->lock, m->irq_flags); #else if (m->preempt_reenabled) { m->preempt_reenabled = 0; preempt_disable(); } spin_unlock(&m->lock); #endif return 0; } #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 14, 0) /* linux "module: Constify 'struct module_attribute'" (v6.14-rc1) */ #define WC_MODULE_ATTR_CONST const #else #define WC_MODULE_ATTR_CONST #endif /* Note, nodes on THIS_MODULE->mkobj.kobj are dispatched by module_sysfs_ops * (see module_ktype in kernel/params.c), so their handlers must be typed as * struct module_attribute callbacks, not struct kobj_attribute callbacks -- * the latter only work by layout coincidence, and trap on kernels with * CONFIG_CFI_CLANG. */ WC_MAYBE_UNUSED static int linuxkm_sysfs_install_attr(struct attribute *attr, int *installed_flag) { if ((installed_flag == NULL) || (! *installed_flag)) { int ret = sysfs_create_file(&THIS_MODULE->mkobj.kobj, attr); if (ret) { pr_err("ERROR: sysfs_create_file failed for %s: %d\n", attr->name, ret); return ret; } if (installed_flag) *installed_flag = 1; } return 0; } WC_MAYBE_UNUSED static int linuxkm_sysfs_deinstall_attr(struct attribute *attr, int *installed_flag) { if ((installed_flag == NULL) || *installed_flag) { sysfs_remove_file(&THIS_MODULE->mkobj.kobj, attr); if (installed_flag) *installed_flag = 0; } return 0; } /* Transitional wrappers for not-yet-converted struct kobj_attribute callers. * Remove these, and the callers' CFI-incompatible handler typing, by * converting the callers to struct module_attribute per the pattern above. */ WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_install_node(struct kobj_attribute *node, int *installed_flag) { return linuxkm_sysfs_install_attr(&node->attr, installed_flag); } WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_deinstall_node(struct kobj_attribute *node, int *installed_flag) { return linuxkm_sysfs_deinstall_attr(&node->attr, installed_flag); } #ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE static ssize_t dump_to_file(const char *path, const u8 *buf, size_t buf_len) { loff_t pos = 0; struct file *fp; ssize_t ret; char tmp; if (buf_len == 0) { pr_info("libwolfssl: dump_to_file() called with buf_len == 0. Not dumping.\n"); return 0; } WC_SANITIZE_DISABLE(); #if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0) ret = probe_kernel_read(&tmp, buf, 1); if (ret == 0) ret = probe_kernel_read(&tmp, buf + buf_len - 1, 1); #else ret = copy_from_kernel_nofault(&tmp, buf, 1); if (ret == 0) ret = copy_from_kernel_nofault(&tmp, buf + buf_len - 1, 1); #endif WC_SANITIZE_ENABLE(); if (ret != 0) { pr_err("libwolfssl: cannot safely read from buffer %px: %d\n", buf, (int)ret); return ret; } fp = filp_open(path, O_WRONLY | O_CREAT, 0644); if (IS_ERR(fp)) { pr_err("libwolfssl: cannot open %s: %d\n", path, (int)PTR_ERR(fp)); return PTR_ERR(fp); } WC_SANITIZE_DISABLE(); #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0) /* kernel_write() fixed by e13ec939e9 */ ret = kernel_write(fp, buf, buf_len, &pos); #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) /* kernel_write() exported by 7bb307e894d51 */ ret = kernel_write(fp, (char *)buf, buf_len, pos); #else ret = vfs_write(fp, buf, buf_len, &pos); #endif WC_SANITIZE_ENABLE(); filp_close(fp, NULL); if (ret < 0) pr_err("libwolfssl: write to %s failed: %zd\n", path, ret); else if ((size_t)ret != buf_len) pr_warn("libwolfssl: short write to %s: %zd of %zu bytes\n", path, ret, buf_len); return ret; } static char *text_dump_path; static char *rodata_dump_path; /* indent these so they don't look like flush-left function calls. */ module_param(text_dump_path, charp, 0444); module_param(rodata_dump_path, charp, 0444); MODULE_PARM_DESC(text_dump_path, "Path to dump live .wolfcrypt_text section to (e.g. /tmp/wc_text.bin)"); MODULE_PARM_DESC(rodata_dump_path, "Path to dump live .wolfcrypt_rodata section to (e.g. /tmp/wc_rodata.bin)"); #endif /* WC_LINUXKM_SUPPORT_DUMP_TO_FILE */ #ifdef HAVE_FIPS static ssize_t FIPS_rerun_self_test_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count); static struct module_attribute FIPS_rerun_self_test_attr = __ATTR(FIPS_rerun_self_test, 0220, NULL, FIPS_rerun_self_test_handler); static int installed_sysfs_FIPS_files = 0; #endif #ifdef LINUXKM_LKCAPI_REGISTER #include "linuxkm/lkcapi_glue.c" #endif /* NMI-safe Epoch time accessors */ /* Shared calibration state for the pre-4.15 compositions; see * wc_linuxkm_time_epoch_secs() for the full scheme notes. Primed from * wolfssl_init(); refreshed by every non-NMI call to either accessor. */ #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) static struct { unsigned long ref_jiffies; unsigned long ref_secs; unsigned long ref_msfrac; /* 0-999: ms-within-second at calibration */ } wc_time_refs[2]; static unsigned int wc_time_refs_seq = 0; static DEFINE_SPINLOCK(wc_time_refs_lock); /* writers only; see below */ #elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) static unsigned long wc_real_offset_s = 0; #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) /* READ_ONCE()/WRITE_ONCE() arrived in 3.19 (include/linux/compiler.h); * the 3.17 and 3.18 composition of the accessors below is spelled with * them, so ACCESS_ONCE() stands in. */ #ifndef READ_ONCE #define READ_ONCE(x) ACCESS_ONCE(x) #endif #ifndef WRITE_ONCE #define WRITE_ONCE(x, val) (ACCESS_ONCE(x) = (val)) #endif #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) static void wc_linuxkm_time_recalibrate(unsigned long now_jiffies, unsigned long secs, unsigned long msfrac) { if (spin_trylock(&wc_time_refs_lock)) { unsigned int seq = wc_time_refs_seq; ACCESS_ONCE(wc_time_refs[(seq + 1) & 1].ref_jiffies) = now_jiffies; ACCESS_ONCE(wc_time_refs[(seq + 1) & 1].ref_secs) = secs; ACCESS_ONCE(wc_time_refs[(seq + 1) & 1].ref_msfrac) = msfrac; smp_wmb(); ACCESS_ONCE(wc_time_refs_seq) = seq + 1; smp_wmb(); ACCESS_ONCE(wc_time_refs[seq & 1].ref_jiffies) = now_jiffies; ACCESS_ONCE(wc_time_refs[seq & 1].ref_secs) = secs; ACCESS_ONCE(wc_time_refs[seq & 1].ref_msfrac) = msfrac; smp_wmb(); ACCESS_ONCE(wc_time_refs_seq) = seq + 2; spin_unlock(&wc_time_refs_lock); } } static void wc_linuxkm_time_read_latch(unsigned long *rj, unsigned long *rs, unsigned long *rf) { unsigned int seq; do { seq = ACCESS_ONCE(wc_time_refs_seq); smp_rmb(); *rj = ACCESS_ONCE(wc_time_refs[seq & 1].ref_jiffies); *rs = ACCESS_ONCE(wc_time_refs[seq & 1].ref_secs); *rf = ACCESS_ONCE(wc_time_refs[seq & 1].ref_msfrac); smp_rmb(); } while (ACCESS_ONCE(wc_time_refs_seq) != seq); } #endif time_t wc_linuxkm_time_epoch_secs(time_t *tp) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0) u64 s = div_u64(ktime_get_real_fast_ns(), NSEC_PER_SEC); if (tp) *tp = (time_t)s; return (time_t)s; #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) /* Note that an NMI-context read can lag a bracketing task-context read by * up to a second, i.e. time can appear to step back 1 s across contexts. * In practice this doesn't matter. */ u64 s; if (! in_nmi()) { u64 mono_ns = ktime_get_mono_fast_ns(); u64 ns = ktime_get_real_ns(); WRITE_ONCE(wc_real_offset_s, div_u64(ns - mono_ns, NSEC_PER_SEC)); s = div_u64(ns, NSEC_PER_SEC); } else s = div_u64(ktime_get_mono_fast_ns(), NSEC_PER_SEC) + READ_ONCE(wc_real_offset_s); if (tp) *tp = (time_t)s; return (time_t)s; #else /* < 3.17: [existing contraption comment, unchanged] */ unsigned long now_jiffies = jiffies; unsigned long s; if (! in_nmi()) { struct timespec ts; getnstimeofday(&ts); s = (unsigned long)ts.tv_sec; wc_linuxkm_time_recalibrate(now_jiffies, s, (unsigned long)(ts.tv_nsec / NSEC_PER_MSEC)); } else { unsigned long rj, rs, rf; wc_linuxkm_time_read_latch(&rj, &rs, &rf); /* jiffies is read after the latch: a calibration published by * another CPU between the entry-time snapshot and the latch read can * carry a later tick than the snapshot, and the unsigned difference * would wrap. */ s = rs + (jiffies - rj) / HZ; } if (tp) *tp = (time_t)s; return (time_t)s; #endif } long long wc_linuxkm_time_epoch_msecs(void) { #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0) return (long long)div_u64(ktime_get_real_fast_ns(), NSEC_PER_MSEC); #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) /* The cached offset is whole seconds (word-sized for 32-bit store * atomicity), so the NMI-context absolute value is quantized to the * second; millisecond granularity within and between NMI-context reads * is preserved (deltas ride ktime_get_mono_fast_ns()). Cross-context * skew up to ~1 s, as for the seconds accessor. */ if (! in_nmi()) { u64 mono_ns = ktime_get_mono_fast_ns(); u64 ns = ktime_get_real_ns(); WRITE_ONCE(wc_real_offset_s, div_u64(ns - mono_ns, NSEC_PER_SEC)); return (long long)div_u64(ns, NSEC_PER_MSEC); } else return (long long)(div_u64(ktime_get_mono_fast_ns(), NSEC_PER_MSEC) + (u64)READ_ONCE(wc_real_offset_s) * MSEC_PER_SEC); #else /* Same latch as the seconds accessor; ref_msfrac restores ms-within- * second at the calibration instant, and the jiffies delta carries ms * granularity (1/HZ resolution) from there. */ unsigned long now_jiffies = jiffies; if (! in_nmi()) { struct timespec ts; unsigned long s, f; getnstimeofday(&ts); s = (unsigned long)ts.tv_sec; f = (unsigned long)(ts.tv_nsec / NSEC_PER_MSEC); wc_linuxkm_time_recalibrate(now_jiffies, s, f); return (long long)s * MSEC_PER_SEC + (long long)f; } else { unsigned long rj, rs, rf; wc_linuxkm_time_read_latch(&rj, &rs, &rf); /* jiffies read after the latch, as in the seconds accessor. */ return (long long)rs * MSEC_PER_SEC + (long long)rf + (long long)div_u64((u64)(jiffies - rj) * MSEC_PER_SEC, HZ); } #endif } int wc_linuxkm_can_block(void) { /* We can't use preemptible() for this, because we need an accurate test * even in !CONFIG_PREEMPT_COUNT configs where preemptible() is always 0. */ return (preempt_count() == 0) && (! irqs_disabled()); } int wc_linuxkm_can_long_loop(void) { return in_task() && !irqs_disabled(); } /* for simplicity, we use a global count to suspend signal processing while any * thread is running fipsEntry(), wolfCrypt_IntegrityTest_fips(), * linuxkm_lkcapi_register(), or linuxkm_lkcapi_unregister(). This only affects * startup dynamics and the FIPS runtime. Once the uninterruptible routine * completes, signal handling resumes, and any still-pending signal on * continuing threads will be processed in a timely fashion. */ static wolfSSL_Atomic_Int wc_linuxkm_sig_ignore_count = WOLFSSL_ATOMIC_INITIALIZER(0); int wc_linuxkm_sig_ignore_begin(void) { return wolfSSL_Atomic_Int_AddFetch(&wc_linuxkm_sig_ignore_count, 1); } int wc_linuxkm_sig_ignore_end(void) { return wolfSSL_Atomic_Int_SubFetch(&wc_linuxkm_sig_ignore_count, 1); } int wc_linuxkm_check_for_intr_signals(void) { static const int intr_signals[] = WC_LINUXKM_INTR_SIGNALS; if (! wc_linuxkm_can_block()) return 0; if (signal_pending(current)) { int i; for (i = 0; i < (int)sizeof(intr_signals) / (int)sizeof(intr_signals[0]); ++i) { if (sigismember(¤t->pending.signal, intr_signals[i])) { if (WOLFSSL_ATOMIC_LOAD(wc_linuxkm_sig_ignore_count) > 0) { #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG pr_info("INFO: wc_linuxkm_check_for_intr_signals ignoring " "signal %d\n", intr_signals[i]); #endif return 0; } #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG pr_info("INFO: wc_linuxkm_check_for_intr_signals returning " "INTERRUPTED_E on signal %d\n", intr_signals[i]); #endif return INTERRUPTED_E; } } } return 0; } void wc_linuxkm_relax_long_loop(void) { #if WC_LINUXKM_MAX_NS_WITHOUT_YIELD >= 0 if (wc_linuxkm_can_block()) { #if (WC_LINUXKM_MAX_NS_WITHOUT_YIELD == 0) || !defined(CONFIG_SCHED_INFO) cond_resched(); #else /* note that local_clock() wraps a local_clock_noinstr() in a * preempt_disable_notrace(), which sounds expensive but isn't -- * preempt_disable_notrace() is actually just a nonlocking integer * increment of current_thread_info()->preempt.count, protected only by * various compiler optimizer barriers. */ u64 now = local_clock(); u64 current_last_arrival = current->sched_info.last_arrival; s64 delta = (s64)(now - current_last_arrival); if (delta > WC_LINUXKM_MAX_NS_WITHOUT_YIELD) { cond_resched(); /* if nothing else is runnable, cond_resched() is a no-op and * doesn't even update .last_arrival. we could force update by * sleeping, but there's no need. we've been nice enough by just * cond_resched()ing, and it's actually preferable to call * cond_resched() frequently once computation has looped * continuously for longer than WC_LINUXKM_MAX_NS_WITHOUT_YIELD. */ } #endif return; } #endif cpu_relax(); } #if defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER) int wc_linuxkm_GenerateSeed_wolfEntropy(OS_Seed* os, byte* output, word32 sz) { (void)os; return wc_Entropy_Get(MAX_ENTROPY_BITS, output, sz); } #elif defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER) /* backported wc_GenerateSeed_IntelRD() for FIPS v5, before breakout of wolfentropy.c. */ #include static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER; static inline void wc_InitRng_IntelRD(void) { cpuid_get_flags_ex(&intel_flags); } #define INTELRD_RETRY 32 static WC_INLINE int IntelRDseed64(word64* seed) { unsigned char ok; __asm__ volatile("rdseed %0; setc %1":"=r"(*seed), "=qm"(ok)); return (ok) ? 0 : -1; } /* return 0 on success */ static WC_INLINE int IntelRDseed64_r(word64* rnd) { int iters, retry_counter; word64 buf; #if defined(HAVE_AMD_RDSEED) /* See "AMD RNG ESV Public Use Document". Version 0.7 of October 24, * 2024 specifies 0.656 to 1.312 bits of entropy per 128 bit block of * RDSEED output, depending on CPU family. * * FIPS v5 random.c sets ENTROPY_SCALE_FACTOR to 1 for * HAVE_INTEL_RDSEED. */ iters = 128; #elif defined(HAVE_INTEL_RDSEED) /* The value of 2 applies to Intel's RDSEED which provides about * 0.5 bits minimum of entropy per bit. The value of 4 gives a * conservative margin for FIPS. * * FIPS v5 random.c sets ENTROPY_SCALE_FACTOR to 2 for * HAVE_INTEL_RDSEED. */ iters = 2; #else #error WC_LINUXKM_RDSEED_IN_GLUE_LAYER requires HAVE_INTEL_RDSEED or HAVE_AMD_RDSEED #endif while (--iters >= 0) { for (retry_counter = 0; retry_counter < INTELRD_RETRY; retry_counter++) { if (IntelRDseed64(&buf) == 0) break; } if (retry_counter == INTELRD_RETRY) return -1; WC_SANITIZE_DISABLE(); *rnd ^= buf; /* deliberately retain any garbage passed in the dest buffer. */ WC_SANITIZE_ENABLE(); buf = 0; } wc_ForceZero(&buf, sizeof buf); return 0; } /* return 0 on success */ int wc_linuxkm_GenerateSeed_IntelRD(struct OS_Seed* os, byte* output, word32 sz) { int ret; word64 rndTmp; (void)os; wc_InitRng_IntelRD(); if (!IS_INTEL_RDSEED(intel_flags)) { static wolfSSL_Atomic_Int warned_on_missing_RDSEED = WOLFSSL_ATOMIC_INITIALIZER(0); int expected_warned_on_missing_RDSEED = 0; if (wolfSSL_Atomic_Int_CompareExchange( &warned_on_missing_RDSEED, &expected_warned_on_missing_RDSEED, 1)) { pr_err("ERROR: wc_linuxkm_GenerateSeed_IntelRD() called on CPU without RDSEED support.\n"); } return -1; } for (; (sz / sizeof(word64)) > 0; sz -= sizeof(word64), output += sizeof(word64)) { ret = IntelRDseed64_r((word64*)output); if (ret != 0) { #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG pr_err("ERROR: IntelRDseed64_r() returned code %d.\n", ret); #endif return ret; } } if (sz == 0) return 0; /* handle unaligned remainder */ ret = IntelRDseed64_r(&rndTmp); if (ret != 0) { #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG pr_err("ERROR: IntelRDseed64_r() returned code %d.\n", ret); #endif return ret; } XMEMCPY(output, &rndTmp, sz); wc_ForceZero(&rndTmp, sizeof(rndTmp)); return 0; } #endif /* WC_LINUXKM_RDSEED_IN_GLUE_LAYER */ #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86) /* i386 this_cpu_read() declares an extern inside the function. */ PRAGMA_GCC_DIAG_PUSH PRAGMA_GCC("GCC diagnostic ignored \"-Wnested-externs\"") #include "linuxkm/x86_vector_register_glue.c" PRAGMA_GCC_DIAG_POP #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ (defined(CONFIG_ARM64) || defined(CONFIG_ARM)) #include "linuxkm/arm64_vector_register_glue.c" #endif #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(WC_C_DYNAMIC_FALLBACK) && \ (defined(WC_LINUXKM_FORCE_SVR_DYNAMIC_AUDITING) || \ (!defined(DEBUG_VECTOR_REGISTER_ACCESS_FUZZING) && \ !defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_ON) && \ !defined(DEBUG_VECTOR_REGISTER_ACCESS_ALWAYS_OFF) && \ !defined(WC_SVR_USE_NATIVE_REG_BUFS) && \ !defined(WC_LINUXKM_NO_SVR_DYNAMIC_AUDITING))) #define WC_LINUXKM_SVR_DYNAMIC_AUDITING static int svr_dynamic_auditing = 0; #endif #ifdef FIPS_OPTEST #ifndef HAVE_FIPS #error FIPS_OPTEST requires HAVE_FIPS. #endif extern int linuxkm_op_test_1(int argc, const char* argv[]); extern int linuxkm_op_test_wrapper(void); static wolfSSL_Atomic_Int *conTestFailure_ptr = NULL; #ifdef HAVE_WC_FIPS_OPTEST_CONTESTFAILURE_EXPORT WOLFSSL_API extern wolfSSL_Atomic_Int wc_fips_optest_conTestFailure; #endif static ssize_t FIPS_optest_trig_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count); static struct module_attribute FIPS_optest_trig_attr = __ATTR(FIPS_optest_run_code, 0220, NULL, FIPS_optest_trig_handler); static int installed_sysfs_FIPS_optest_trig_files = 0; #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING static ssize_t FIPS_optest_trig_audit_accel_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count); static struct module_attribute FIPS_optest_trig_audit_accel_attr = __ATTR(FIPS_optest_run_code_audit_accel, 0220, NULL, FIPS_optest_trig_audit_accel_handler); static int installed_sysfs_FIPS_optest_trig_audit_accel_files = 0; static ssize_t FIPS_optest_trig_audit_c_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count); static struct module_attribute FIPS_optest_trig_audit_c_attr = __ATTR(FIPS_optest_run_code_audit_c, 0220, NULL, FIPS_optest_trig_audit_c_handler); static int installed_sysfs_FIPS_optest_trig_audit_c_files = 0; #endif #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) static int __init wolfssl_init(void) #else static int wolfssl_init(void) #endif { int ret; #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_current(); pr_info("STACK INFO: usage at entry to wolfssl_init(): %lu of %lu total\n", stack_usage, THREAD_SIZE); wc_linuxkm_stack_hwm_prepare(0xee); } #endif #ifdef WC_SYM_RELOC_TABLES ret = set_up_wolfssl_linuxkm_pie_redirect_table(); if (ret < 0) return ret; #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) /* prime real_offset_s */ (void)wc_linuxkm_time_epoch_secs(NULL); #endif #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(HAVE_FIPS) ret = wc_linuxkm_allocate_svr_states(); if (ret < 0) { pr_err("wolfssl_init(): wc_linuxkm_allocate_svr_states() returned code %d.\n", ret); return -ECANCELED; } #endif #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING #ifdef IS_INTEL_AVX1 /* inhibit auditing if no vector extensions are available on the host CPU. */ { cpuid_flags_t cpuflags = cpuid_get_flags(); if (IS_INTEL_AVX1(cpuflags)) { svr_dynamic_auditing = 1; } #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG else { pr_info("wolfssl_init(): SVR dynamic auditing disabled -- host cpuflags missing AVX (0x%lx).\n", (unsigned long)cpuflags); } #endif } #else svr_dynamic_auditing = 1; #endif #endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ #ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE #ifdef WC_SYM_RELOC_TABLES if (text_dump_path) { if (dump_to_file(text_dump_path, (u8 *)__wc_text_start, (size_t)((uintptr_t)__wc_text_end - (uintptr_t)__wc_text_start)) > 0) { pr_info("libwolfssl: dumped .wolfcrypt_text (%zu bytes) to %s.\n", (size_t)((uintptr_t)__wc_text_end - (uintptr_t)__wc_text_start), text_dump_path); } } if (rodata_dump_path) { if (dump_to_file(rodata_dump_path, (u8 *)__wc_rodata_start, (size_t)((uintptr_t)__wc_rodata_end - (uintptr_t)__wc_rodata_start)) > 0) { pr_info("libwolfssl: dumped .wolfcrypt_rodata (%zu bytes) to %s.\n", (size_t)((uintptr_t)__wc_rodata_end - (uintptr_t)__wc_rodata_start), rodata_dump_path); } } #else if ((text_dump_path != NULL) || (rodata_dump_path != NULL)) { pr_info("libwolfssl: ignoring module dump path argument(s) -- module lacks WC_SYM_RELOC_TABLES.\n"); } #endif #endif /* WC_LINUXKM_SUPPORT_DUMP_TO_FILE */ #ifdef WC_LINUXKM_TEST_INET_PTON { const char *src; byte dst[16] = { }; int pton_ret; src = "1.2.3.4"; pton_ret = wc_linuxkm_inet_pton(AF_INET, src, dst); printf("pton_ret=%d src=%s dst=%d.%d.%d.%d\n", pton_ret, src, (int)dst[0], (int)dst[1], (int)dst[2], (int)dst[3]); src = "89ab::cdef"; pton_ret = wc_linuxkm_inet_pton(AF_INET6, src, dst); printf("pton_ret=%d src=%s dst=%02x%02x::%02x%02x\n", pton_ret, src, (int)dst[0], (int)dst[1], (int)dst[14], (int)dst[15]); } #endif /* WC_LINUXKM_TEST_INET_PTON */ #if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) /* The compiled-in verifycore must be the right length, else the module * geometry will change when the correct value is passed in, destabilizing * wc_linuxkm_pie_text_reloc_tab. It also must be the right length for the * module-update-fips-hash recipe (in-place overwrite) to work, and for * updateFipsHash() (WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE) to be safe from * overruns. */ { size_t verifyCore_len; #if FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE) verifyCore_len = strlen(verifyCore); #else #ifdef CONFIG_HAVE_KPROBES char *verifyCore_ptr = my_kallsyms_lookup_name("verifyCore"); if (verifyCore_ptr) verifyCore_len = strlen(verifyCore_ptr); else #endif /* CONFIG_HAVE_KPROBES */ { /* can't check -- have to assume. */ #if defined(CONFIG_HAVE_KPROBES) && (defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG)) pr_err("INFO: couldn't get verifyCore_ptr -- skipping verifyCore length check.\n"); #endif verifyCore_len = FIPS_IN_CORE_DIGEST_SIZE*2; } #endif if (verifyCore_len != FIPS_IN_CORE_DIGEST_SIZE*2) { pr_err("ERROR: compile-time FIPS hash is the wrong length (expected %d hex digits, got %zu).\n", FIPS_IN_CORE_DIGEST_SIZE*2, verifyCore_len); (void)libwolfssl_cleanup(); return -ECANCELED; } } #ifdef WC_SYM_RELOC_TABLES if (((uintptr_t)__wc_text_start > (uintptr_t)wolfCrypt_FIPS_first) || ((uintptr_t)__wc_text_end < (uintptr_t)wolfCrypt_FIPS_last) || ((uintptr_t)__wc_rodata_start > (uintptr_t)wolfCrypt_FIPS_ro_start) || ((uintptr_t)__wc_rodata_end < (uintptr_t)wolfCrypt_FIPS_ro_end)) { pr_err("ERROR: ELF segment fenceposts and FIPS fenceposts conflict.\n"); (void)libwolfssl_cleanup(); return -ECANCELED; } #endif #if defined(WC_LINUXKM_USE_HEAP_WRAPPERS) && defined(CONFIG_HAVE_KPROBES) /* cache the function pointer to find_vm_area(). */ (void)wc_linuxkm_malloc_usable_size(NULL); #endif #ifdef WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE #ifdef CONFIG_MODULE_SIG if (THIS_MODULE->sig_ok == false) { pr_err("ERROR: wolfSSL module load aborted -- bad or missing module signature with FIPS dynamic hash.\n"); (void)libwolfssl_cleanup(); return -ECANCELED; } #endif #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT) reloc_counts.text = reloc_counts.rodata = reloc_counts.rwdata = reloc_counts.bss = reloc_counts.other = 0; #endif ret = updateFipsHash(); if (ret < 0) { pr_err("ERROR: wolfSSL module load aborted -- updateFipsHash: %s\n",wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT) pr_info("FIPS-bounded relocation normalizations from updateFipsHash(): text=%d, rodata=%d, rwdata=%d, bss=%d, other=%d\n", reloc_counts.text, reloc_counts.rodata, reloc_counts.rwdata, reloc_counts.bss, reloc_counts.other); #endif #endif /* WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE */ #endif /* HAVE_FIPS */ #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT) #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0) /* see linux commit ac3b432839 */ #define THIS_MODULE_TEXT_BASE (THIS_MODULE->mem[MOD_TEXT].base) #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->mem[MOD_TEXT].size) #define THIS_MODULE_RO_BASE (THIS_MODULE->mem[MOD_RODATA].base) #define THIS_MODULE_RO_SIZE (THIS_MODULE->mem[MOD_RODATA].size) #elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) #define THIS_MODULE_TEXT_BASE (THIS_MODULE->core_layout.base) #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_layout.text_size) #define THIS_MODULE_RO_BASE ((char *)THIS_MODULE->core_layout.base + THIS_MODULE->core_layout.text_size) #define THIS_MODULE_RO_SIZE (THIS_MODULE->core_layout.ro_size) #else #define THIS_MODULE_TEXT_BASE (THIS_MODULE->module_core) #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_text_size) #define THIS_MODULE_RO_BASE ((char *)THIS_MODULE->module_core + THIS_MODULE->core_ro_size) #define THIS_MODULE_RO_SIZE (THIS_MODULE->core_ro_size) #endif { unsigned int text_hash = hash_span((const u8 *)__wc_text_start, (const u8 *)__wc_text_end, 1); unsigned int rodata_hash = hash_span((const u8 *)__wc_rodata_start, (const u8 *)__wc_rodata_end, 1); u8 *canon_buf = malloc(WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ); ssize_t cur_reloc_index; const u8 *text_p = (const u8 *)__wc_text_start; const u8 *rodata_p = (const u8 *)__wc_rodata_start; unsigned int stabilized_text_hash = 1; unsigned int stabilized_rodata_hash = 1; if (! canon_buf) { pr_err("ERROR: malloc(%d) for WOLFSSL_*_SEGMENT_CANONICALIZER failed.\n", WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ); (void)libwolfssl_cleanup(); return -ECANCELED; } reloc_counts.text = reloc_counts.rodata = reloc_counts.rwdata = reloc_counts.bss = reloc_counts.other = 0; cur_reloc_index = -1; while (text_p < (const u8 *)__wc_text_end) { size_t text_in_out_len = min(WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ, (size_t)((const u8 *)__wc_text_end - text_p)); ssize_t progress = WOLFSSL_SEGMENT_CANONICALIZER( text_p, &text_in_out_len, canon_buf, &cur_reloc_index); if (progress <= 0) { pr_err("ERROR: progress=%ld from WOLFSSL_SEGMENT_CANONICALIZER() at offset %x (text=%x-%x).\n", (long)progress, (unsigned)(uintptr_t)text_p, (unsigned)(uintptr_t)__wc_text_start, (unsigned)(uintptr_t)__wc_text_end); free(canon_buf); (void)libwolfssl_cleanup(); return -ECANCELED; } stabilized_text_hash = hash_span(canon_buf, canon_buf + text_in_out_len, stabilized_text_hash); text_p += progress; } /* note verifyCore is hashed along with the rest of .rodata_wolfcrypt. */ cur_reloc_index = -1; while (rodata_p < (const u8 *)__wc_rodata_end) { size_t rodata_in_out_len = min(WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ, (size_t)((const u8 *)__wc_rodata_end - rodata_p)); ssize_t progress = WOLFSSL_SEGMENT_CANONICALIZER( rodata_p, &rodata_in_out_len, canon_buf, &cur_reloc_index); if (progress <= 0) { pr_err("ERROR: progress=%ld from WOLFSSL_SEGMENT_CANONICALIZER() at offset %x (rodata=%x-%x).\n", (long)progress, (unsigned)(uintptr_t)rodata_p, (unsigned)(uintptr_t)__wc_rodata_start, (unsigned)(uintptr_t)__wc_rodata_end); free(canon_buf); (void)libwolfssl_cleanup(); return -ECANCELED; } stabilized_rodata_hash = hash_span(canon_buf, canon_buf + rodata_in_out_len, stabilized_rodata_hash); rodata_p += progress; } free(canon_buf); canon_buf = 0; /* note, "%pK" conceals the actual layout information. "%px" exposes * the true module start address, which is potentially useful to an * attacker. */ pr_info("wolfCrypt segment hashes (spans): text 0x%x (%llu), rodata 0x%x (%llu), offset %c0x%llx, canon text 0x%x, canon rodata 0x%x\n", text_hash, (unsigned long long)((uintptr_t)__wc_text_end - (uintptr_t)__wc_text_start), rodata_hash, (unsigned long long)((uintptr_t)__wc_rodata_end - (uintptr_t)__wc_rodata_start), (uintptr_t)__wc_text_start < (uintptr_t)&__wc_rodata_start[0] ? '+' : '-', (uintptr_t)__wc_text_start < (uintptr_t)&__wc_rodata_start[0] ? (unsigned long long)((uintptr_t)&__wc_rodata_start[0] - (uintptr_t)__wc_text_start) : (unsigned long long)((uintptr_t)__wc_text_start - (uintptr_t)&__wc_rodata_start[0]), stabilized_text_hash, stabilized_rodata_hash); pr_info("wolfCrypt segments: text=%llx-%llx, rodata=%llx-%llx, " "rwdata=%llx-%llx, bss=%llx-%llx\n", (unsigned long long)(uintptr_t)__wc_text_start, (unsigned long long)(uintptr_t)__wc_text_end, (unsigned long long)(uintptr_t)__wc_rodata_start, (unsigned long long)(uintptr_t)__wc_rodata_end, (unsigned long long)(uintptr_t)__wc_rwdata_start, (unsigned long long)(uintptr_t)__wc_rwdata_end, (unsigned long long)(uintptr_t)__wc_bss_start, (unsigned long long)(uintptr_t)__wc_bss_end); pr_info("whole-segment relocation normalizations: text=%d, rodata=%d, rwdata=%d, bss=%d, other=%d\n", reloc_counts.text, reloc_counts.rodata, reloc_counts.rwdata, reloc_counts.bss, reloc_counts.other); } #endif /* WC_SYM_RELOC_TABLES && DEBUG_LINUXKM_PIE_SUPPORT */ #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by wolfssl_init() initial setup: %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif #if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) ret = wolfCrypt_SetCb_fips(lkmFipsCb); if (ret != 0) { pr_err("ERROR: wolfCrypt_SetCb_fips() failed: %s\n", wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT) reloc_counts.text = reloc_counts.rodata = reloc_counts.rwdata = reloc_counts.bss = reloc_counts.other = 0; #endif /* In asm builds, we run the FIPS self-test twice, once via fipsEntry() checking * afterwards that no C fallbacks occurred, and a second time via * wolfCrypt_IntegrityTest_fips() with asm disabled. */ #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING wc_svr_disallowed_count_reset(); #endif if (WC_SIG_IGNORE_BEGIN() >= 0) { fipsEntry(); (void)WC_SIG_IGNORE_END(); } else pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n"); #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT) pr_info("FIPS-bounded relocation normalizations: text=%d, rodata=%d, rwdata=%d, bss=%d, other=%d\n", reloc_counts.text, reloc_counts.rodata, reloc_counts.rwdata, reloc_counts.bss, reloc_counts.other); #endif ret = wolfCrypt_GetStatus_fips(); if (ret != 0) { pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret)); if (ret == WC_NO_ERR_TRACE(IN_CORE_FIPS_E)) { const char *newhash = wolfCrypt_GetCoreHash_fips(); if (newhash) { pr_err("In-core integrity hash check failure.\n" "Update FIPS hash with \"make module-update-fips-hash FIPS_HASH=%s\".\n", newhash); } else { pr_err("In-core integrity hash check failure.\n"); pr_err("ERROR: could not compute new hash. Contact customer support.\n"); } } (void)libwolfssl_cleanup(); return -ECANCELED; } #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { long long unsigned int svr_disallowed_count = wc_svr_disallowed_count_current(); #if !(defined(WOLFSSL_AESNI) && !defined(USE_INTEL_SPEEDUP)) long long unsigned int svr_disallowed_snapshot; #endif if (svr_disallowed_count > 0) { pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after fipsEntry().\n", svr_disallowed_count); (void)libwolfssl_cleanup(); return -ECANCELED; } ret = DISABLE_VECTOR_REGISTERS(); if (ret != 0) { pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for wolfCrypt_IntegrityTest_fips() returned %d.\n", ret); (void)libwolfssl_cleanup(); return -ECANCELED; } #if !(defined(WOLFSSL_AESNI) && !defined(USE_INTEL_SPEEDUP)) /* DISABLE_VECTOR_REGISTERS() itself increments the disallowed * count (it's a save call with WC_SVR_FLAG_INHIBIT), so snapshot * after it and require the test run to increment past the * snapshot -- a bare != 0 check after the test is vacuously * satisfied by the DISABLE itself and detects nothing. */ svr_disallowed_snapshot = wc_svr_disallowed_count_current(); #endif ret = wolfCrypt_IntegrityTest_fips(); REENABLE_VECTOR_REGISTERS(); #if !(defined(WOLFSSL_AESNI) && !defined(USE_INTEL_SPEEDUP)) svr_disallowed_count = wc_svr_disallowed_count_current(); if (svr_disallowed_count <= svr_disallowed_snapshot) { pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wolfCrypt_IntegrityTest_fips() with DISABLE_VECTOR_REGISTERS() (snapshot %llu): inhibited-save instrumentation was not exercised.\n", svr_disallowed_count, svr_disallowed_snapshot); (void)libwolfssl_cleanup(); return -ECANCELED; } #endif if (ret != 0) { pr_err("ERROR: wolfCrypt_IntegrityTest_fips() with DISABLE_VECTOR_REGISTERS() returned %d.\n", ret); (void)libwolfssl_cleanup(); return -ECANCELED; } ret = wolfCrypt_GetStatus_fips(); if (ret != 0) { pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } } #endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ #endif /* HAVE_FIPS && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef WC_RNG_SEED_CB ret = wc_SetSeed_Cb(WC_GENERATE_SEED_DEFAULT); if (ret < 0) { pr_err("ERROR: wc_SetSeed_Cb() failed with return code %d.\n", ret); (void)libwolfssl_cleanup(); msleep(10); return -ECANCELED; } #endif /* WC_RNG_SEED_CB */ /* Always call wolfCrypt_Init() directly, even in TLS builds, to assure * mutex-free scheduler context during allocations. wolfSSL_Init() still * uses a mutex, which puts the initialization thread in atomic context. * The redundant wolfCrypt_Init() via wolfSSL_Init() is harmless -- * wolfCrypt counts initialization depth. */ ret = wolfCrypt_Init(); if (ret != 0) { pr_err("ERROR: wolfCrypt_Init() failed: %s\n", wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } wolfcrypt_inited = 1; #ifndef WOLFCRYPT_ONLY ret = wolfSSL_Init(); if (ret != WOLFSSL_SUCCESS) { pr_err("ERROR: wolfSSL_Init() failed: %s\n", wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } wolfssl_inited = 1; #endif #ifdef WC_RNG_HAVE_GLOBAL_FALLBACK_RNG { unsigned long uncredited_nonce = random_get_entropy(); const char *perso = dmi_get_system_info(DMI_PRODUCT_UUID); word32 persoSz; if (perso) persoSz = strlen(perso); else { static uintptr_t init_addr; init_addr = (uintptr_t)&wolfssl_init; perso = (const char *)&init_addr; persoSz = (word32)sizeof(init_addr); } ret = wc_RNG_global_fallback_init((const byte *)&uncredited_nonce, (word32)sizeof uncredited_nonce, (const byte *)perso, persoSz, NULL /* heap */, INVALID_DEVID, WC_RNG_INIT_FLAG_NONE); if (ret != 0) { pr_err("ERROR: wc_RNG_global_fallback_init() failed: %s\n", wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } } #endif /* WC_RNG_HAVE_GLOBAL_FALLBACK_RNG */ #if defined(HAVE_FIPS) && FIPS_VERSION3_GT(5,2,0) && !defined(WOLFSSL_FIPS_DEV_NO_POST) #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_current(); pr_info("STACK INFO: usage at call to wc_RunAllCast_fips(): %lu of %lu total\n", stack_usage, THREAD_SIZE); wc_linuxkm_stack_hwm_prepare(0xee); #endif #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING wc_svr_disallowed_count_reset(); #endif ret = wc_RunAllCast_fips(); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by wc_RunAllCast_fips(): %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif if (ret != 0) { pr_err("ERROR: wc_RunAllCast_fips() failed with return value %d\n", ret); (void)libwolfssl_cleanup(); return -ECANCELED; } #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { long long unsigned int svr_disallowed_count = wc_svr_disallowed_count_current(); long long unsigned int svr_disallowed_snapshot; if (svr_disallowed_count > 0) { pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wc_RunAllCast_fips().\n", svr_disallowed_count); (void)libwolfssl_cleanup(); return -ECANCELED; } #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; wc_linuxkm_stack_hwm_prepare(0xee); #endif ret = DISABLE_VECTOR_REGISTERS(); if (ret != 0) { pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for wc_RunAllCast_fips() returned %d.\n", ret); (void)libwolfssl_cleanup(); return -ECANCELED; } /* See the snapshot rationale in the wolfCrypt_IntegrityTest_fips() * block above. */ svr_disallowed_snapshot = wc_svr_disallowed_count_current(); ret = wc_RunAllCast_fips(); REENABLE_VECTOR_REGISTERS(); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS(): %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif svr_disallowed_count = wc_svr_disallowed_count_current(); if (svr_disallowed_count <= svr_disallowed_snapshot) { pr_err("ERROR: wc_svr_disallowed_count_current() returned %llu after wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() (snapshot %llu): inhibited-save instrumentation was not exercised.\n", svr_disallowed_count, svr_disallowed_snapshot); (void)libwolfssl_cleanup(); return -ECANCELED; } if (ret != 0) { pr_err("ERROR: wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() returned %d.\n", ret); (void)libwolfssl_cleanup(); return -ECANCELED; } ret = wolfCrypt_GetStatus_fips(); if (ret != 0) { pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } } #endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ pr_info("FIPS 140-3 wolfCrypt-fips v%d.%d.%d%s%s startup " "self-test succeeded.\n", #ifdef HAVE_FIPS_VERSION_MAJOR HAVE_FIPS_VERSION_MAJOR, #else HAVE_FIPS_VERSION, #endif #ifdef HAVE_FIPS_VERSION_MINOR HAVE_FIPS_VERSION_MINOR, #else 0, #endif #ifdef HAVE_FIPS_VERSION_PATCH HAVE_FIPS_VERSION_PATCH, #else 0, #endif #ifdef HAVE_FIPS_VERSION_PORT "-", HAVE_FIPS_VERSION_PORT #else "", "" #endif ); #endif /* HAVE_FIPS && FIPS_VERSION3_GT(5,2,0) && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef FIPS_OPTEST #ifdef HAVE_WC_FIPS_OPTEST_CONTESTFAILURE_EXPORT conTestFailure_ptr = &wc_fips_optest_conTestFailure; #else conTestFailure_ptr = (wolfSSL_Atomic_Int *)my_kallsyms_lookup_name("conTestFailure"); if (conTestFailure_ptr == NULL) { pr_err("ERROR: couldn't obtain conTestFailure_ptr.\n"); (void)libwolfssl_cleanup(); return -ECANCELED; } #endif ret = linuxkm_sysfs_install_attr(&FIPS_optest_trig_attr.attr, &installed_sysfs_FIPS_optest_trig_files); if (ret != 0) { pr_err("ERROR: linuxkm_sysfs_install_attr() failed for %s (code %d).\n", FIPS_optest_trig_attr.attr.name, ret); (void)libwolfssl_cleanup(); return -ECANCELED; } #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { ret = linuxkm_sysfs_install_attr(&FIPS_optest_trig_audit_accel_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_accel_files); if (ret != 0) { pr_err("ERROR: linuxkm_sysfs_install_attr() failed for %s (code %d).\n", FIPS_optest_trig_audit_accel_attr.attr.name, ret); (void)libwolfssl_cleanup(); return -ECANCELED; } ret = linuxkm_sysfs_install_attr(&FIPS_optest_trig_audit_c_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_c_files); if (ret != 0) { pr_err("ERROR: linuxkm_sysfs_install_attr() failed for %s (code %d).\n", FIPS_optest_trig_audit_c_attr.attr.name, ret); (void)libwolfssl_cleanup(); return -ECANCELED; } } #endif #ifdef FIPS_OPTEST_FULL_RUN_AT_MODULE_INIT #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_current(); pr_info("STACK INFO: usage at call to linuxkm_op_test_wrapper(): %lu of %lu total\n", stack_usage, THREAD_SIZE); wc_linuxkm_stack_hwm_prepare(0xee); #endif (void)linuxkm_op_test_wrapper(); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by linuxkm_op_test_wrapper(): %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif WOLFSSL_ATOMIC_STORE(*conTestFailure_ptr, 0); { int i; for (i = 0; i < FIPS_CAST_COUNT; ++i) fipsCastStatus_put(i, FIPS_CAST_STATE_INIT); } /* note, must call fipsEntry() here, not wolfCrypt_IntegrityTest_fips(), * because wc_GetCastStatus_fips(FIPS_CAST_HMAC_SHA2_256) isn't available * anymore. */ if (WC_SIG_IGNORE_BEGIN() >= 0) { fipsEntry(); (void)WC_SIG_IGNORE_END(); } else pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n"); ret = wolfCrypt_GetStatus_fips(); if (ret != 0) { pr_err("ERROR: wolfCrypt_GetStatus_fips() after reset failed with code %d: %s\n", ret, wc_GetErrorString(ret)); (void)libwolfssl_cleanup(); return -ECANCELED; } #endif #endif /* FIPS_OPTEST */ #ifndef NO_CRYPT_TEST #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_current(); pr_info("STACK INFO: usage at call to wolfcrypt_test(): %lu of %lu total\n", stack_usage, THREAD_SIZE); wc_linuxkm_stack_hwm_prepare(0xee); #endif ret = wolfcrypt_test(NULL); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by wolfcrypt_test(): %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif if (ret < 0) { pr_err("ERROR: wolfcrypt self-test failed with return code %d.\n", ret); (void)libwolfssl_cleanup(); msleep(10); return -ECANCELED; } pr_info("wolfCrypt self-test passed.\n"); #else #if !defined(HAVE_FIPS) || FIPS_VERSION3_LE(5,2,0) || defined(WOLFSSL_FIPS_DEV_NO_POST) pr_info("skipping full wolfcrypt_test() " "(configure with --enable-crypttests to enable).\n"); #endif #endif #ifdef LINUXKM_LKCAPI_REGISTER #ifdef LINUXKM_LKCAPI_REGISTER_ONLY_ON_COMMAND ret = linuxkm_lkcapi_sysfs_install(); if (ret) { pr_err("ERROR: linuxkm_lkcapi_sysfs_install() failed with return code %d.\n", ret); (void)libwolfssl_cleanup(); msleep(10); return -ECANCELED; } #else /* !LINUXKM_LKCAPI_REGISTER_ONLY_ON_COMMAND */ #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_current(); pr_info("STACK INFO: usage at call to linuxkm_lkcapi_register(): %lu of %lu total\n", stack_usage, THREAD_SIZE); wc_linuxkm_stack_hwm_prepare(0xee); #endif ret = linuxkm_lkcapi_register(); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by linuxkm_lkcapi_register(): %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif if (ret) { pr_err("ERROR: linuxkm_lkcapi_register() failed with return code %d.\n", ret); linuxkm_lkcapi_unregister(); (void)libwolfssl_cleanup(); msleep(10); return -ECANCELED; } #endif /* !LINUXKM_LKCAPI_REGISTER_ONLY_ON_COMMAND */ #endif /* LINUXKM_LKCAPI_REGISTER */ #ifdef HAVE_FIPS (void)linuxkm_sysfs_install_attr(&FIPS_rerun_self_test_attr.attr, &installed_sysfs_FIPS_files); #endif #ifdef WOLFSSL_LINUXKM_BENCHMARKS #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_current(); pr_info("STACK INFO: usage at call to wolfcrypt_benchmark_main(): %lu of %lu total\n", stack_usage, THREAD_SIZE); wc_linuxkm_stack_hwm_prepare(0xee); #endif wolfcrypt_benchmark_main(0, (char**)NULL); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by wolfcrypt_benchmark_main(): %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif #endif #ifdef WOLFCRYPT_ONLY pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded%s" ".\nSee https://www.wolfssl.com/ for more information.\n" "wolfCrypt Copyright (C) 2006-present wolfSSL Inc. Licensed under " WOLFSSL_LICENSE ".\n", #ifdef CONFIG_MODULE_SIG THIS_MODULE->sig_ok ? " with valid module signature" : " without valid module signature" #else "" #endif ); #else pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " loaded%s" ".\nSee https://www.wolfssl.com/ for more information.\n" "wolfSSL Copyright (C) 2006-present wolfSSL Inc. Licensed under " WOLFSSL_LICENSE ".\n", #ifdef CONFIG_MODULE_SIG THIS_MODULE->sig_ok ? " with valid module signature" : " without valid module signature" #else "" #endif ); #endif return 0; } module_init(wolfssl_init); static void wc_linuxkm_deinstall(void) { #ifdef HAVE_FIPS (void)linuxkm_sysfs_deinstall_attr(&FIPS_rerun_self_test_attr.attr, &installed_sysfs_FIPS_files); #ifdef FIPS_OPTEST (void)linuxkm_sysfs_deinstall_attr(&FIPS_optest_trig_attr.attr, &installed_sysfs_FIPS_optest_trig_files); #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { (void)linuxkm_sysfs_deinstall_attr(&FIPS_optest_trig_audit_accel_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_accel_files); (void)linuxkm_sysfs_deinstall_attr(&FIPS_optest_trig_audit_c_attr.attr, &installed_sysfs_FIPS_optest_trig_audit_c_files); } #endif #endif #endif #ifdef LINUXKM_LKCAPI_REGISTER (void)linuxkm_lkcapi_unregister(); (void)linuxkm_lkcapi_sysfs_deinstall(); #endif } #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0) static void __exit wolfssl_exit(void) #else static void wolfssl_exit(void) #endif { wc_linuxkm_deinstall(); #if defined(HAVE_FIPS) && !defined(WOLFSSL_FIPS_DEV_NO_POST) { int ret = wc_RunAllCast_fips(); if (ret != 0) { pr_err("ERROR: wc_RunAllCast_fips() failed at shutdown with return value %d\n", ret); } else pr_info("wolfCrypt FIPS re-self-test succeeded at unload: all algorithms re-verified.\n"); } #endif (void)libwolfssl_cleanup(); return; } module_exit(wolfssl_exit); #if defined(LINUXKM_LKCAPI_REGISTER) || !defined(WOLFSSL_NO_ASM) /* When registering algorithms with crypto_register_skcipher() and friends, * or using kernel_fpu_begin_mask() and _end() to wrap vector register * usage, we use a "GPL" license unconditionally here to meet the GPL-only * requirements for those calls, satisfying license_is_gpl_compatible() (see * /usr/src/linux/include/linux/license.h). */ MODULE_LICENSE("GPL"); #else MODULE_LICENSE(WOLFSSL_LICENSE); #endif MODULE_AUTHOR("https://www.wolfssl.com/"); MODULE_DESCRIPTION("libwolfssl cryptographic and protocol facilities"); MODULE_VERSION(LIBWOLFSSL_VERSION_STRING); #ifdef WC_SYM_RELOC_TABLES extern const struct wc_reloc_table_ent wc_linuxkm_pie_text_reloc_tab[]; extern const unsigned int wc_linuxkm_pie_text_reloc_tab_length; extern const struct wc_reloc_table_ent wc_linuxkm_pie_rodata_reloc_tab[]; extern const unsigned int wc_linuxkm_pie_rodata_reloc_tab_length; static const struct wc_reloc_table_segments seg_map = { .start = 0, .end = 0, .text_start = (size_t)(uintptr_t)__wc_text_start, .text_end = (size_t)(uintptr_t)__wc_text_end, .text_reloc_tab.start = (size_t)(uintptr_t)wc_linuxkm_pie_text_reloc_tab, .text_reloc_tab.end = 0, .text_reloc_tab.len_start = (size_t)(uintptr_t)&wc_linuxkm_pie_text_reloc_tab_length, .text_reloc_tab.len_end = 0, .rodata_reloc_tab.start = (size_t)(uintptr_t)wc_linuxkm_pie_rodata_reloc_tab, .rodata_reloc_tab.end = 0, .rodata_reloc_tab.len_start = (size_t)(uintptr_t)&wc_linuxkm_pie_rodata_reloc_tab_length, .rodata_reloc_tab.len_end = 0, #ifdef HAVE_FIPS #ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS .fips_text_start = (size_t)(uintptr_t)__wc_text_start, .fips_text_end = (size_t)(uintptr_t)__wc_text_end, #else .fips_text_start = (size_t)(uintptr_t)wolfCrypt_FIPS_first, .fips_text_end = (size_t)(uintptr_t)wolfCrypt_FIPS_last, #endif #endif /* HAVE_FIPS */ .rodata_start = (size_t)(uintptr_t)__wc_rodata_start, .rodata_end = (size_t)(uintptr_t)__wc_rodata_end, #ifdef HAVE_FIPS #ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS .fips_rodata_start = (size_t)(uintptr_t)__wc_rodata_start, .fips_rodata_end = (size_t)(uintptr_t)__wc_rodata_end, #else .fips_rodata_start = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_start, .fips_rodata_end = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_end, #endif #if (FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE)) && !defined(WOLFSSL_FIPS_DEV_NO_POST) .verifyCore_start = (uintptr_t)verifyCore, .verifyCore_end = (uintptr_t)verifyCore + FIPS_IN_CORE_DIGEST_SIZE*2 + 1, #endif #endif /* HAVE_FIPS */ .data_start = (size_t)(uintptr_t)__wc_rwdata_start, .data_end = (size_t)(uintptr_t)__wc_rwdata_end, .bss_start = (size_t)(uintptr_t)__wc_bss_start, .bss_end = (size_t)(uintptr_t)__wc_bss_end, .text_is_live = 1 }; ssize_t wc_linuxkm_normalize_relocations( const u8 *seg_in, size_t *seg_in_out_len, u8 *seg_out, ssize_t *cur_index_p) { return wc_reloc_normalize_segment(seg_in, seg_in_out_len, seg_out, cur_index_p, &seg_map, #ifdef DEBUG_LINUXKM_PIE_SUPPORT &reloc_counts #else NULL #endif ); } ssize_t wc_linuxkm_normalize_relocations_noresize( const u8 *seg_in, size_t seg_in_len, u8 *seg_out, ssize_t *cur_index_p) { ssize_t ret; ret = wc_reloc_normalize_segment(seg_in, &seg_in_len, seg_out, cur_index_p, &seg_map, #ifdef DEBUG_LINUXKM_PIE_SUPPORT &reloc_counts #else NULL #endif ); if (ret < 0) return ret; if ((size_t)ret != seg_in_len) return -EINVAL; else return seg_in_len; } #elif defined(HAVE_FIPS) static const struct wc_reloc_table_segments seg_map = { .start = 0, .end = 0, #ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS .fips_text_start = (size_t)(uintptr_t)__wc_text_start, .fips_text_end = (size_t)(uintptr_t)__wc_text_end, #else .fips_text_start = (size_t)(uintptr_t)wolfCrypt_FIPS_first, .fips_text_end = (size_t)(uintptr_t)wolfCrypt_FIPS_last, #endif #ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS .fips_rodata_start = (size_t)(uintptr_t)__wc_rodata_start, .fips_rodata_end = (size_t)(uintptr_t)__wc_rodata_end, #else .fips_rodata_start = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_start, .fips_rodata_end = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_end, #endif #if (FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE)) && !defined(WOLFSSL_FIPS_DEV_NO_POST) .verifyCore_start = (uintptr_t)verifyCore, .verifyCore_end = (uintptr_t)verifyCore + FIPS_IN_CORE_DIGEST_SIZE*2 + 1 #endif }; #endif /* !WC_SYM_RELOC_TABLES && HAVE_FIPS */ #ifdef WC_LINUXKM_USE_HEAP_WRAPPERS #ifndef USE_KVREALLOC #error WC_LINUXKM_USE_HEAP_WRAPPERS requires USE_KVREALLOC #endif void *wc_linuxkm_malloc(size_t size) { if (in_nmi()) return NULL; else return kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), (wc_linuxkm_can_block() ? GFP_KERNEL : GFP_ATOMIC), NUMA_NO_NODE); } void wc_linuxkm_free(void *ptr) { if (in_nmi()) return; /* better to leak than to UB */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(7, 2, 0) if (wc_linuxkm_can_block()) kvfree(ptr); else kvfree_atomic(ptr); #else kvfree(ptr); #endif } void *wc_linuxkm_realloc(void *ptr, size_t newsize) { if (in_nmi()) return NULL; else return kvrealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (wc_linuxkm_can_block() ? GFP_KERNEL : GFP_ATOMIC)); } size_t wc_linuxkm_malloc_usable_size(void *ptr) { if ((ptr == NULL) || is_vmalloc_addr(ptr)) { #ifdef CONFIG_HAVE_KPROBES static typeof(find_vm_area) *find_vm_area_ptr = NULL; if (find_vm_area_ptr == NULL) { find_vm_area_ptr = my_kallsyms_lookup_name("find_vm_area"); if (find_vm_area_ptr == NULL) return 0; } if (ptr == NULL) return 0; else { struct vm_struct *vm = find_vm_area_ptr(ptr); if (vm) return get_vm_area_size(vm); else return 0; } #else return 0; #endif } else { return ksize(ptr); } } #endif /* WC_LINUXKM_USE_HEAP_WRAPPERS */ #ifdef WC_SYM_RELOC_TABLES /* get_current() is an inline or macro, depending on the target -- sidestep the * whole issue with a wrapper func. */ static struct task_struct *my_get_current_thread(void) { return get_current(); } /* preempt_count() is an inline function in arch/x86/include/asm/preempt.h that * accesses __preempt_count, which is an int array declared with * DECLARE_PER_CPU_CACHE_HOT. */ static int my_preempt_count(void) { return preempt_count(); } static int set_up_wolfssl_linuxkm_pie_redirect_table(void) { XMEMSET( &wolfssl_linuxkm_pie_redirect_table, 0, sizeof wolfssl_linuxkm_pie_redirect_table); #ifdef HAVE_FIPS wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_normalize_relocations = wc_linuxkm_normalize_relocations; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_normalize_relocations_noresize = wc_linuxkm_normalize_relocations_noresize; #endif #ifndef __ARCH_MEMCMP_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.memcmp = memcmp; #endif #ifdef CONFIG_ARM wolfssl_linuxkm_pie_redirect_table.__div0 = __div0; #endif #ifndef CONFIG_FORTIFY_SOURCE #ifndef __ARCH_MEMCPY_NO_REDIRECT #ifdef CONFIG_ARM64 /* The plain names resolve to the module's own definitions here, so name * the kernel's implementations (arch/arm64/lib/memcpy.S:243, memset.S:206). */ wolfssl_linuxkm_pie_redirect_table.memcpy = __memcpy; #else wolfssl_linuxkm_pie_redirect_table.memcpy = memcpy; #endif #endif #ifndef __ARCH_MEMSET_NO_REDIRECT #ifdef CONFIG_ARM64 wolfssl_linuxkm_pie_redirect_table.memset = __memset; #else wolfssl_linuxkm_pie_redirect_table.memset = memset; #endif #endif #ifndef __ARCH_MEMMOVE_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.memmove = memmove; #endif #endif /* !CONFIG_FORTIFY_SOURCE */ #ifndef __ARCH_STRCMP_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strcmp = strcmp; #endif #ifndef __ARCH_STRNCMP_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strncmp = strncmp; #endif #ifndef __ARCH_STRCASECMP_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strcasecmp = strcasecmp; #endif #ifndef __ARCH_STRNCASECMP_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strncasecmp = strncasecmp; #endif #ifndef __ARCH_STRLEN_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strlen = strlen; #endif #ifndef __ARCH_STRSTR_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strstr = strstr; #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(7, 2, 0) #ifndef __ARCH_STRNCPY_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strncpy = strncpy; #endif #endif #ifndef __ARCH_STRNCAT_NO_REDIRECT wolfssl_linuxkm_pie_redirect_table.strncat = strncat; #endif wolfssl_linuxkm_pie_redirect_table.kstrtoll = kstrtoll; #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \ (defined(RHEL_MAJOR) && \ ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5)))) wolfssl_linuxkm_pie_redirect_table._printk = _printk; #else wolfssl_linuxkm_pie_redirect_table.printk = printk; #endif #ifdef CONFIG_FORTIFY_SOURCE wolfssl_linuxkm_pie_redirect_table.__warn_printk = __warn_printk; #endif wolfssl_linuxkm_pie_redirect_table.snprintf = snprintf; wolfssl_linuxkm_pie_redirect_table._ctype = _ctype; #ifdef WC_LINUXKM_USE_HEAP_WRAPPERS wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_malloc = wc_linuxkm_malloc; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_free = wc_linuxkm_free; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_realloc = wc_linuxkm_realloc; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_malloc_usable_size = wc_linuxkm_malloc_usable_size; #else /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0) wolfssl_linuxkm_pie_redirect_table.kmalloc_noprof = kmalloc_noprof; wolfssl_linuxkm_pie_redirect_table.krealloc_node_align_noprof = krealloc_node_align_noprof; wolfssl_linuxkm_pie_redirect_table.kzalloc_noprof = kzalloc_noprof; wolfssl_linuxkm_pie_redirect_table.__kvmalloc_node_noprof = __kvmalloc_node_noprof; wolfssl_linuxkm_pie_redirect_table.__kmalloc_cache_noprof = __kmalloc_cache_noprof; #ifdef HAVE_KVREALLOC wolfssl_linuxkm_pie_redirect_table.kvrealloc_node_align_noprof = kvrealloc_node_align_noprof; #endif #elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0) wolfssl_linuxkm_pie_redirect_table.kmalloc_noprof = kmalloc_noprof; wolfssl_linuxkm_pie_redirect_table.krealloc_noprof = krealloc_noprof; wolfssl_linuxkm_pie_redirect_table.kzalloc_noprof = kzalloc_noprof; wolfssl_linuxkm_pie_redirect_table.__kvmalloc_node_noprof = __kvmalloc_node_noprof; wolfssl_linuxkm_pie_redirect_table.__kmalloc_cache_noprof = __kmalloc_cache_noprof; #ifdef HAVE_KVREALLOC wolfssl_linuxkm_pie_redirect_table.kvrealloc_noprof = kvrealloc_noprof; #endif #elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0) wolfssl_linuxkm_pie_redirect_table.kmalloc_noprof = kmalloc_noprof; wolfssl_linuxkm_pie_redirect_table.krealloc_noprof = krealloc_noprof; wolfssl_linuxkm_pie_redirect_table.kzalloc_noprof = kzalloc_noprof; wolfssl_linuxkm_pie_redirect_table.kvmalloc_node_noprof = kvmalloc_node_noprof; wolfssl_linuxkm_pie_redirect_table.kmalloc_trace_noprof = kmalloc_trace_noprof; #ifdef HAVE_KVREALLOC wolfssl_linuxkm_pie_redirect_table.kvrealloc_noprof = kvrealloc_noprof; #endif #else wolfssl_linuxkm_pie_redirect_table.kmalloc = kmalloc; wolfssl_linuxkm_pie_redirect_table.krealloc = krealloc; #ifdef HAVE_KVMALLOC wolfssl_linuxkm_pie_redirect_table.kvmalloc_node = kvmalloc_node; #endif #ifdef HAVE_KVREALLOC wolfssl_linuxkm_pie_redirect_table.kvrealloc = kvrealloc; #endif #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \ (defined(RHEL_MAJOR) && \ ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5)))) wolfssl_linuxkm_pie_redirect_table.kmalloc_trace = kmalloc_trace; #else wolfssl_linuxkm_pie_redirect_table.kmem_cache_alloc_trace = kmem_cache_alloc_trace; wolfssl_linuxkm_pie_redirect_table.kmalloc_order_trace = kmalloc_order_trace; #endif #endif wolfssl_linuxkm_pie_redirect_table.kfree = kfree; wolfssl_linuxkm_pie_redirect_table.ksize = ksize; #ifdef HAVE_KVMALLOC wolfssl_linuxkm_pie_redirect_table.kvfree = kvfree; #endif #endif /* !WC_LINUXKM_USE_HEAP_WRAPPERS */ #ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT wolfssl_linuxkm_pie_redirect_table.get_random_bytes = get_random_bytes; #endif wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_time_epoch_secs = wc_linuxkm_time_epoch_secs; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_time_epoch_msecs = wc_linuxkm_time_epoch_msecs; wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread; #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \ (defined(CONFIG_X86) || defined(CONFIG_ARM64) || defined(CONFIG_ARM)) wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_allocate_svr_states = wc_linuxkm_allocate_svr_states; wolfssl_linuxkm_pie_redirect_table.wc_can_save_vector_registers_x86 = wc_can_save_vector_registers_x86; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_free_svr_states = wc_linuxkm_free_svr_states; wolfssl_linuxkm_pie_redirect_table.wc_restore_vector_registers_x86 = wc_restore_vector_registers_x86; wolfssl_linuxkm_pie_redirect_table.wc_save_vector_registers_x86 = wc_save_vector_registers_x86; #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture. #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */ #ifndef CONFIG_PREEMPT_RT wolfssl_linuxkm_pie_redirect_table.__mutex_init = __mutex_init; #else wolfssl_linuxkm_pie_redirect_table.__rt_mutex_init = __rt_mutex_init; wolfssl_linuxkm_pie_redirect_table.rt_mutex_base_init = rt_mutex_base_init; wolfssl_linuxkm_pie_redirect_table.rt_spin_lock = rt_spin_lock; wolfssl_linuxkm_pie_redirect_table.rt_spin_unlock = rt_spin_unlock; #endif #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) wolfssl_linuxkm_pie_redirect_table.mutex_lock_nested = mutex_lock_nested; #else wolfssl_linuxkm_pie_redirect_table.mutex_lock = mutex_lock; #endif wolfssl_linuxkm_pie_redirect_table.mutex_unlock = mutex_unlock; #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) wolfssl_linuxkm_pie_redirect_table.mutex_destroy = mutex_destroy; #endif #ifdef HAVE_FIPS #ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS /* use __wc_text_start and __wc_text_end, not wolfCrypt_FIPS_first and * wolfCrypt_FIPS_last, thereby including the whole container in the HMAC * span. Note there are runtime asserts at entry to wolfssl_init() above * confirming that __wc_*_{start,end} correctly contain the wolfCrypt_FIPS_* * fenceposts. */ wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_first = __wc_text_start; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_last = __wc_text_end; /* ditto for wolfCrypt_FIPS_ro_start and wolfCrypt_FIPS_ro_end. */ wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_start = &__wc_rodata_start; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_end = &__wc_rodata_end; #else wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_first = wolfCrypt_FIPS_first; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_last = wolfCrypt_FIPS_last; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_start = &wolfCrypt_FIPS_ro_start; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_end = &wolfCrypt_FIPS_ro_end; #endif #if FIPS_VERSION3_GE(6,0,0) #ifndef NO_AES wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_AES_sanity = wolfCrypt_FIPS_AES_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_aes_ro_sanity = &wolfCrypt_FIPS_aes_ro_sanity; #if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT) wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_CMAC_sanity = wolfCrypt_FIPS_CMAC_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_cmac_ro_sanity = &wolfCrypt_FIPS_cmac_ro_sanity; #endif #endif #ifndef NO_DH wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_DH_sanity = wolfCrypt_FIPS_DH_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_dh_ro_sanity = &wolfCrypt_FIPS_dh_ro_sanity; #endif #ifdef HAVE_ECC wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ECC_sanity = wolfCrypt_FIPS_ECC_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ecc_ro_sanity = &wolfCrypt_FIPS_ecc_ro_sanity; #endif #ifdef HAVE_ED25519 wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ED25519_sanity = wolfCrypt_FIPS_ED25519_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ed25519_ro_sanity = &wolfCrypt_FIPS_ed25519_ro_sanity; #endif #ifdef HAVE_ED448 wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ED448_sanity = wolfCrypt_FIPS_ED448_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ed448_ro_sanity = &wolfCrypt_FIPS_ed448_ro_sanity; #endif wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_HMAC_sanity = wolfCrypt_FIPS_HMAC_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_hmac_ro_sanity = &wolfCrypt_FIPS_hmac_ro_sanity; #ifndef NO_KDF wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_KDF_sanity = wolfCrypt_FIPS_KDF_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_kdf_ro_sanity = &wolfCrypt_FIPS_kdf_ro_sanity; #endif #ifdef HAVE_PBKDF2 wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_PBKDF_sanity = wolfCrypt_FIPS_PBKDF_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_pbkdf_ro_sanity = &wolfCrypt_FIPS_pbkdf_ro_sanity; #endif #ifdef HAVE_HASHDRBG wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_DRBG_sanity = wolfCrypt_FIPS_DRBG_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_drbg_ro_sanity = &wolfCrypt_FIPS_drbg_ro_sanity; #endif #ifndef NO_RSA wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_RSA_sanity = wolfCrypt_FIPS_RSA_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_rsa_ro_sanity = &wolfCrypt_FIPS_rsa_ro_sanity; #endif #ifndef NO_SHA wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA_sanity = wolfCrypt_FIPS_SHA_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha_ro_sanity = &wolfCrypt_FIPS_sha_ro_sanity; #endif #ifndef NO_SHA256 wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA256_sanity = wolfCrypt_FIPS_SHA256_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha256_ro_sanity = &wolfCrypt_FIPS_sha256_ro_sanity; #endif #ifdef WOLFSSL_SHA512 wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA512_sanity = wolfCrypt_FIPS_SHA512_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha512_ro_sanity = &wolfCrypt_FIPS_sha512_ro_sanity; #endif #ifdef WOLFSSL_SHA3 wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA3_sanity = wolfCrypt_FIPS_SHA3_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha3_ro_sanity = &wolfCrypt_FIPS_sha3_ro_sanity; #endif /* PQ boundary members added in v7.0.0, staged ahead of their consumer: * fips_test.c DoInCoreCheck has no PQ sanity calls yet. */ #if defined(WOLFSSL_HAVE_MLKEM) && FIPS_VERSION3_GE(7,0,0) wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_MLKEM_sanity = wolfCrypt_FIPS_MLKEM_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_mlkem_ro_sanity = &wolfCrypt_FIPS_mlkem_ro_sanity; #endif #if defined(WOLFSSL_HAVE_MLDSA) && FIPS_VERSION3_GE(7,0,0) wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_MLDSA_sanity = wolfCrypt_FIPS_MLDSA_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_mldsa_ro_sanity = &wolfCrypt_FIPS_mldsa_ro_sanity; #endif #if defined(WOLFSSL_HAVE_SLHDSA) && FIPS_VERSION3_GE(7,0,0) wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SLHDSA_sanity = wolfCrypt_FIPS_SLHDSA_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_slhdsa_ro_sanity = &wolfCrypt_FIPS_slhdsa_ro_sanity; #endif #if defined(WOLFSSL_HAVE_LMS) && FIPS_VERSION3_GE(7,0,0) wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_LMS_sanity = wolfCrypt_FIPS_LMS_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_lms_ro_sanity = &wolfCrypt_FIPS_lms_ro_sanity; #endif #if defined(WOLFSSL_HAVE_XMSS) && FIPS_VERSION3_GE(7,0,0) wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_XMSS_sanity = wolfCrypt_FIPS_XMSS_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_xmss_ro_sanity = &wolfCrypt_FIPS_xmss_ro_sanity; #endif #ifndef WOLFSSL_FIPS_DEV_NO_POST wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_FT_sanity = wolfCrypt_FIPS_FT_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ft_ro_sanity = &wolfCrypt_FIPS_ft_ro_sanity; wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_f_ro_sanity = &wolfCrypt_FIPS_f_ro_sanity; wolfssl_linuxkm_pie_redirect_table.wc_RunAllCast_fips = wc_RunAllCast_fips; #endif /* !WOLFSSL_FIPS_DEV_NO_POST */ #endif /* FIPS_VERSION3_GE(6,0,0) */ #endif #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS) wolfssl_linuxkm_pie_redirect_table.GetCA = GetCA; #ifndef NO_SKID wolfssl_linuxkm_pie_redirect_table.GetCAByName = GetCAByName; #ifdef HAVE_OCSP wolfssl_linuxkm_pie_redirect_table.GetCAByKeyHash = GetCAByKeyHash; #endif /* HAVE_OCSP */ #endif /* NO_SKID */ #ifdef WOLFSSL_AKID_NAME wolfssl_linuxkm_pie_redirect_table.GetCAByAKID = GetCAByAKID; #endif /* WOLFSSL_AKID_NAME */ #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL) wolfssl_linuxkm_pie_redirect_table.wolfSSL_X509_NAME_add_entry_by_NID = wolfSSL_X509_NAME_add_entry_by_NID; wolfssl_linuxkm_pie_redirect_table.wolfSSL_X509_NAME_free = wolfSSL_X509_NAME_free; wolfssl_linuxkm_pie_redirect_table.wolfSSL_X509_NAME_new_ex = wolfSSL_X509_NAME_new_ex; #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */ #ifdef HAVE_OCSP wolfssl_linuxkm_pie_redirect_table.CheckOcspResponder = CheckOcspResponder; #endif #endif /* !WOLFCRYPT_ONLY && !NO_CERTS */ wolfssl_linuxkm_pie_redirect_table.dump_stack = dump_stack; wolfssl_linuxkm_pie_redirect_table.preempt_count = my_preempt_count; #ifndef _raw_spin_lock_irqsave wolfssl_linuxkm_pie_redirect_table._raw_spin_lock_irqsave = _raw_spin_lock_irqsave; #endif #ifndef _raw_spin_trylock wolfssl_linuxkm_pie_redirect_table._raw_spin_trylock = _raw_spin_trylock; #endif #ifndef _raw_spin_unlock_irqrestore wolfssl_linuxkm_pie_redirect_table._raw_spin_unlock_irqrestore = _raw_spin_unlock_irqrestore; #endif wolfssl_linuxkm_pie_redirect_table._cond_resched = _cond_resched; #ifndef WOLFSSL_LINUXKM_USE_MUTEXES wolfssl_linuxkm_pie_redirect_table.wc_lkm_LockMutex = wc_lkm_LockMutex; wolfssl_linuxkm_pie_redirect_table.wc_lkm_UnlockMutex = wc_lkm_UnlockMutex; #endif #ifdef CONFIG_ARM64 #ifndef CONFIG_ARCH_TEGRA #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) wolfssl_linuxkm_pie_redirect_table.alt_cb_patch_nops = alt_cb_patch_nops; #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) */ wolfssl_linuxkm_pie_redirect_table.queued_spin_lock_slowpath = queued_spin_lock_slowpath; #endif #endif wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_can_block = wc_linuxkm_can_block; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_can_long_loop = wc_linuxkm_can_long_loop; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_sig_ignore_begin = wc_linuxkm_sig_ignore_begin; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_sig_ignore_end = wc_linuxkm_sig_ignore_end; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_check_for_intr_signals = wc_linuxkm_check_for_intr_signals; wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_relax_long_loop = wc_linuxkm_relax_long_loop; #ifdef CONFIG_KASAN wolfssl_linuxkm_pie_redirect_table.kasan_disable_current = kasan_disable_current; wolfssl_linuxkm_pie_redirect_table.kasan_enable_current = kasan_enable_current; #endif /* runtime assert that the table has no null slots after initialization. */ { unsigned long *i; static_assert(sizeof(unsigned long) == sizeof(void *), "unexpected pointer size"); for (i = (unsigned long *)&wolfssl_linuxkm_pie_redirect_table; i < (unsigned long *)&wolfssl_linuxkm_pie_redirect_table._last_slot; ++i) if (*i == 0) { pr_err("ERROR: wolfCrypt container redirect table initialization was " "incomplete [%u].\n", (unsigned)(i-(unsigned long *)&wolfssl_linuxkm_pie_redirect_table)); return -EFAULT; } } return 0; } #endif /* WC_SYM_RELOC_TABLES */ #if defined(HAVE_FIPS) && defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE) && !defined(WOLFSSL_FIPS_DEV_NO_POST) #include PRAGMA_GCC_DIAG_PUSH PRAGMA_GCC("GCC diagnostic ignored \"-Wnested-externs\"") PRAGMA_GCC("GCC diagnostic ignored \"-Wpointer-arith\"") PRAGMA_GCC("GCC diagnostic ignored \"-Wunused-parameter\"") #include PRAGMA_GCC_DIAG_POP /* failsafe definitions for FIPS <5.3 */ #ifndef FIPS_IN_CORE_DIGEST_SIZE #ifndef NO_SHA256 #define FIPS_IN_CORE_DIGEST_SIZE 32 #define FIPS_IN_CORE_HASH_TYPE WC_SHA256 #define FIPS_IN_CORE_KEY_SZ 32 #define FIPS_IN_CORE_VERIFY_SZ FIPS_IN_CORE_KEY_SZ #elif defined(WOLFSSL_SHA384) #define FIPS_IN_CORE_DIGEST_SIZE 48 #define FIPS_IN_CORE_HASH_TYPE WC_SHA384 #define FIPS_IN_CORE_KEY_SZ 48 #define FIPS_IN_CORE_VERIFY_SZ FIPS_IN_CORE_KEY_SZ #else #error Unsupported FIPS hash alg. #endif #endif extern const char coreKey[FIPS_IN_CORE_KEY_SZ*2 + 1]; extern const unsigned int wolfCrypt_FIPS_ro_start[]; extern const unsigned int wolfCrypt_FIPS_ro_end[]; static int linux_fips_hmac_setkey(struct shash_desc *desc, const byte *key, word32 key_len) { int ret = crypto_shash_setkey(desc->tfm, key, key_len); if (ret) { pr_err("ERROR: crypto_shash_setkey failed: err %d\n", ret); return BAD_STATE_E; } ret = crypto_shash_init(desc); if (ret) { pr_err("ERROR: crypto_shash_init failed: err %d\n", ret); return BAD_STATE_E; } return 0; } static int linux_fips_hmac_update(struct shash_desc *desc, const byte *in, word32 in_len) { int ret = crypto_shash_update(desc, in, in_len); if (ret) { pr_err("ERROR: crypto_shash_update failed: err %d\n", ret); return BAD_STATE_E; } else return 0; } static int linux_fips_hmac_final(struct shash_desc *desc, byte *out, word32 out_sz) { int ret; (void)out_sz; ret = crypto_shash_final(desc, out); if (ret) { pr_err("ERROR: crypto_shash_final failed: err %d\n", ret); return BAD_STATE_E; } else return 0; } static int updateFipsHash(void) { struct crypto_shash *tfm = NULL; struct shash_desc *desc = NULL; int ret; word32 verifyCore_size = XSTRLEN(verifyCore) + 1; char *old_verifyCore = XMALLOC(verifyCore_size, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (old_verifyCore == NULL) return MEMORY_E; XMEMCPY(old_verifyCore, verifyCore, verifyCore_size); wc_static_assert(((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA256) || ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA384) || ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA512) || ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA3_256) || ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA3_384) || ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA3_512)); switch ((unsigned)FIPS_IN_CORE_HASH_TYPE) { case (unsigned)WC_SHA256: tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); break; case (unsigned)WC_SHA384: tfm = crypto_alloc_shash("hmac(sha384)", 0, 0); break; case (unsigned)WC_SHA512: tfm = crypto_alloc_shash("hmac(sha512)", 0, 0); break; case (unsigned)WC_SHA3_256: tfm = crypto_alloc_shash("hmac(sha3-256)", 0, 0); break; case (unsigned)WC_SHA3_384: tfm = crypto_alloc_shash("hmac(sha3-384)", 0, 0); break; case (unsigned)WC_SHA3_512: tfm = crypto_alloc_shash("hmac(sha3-512)", 0, 0); break; } if (IS_ERR(tfm)) { if (PTR_ERR(tfm) == -WC_NO_ERR_TRACE(ENOMEM)) { pr_err("ERROR: crypto_alloc_shash failed: out of memory\n"); ret = MEMORY_E; } else if (PTR_ERR(tfm) == -ENOENT) { pr_err("ERROR: crypto_alloc_shash failed: target kernel is missing algorithm implementation for hash type %u\n", FIPS_IN_CORE_HASH_TYPE); ret = NOT_COMPILED_IN; } else { pr_err("ERROR: crypto_alloc_shash failed with ret %d\n", (int)PTR_ERR(tfm)); ret = HASH_TYPE_E; } tfm = NULL; goto out; } { size_t desc_size = crypto_shash_descsize(tfm) + sizeof *desc; desc = XMALLOC(desc_size, NULL, DYNAMIC_TYPE_TMP_BUFFER); if (desc == NULL) { pr_err("ERROR: failed allocating desc.\n"); ret = MEMORY_E; goto out; } XMEMSET(desc, 0, desc_size); } desc->tfm = tfm; ret = wc_fips_generate_hash( &seg_map, FIPS_IN_CORE_DIGEST_SIZE, coreKey, desc, /* NOLINTBEGIN(clang-diagnostic-cast-function-type-strict) */ (wc_fips_verifyCore_hmac_setkey_fn)linux_fips_hmac_setkey, (wc_fips_verifyCore_hmac_update_fn)linux_fips_hmac_update, (wc_fips_verifyCore_hmac_final_fn)linux_fips_hmac_final, /* NOLINTEND(clang-diagnostic-cast-function-type-strict) */ verifyCore, &verifyCore_size, #if defined(DEBUG_LINUXKM_PIE_SUPPORT) && defined(WC_SYM_RELOC_TABLES) &reloc_counts #else NULL #endif ); if (ret < 0) goto out; if (strcmp(old_verifyCore, verifyCore) == 0) { #if defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG) pr_info("updateFipsHash: verifyCore already matches [%s]\n", verifyCore); #else pr_info("updateFipsHash: verifyCore already matches.\n"); #endif } else { #if defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG) pr_info("updateFipsHash: verifyCore updated [%s].\n", verifyCore); #else pr_info("updateFipsHash: verifyCore updated.\n"); #endif } ret = 0; out: if (tfm != NULL) crypto_free_shash(tfm); XFREE(desc, NULL, DYNAMIC_TYPE_TMP_BUFFER); XFREE(old_verifyCore, NULL, DYNAMIC_TYPE_TMP_BUFFER); return ret; } #endif /* HAVE_FIPS && WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE && !WOLFSSL_FIPS_DEV_NO_POST */ #ifdef CONFIG_HAVE_KPROBES static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name) { static typeof(kallsyms_lookup_name) *kallsyms_lookup_name_ptr = NULL; static struct kprobe kallsyms_lookup_name_kp = { .symbol_name = "kallsyms_lookup_name" }; unsigned long a; if (! kallsyms_lookup_name_ptr) { int ret; if (! wc_linuxkm_can_block()) return NULL; kallsyms_lookup_name_kp.addr = NULL; if ((ret = register_kprobe(&kallsyms_lookup_name_kp)) != 0) { #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG pr_err_once("ERROR: register_kprobe(&kallsyms_lookup_name_kp) failed: %d\n", ret); #endif return 0; } kallsyms_lookup_name_ptr = (typeof(kallsyms_lookup_name_ptr))kallsyms_lookup_name_kp.addr; unregister_kprobe(&kallsyms_lookup_name_kp); if (! kallsyms_lookup_name_ptr) { #ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG pr_err_once("ERROR: kallsyms_lookup_name_kp.addr is null.\n"); #endif return 0; } } a = kallsyms_lookup_name_ptr(name); return (void *)a; } #endif /* CONFIG_HAVE_KPROBES */ #ifdef HAVE_FIPS static wolfSSL_Atomic_Int in_fips_test = WOLFSSL_ATOMIC_INITIALIZER(-1); static ssize_t FIPS_rerun_self_test_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, size_t count) { int ret; (void)mattr; (void)mk; /* only recognize "1" and "1\n". */ if ((count < 1) || (count > 2) || (buf[0] != '1') || ((count == 2) && (buf[1] != '\n'))) { return -EINVAL; } { WC_ATOMIC_INT_ARG expected_in_fips_test = -1; if (! wolfSSL_Atomic_Int_CompareExchange( &in_fips_test, &expected_in_fips_test, task_pid_nr(current))) { pr_err("ERROR: %s called by pid %d while pid %d is running a FIPS test in another thread.\n", __func__, task_pid_nr(current), expected_in_fips_test); return -EBUSY; } } pr_info("wolfCrypt: rerunning FIPS self-test on command.\n"); if (WC_SIG_IGNORE_BEGIN() >= 0) { ret = wolfCrypt_IntegrityTest_fips(); (void)WC_SIG_IGNORE_END(); } else { pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n"); ret = -1; } if (ret != 0) { pr_err("ERROR: wolfCrypt_IntegrityTest_fips: error %d\n", ret); WOLFSSL_ATOMIC_STORE(in_fips_test, -1); return -EINVAL; } ret = wolfCrypt_GetStatus_fips(); if (ret != 0) { WOLFSSL_ATOMIC_STORE(in_fips_test, -1); pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret)); if (ret == WC_NO_ERR_TRACE(IN_CORE_FIPS_E)) return -ELIBBAD; else return -EINVAL; } ret = wc_RunAllCast_fips(); if (ret != 0) { WOLFSSL_ATOMIC_STORE(in_fips_test, -1); pr_err("ERROR: wc_RunAllCast_fips() failed with return value %d\n", ret); return -EINVAL; } #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { /* Note that wc_svr_disallowed_count*() can't be checked in * FIPS_rerun_self_test_handler() -- we're already multiuser at this point * and other threads can and will increment wc_svr_disallowed_count outside * our control. */ ret = DISABLE_VECTOR_REGISTERS(); if (ret != 0) { WOLFSSL_ATOMIC_STORE(in_fips_test, -1); pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for wc_RunAllCast_fips() returned %d.\n", ret); return -EINVAL; } ret = wc_RunAllCast_fips(); REENABLE_VECTOR_REGISTERS(); if (ret != 0) { WOLFSSL_ATOMIC_STORE(in_fips_test, -1); pr_err("ERROR: wc_RunAllCast_fips() with DISABLE_VECTOR_REGISTERS() returned %d.\n", ret); return -EINVAL; } } #endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ WOLFSSL_ATOMIC_STORE(in_fips_test, -1); pr_info("wolfCrypt FIPS re-self-test succeeded: all algorithms verified and available.\n"); return count; } #ifdef FIPS_OPTEST typedef struct test_func_args { int return_code; } test_func_args; enum FIPS_optest_audit_mode { FIPS_OPTEST_AUDIT_NONE = 0, FIPS_OPTEST_AUDIT_ACCEL, FIPS_OPTEST_AUDIT_C }; static ssize_t FIPS_optest_trig_common(enum FIPS_optest_audit_mode audit_mode, const char *buf, const size_t count) { int ret; int argc; const char *argv[3]; char code_buf[6]; size_t corrected_count; int i; #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING long long unsigned int svr_disallowed_before_optest = 0, svr_disallowed_after_optest; ssize_t ret_count = 0; #else (void)audit_mode; #endif /* buf may or may not have an LF at end -- tolerate both. there is no * terminating null in either case. */ if (count < 1) return -EINVAL; if (buf[count-1] == '\n') corrected_count = count - 1; else corrected_count = count; if ((corrected_count < 1) || (corrected_count > 5)) return -EINVAL; XMEMCPY(code_buf, buf, corrected_count); code_buf[corrected_count] = 0; if (strspn(code_buf, "-0123456789") != corrected_count) return -EINVAL; argv[0] = "./optest"; argv[1] = "0"; argv[2] = code_buf; argc = 3; #ifdef LINUXKM_LKCAPI_REGISTER { int current_linuxkm_lkcapi_registering_now = 0; if (! wolfSSL_Atomic_Int_CompareExchange( &linuxkm_lkcapi_registering_now, ¤t_linuxkm_lkcapi_registering_now, 1)) { pr_err("ERROR: FIPS_optest_trig_handler() with linuxkm_lkcapi_registered: before running optest, echo 1 > /sys/module/libwolfssl/deinstall_algs.\n"); return -EBUSY; } if (linuxkm_lkcapi_registered) { WOLFSSL_ATOMIC_STORE(linuxkm_lkcapi_registering_now, 0); pr_err("ERROR: FIPS_optest_trig_handler() with linuxkm_lkcapi_registered: before running optest, echo 1 > /sys/module/libwolfssl/deinstall_algs.\n"); return -EBUSY; } } #endif { WC_ATOMIC_INT_ARG expected_in_fips_test = -1; if (! wolfSSL_Atomic_Int_CompareExchange( &in_fips_test, &expected_in_fips_test, task_pid_nr(current))) { #ifdef LINUXKM_LKCAPI_REGISTER WOLFSSL_ATOMIC_STORE(linuxkm_lkcapi_registering_now, 0); #endif pr_err("ERROR: %s called by pid %d while pid %d is running a FIPS test in another thread.\n", __func__, task_pid_nr(current), expected_in_fips_test); return -EBUSY; } } printf("OK, testing code %s\n", code_buf); #ifdef WC_LINUXKM_HAVE_STACK_DEBUG { unsigned long stack_usage; stack_usage = wc_linuxkm_stack_current(); pr_info("STACK INFO: usage at call to linuxkm_op_test_1(): %lu of %lu total\n", stack_usage, THREAD_SIZE); wc_linuxkm_stack_hwm_prepare(0xee); #endif #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { /* The audits treat wc_svr_disallowed_count as ours alone: in_fips_test * excludes the other trigger nodes, and getting * linuxkm_lkcapi_registering_now then checking linuxkm_lkcapi_registered * excludes registered-algorithm consumers. Optests entail transient * degradation states, necessitating the registered-algorithm exclusion * bidirectionally. */ if ((audit_mode == FIPS_OPTEST_AUDIT_ACCEL) || (audit_mode == FIPS_OPTEST_AUDIT_C)) { if (audit_mode == FIPS_OPTEST_AUDIT_C) { ret = DISABLE_VECTOR_REGISTERS(); if (ret != 0) { pr_err("ERROR: DISABLE_VECTOR_REGISTERS() for FIPS_optest_trig_handler() returned %d.\n", ret); ret_count = -EINVAL; goto out; } } svr_disallowed_before_optest = wc_svr_disallowed_count_current(); } } #endif ret = linuxkm_op_test_1(argc, &argv[0]); #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { if (audit_mode == FIPS_OPTEST_AUDIT_C) { REENABLE_VECTOR_REGISTERS(); svr_disallowed_after_optest = wc_svr_disallowed_count_current(); if (svr_disallowed_after_optest == svr_disallowed_before_optest) { pr_err("ERROR: wc_svr_disallowed_count_current() did not increment during optest with DISABLE_VECTOR_REGISTERS().\n"); ret_count = -EINVAL; } } else if (audit_mode == FIPS_OPTEST_AUDIT_ACCEL) { svr_disallowed_after_optest = wc_svr_disallowed_count_current(); if (svr_disallowed_after_optest != svr_disallowed_before_optest) { pr_err("ERROR: wc_svr_disallowed_count_current() incremented (+%llu) during optest.\n", svr_disallowed_after_optest - svr_disallowed_before_optest); ret_count = -EINVAL; } } } #endif #ifdef WC_LINUXKM_HAVE_STACK_DEBUG stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee); pr_info("STACK INFO: rel usage by linuxkm_op_test_1(): %lu\n", stack_usage); /* shush up false stack HWM reading by kernel: */ wc_linuxkm_stack_hwm_prepare(0); } #endif printf("ret of op_test = %d\n", ret); #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING out: #endif /* reload the library in memory and re-init state */ printf("Reloading the module in memory (equivalent to power " "cycle)\n"); WOLFSSL_ATOMIC_STORE(*conTestFailure_ptr, 0); for (i = 0; i < FIPS_CAST_COUNT; ++i) fipsCastStatus_put(i, FIPS_CAST_STATE_INIT); /* note, must call fipsEntry() here, not wolfCrypt_IntegrityTest_fips(), * because wc_GetCastStatus_fips(FIPS_CAST_HMAC_SHA2_256) isn't available * anymore. */ if (WC_SIG_IGNORE_BEGIN() >= 0) { fipsEntry(); (void)WC_SIG_IGNORE_END(); } else pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n"); ret = wolfCrypt_GetStatus_fips(); printf("Status indicator of library reload/powercycle: %d\n", ret); printf("Module status is: %d\n", wolfCrypt_GetStatus_fips()); printf("Module mode is: %d\n", wolfCrypt_GetMode_fips()); WOLFSSL_ATOMIC_STORE(in_fips_test, -1); #ifdef LINUXKM_LKCAPI_REGISTER WOLFSSL_ATOMIC_STORE(linuxkm_lkcapi_registering_now, 0); #endif #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING if (svr_dynamic_auditing) { return ret_count ? ret_count : (ssize_t)count; } else #endif { return (ssize_t)count; } } static ssize_t FIPS_optest_trig_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, const size_t count) { (void)mattr; (void)mk; return FIPS_optest_trig_common(FIPS_OPTEST_AUDIT_NONE, buf, count); } #ifdef WC_LINUXKM_SVR_DYNAMIC_AUDITING static ssize_t FIPS_optest_trig_audit_accel_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, const size_t count) { (void)mattr; (void)mk; return FIPS_optest_trig_common(FIPS_OPTEST_AUDIT_ACCEL, buf, count); } static ssize_t FIPS_optest_trig_audit_c_handler(WC_MODULE_ATTR_CONST struct module_attribute *mattr, struct module_kobject *mk, const char *buf, const size_t count) { (void)mattr; (void)mk; return FIPS_optest_trig_common(FIPS_OPTEST_AUDIT_C, buf, count); } #endif /* WC_LINUXKM_SVR_DYNAMIC_AUDITING */ #endif /* FIPS_OPTEST */ #endif /* HAVE_FIPS */