From 99a3511376ff40fa9b21d14c3696ab65ee8a3eba Mon Sep 17 00:00:00 2001 From: Daniele Lacamera Date: Mon, 17 Aug 2026 09:25:11 +0200 Subject: [PATCH] tests: sweep the chacha20/poly1305 AVX-512 dispatch rows --- tests/include.am | 1 + .../test_chacha20_poly1305_whitebox.c | 326 ++++++++++++++++++ tests/unit-mcdc/test_chacha_whitebox.c | 171 +++++++++ tests/unit-mcdc/test_poly1305_whitebox.c | 212 ++++++++++++ 4 files changed, 710 insertions(+) create mode 100644 tests/unit-mcdc/test_chacha20_poly1305_whitebox.c diff --git a/tests/include.am b/tests/include.am index d933ff6ea4..5e5ef55ef4 100644 --- a/tests/include.am +++ b/tests/include.am @@ -137,6 +137,7 @@ EXTRA_DIST += \ tests/unit-mcdc/test_asn_whitebox.c \ tests/unit-mcdc/test_blake2b_whitebox.c \ tests/unit-mcdc/test_blake2s_whitebox.c \ + tests/unit-mcdc/test_chacha20_poly1305_whitebox.c \ tests/unit-mcdc/test_chacha_whitebox.c \ tests/unit-mcdc/test_cmac_whitebox.c \ tests/unit-mcdc/test_cryptocb_whitebox.c \ diff --git a/tests/unit-mcdc/test_chacha20_poly1305_whitebox.c b/tests/unit-mcdc/test_chacha20_poly1305_whitebox.c new file mode 100644 index 0000000000..a0f1d36517 --- /dev/null +++ b/tests/unit-mcdc/test_chacha20_poly1305_whitebox.c @@ -0,0 +1,326 @@ +/* test_chacha20_poly1305_whitebox.c + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* White-box MC/DC supplement for wolfcrypt/src/chacha20_poly1305.c. + * + * Three groups of conditions, all in code the AEAD rework added. + * + * 1. AAD argument guards, wc_ChaCha20Poly1305_Encrypt/_Decrypt and + * wc_ChaCha20Poly1305_UpdateAad: + * + * if (!inKey || !inIV || inPlaintext == NULL || + * (inAADLen > 0 && inAAD == NULL) || !outCiphertext || !outAuthTag) + * if (aead == NULL || (inAAD == NULL && inAADLen > 0)) + * + * The AEAD tests pass either a real AAD buffer with a non-zero length or no + * AAD at all, so the "length without a buffer" combination -- the one the + * guard exists for -- is never built, and neither operand of it is paired. + * This file passes inAAD == NULL with a non-zero length, alongside the + * accepting shapes (AAD present, and no AAD at all) in the same binary. + * + * 2. wc_ChaCha20Poly1305_UpdateAad's + * + * if (inAAD && inAADLen > 0) + * + * first operand FALSE. inAAD == NULL only survives the guard above at zero + * length, i.e. exactly the "no AAD this call" no-op the tests never make. + * + * 3. wc_ChaCha20Poly1305_UpdateData's two + * + * if (ret == 0 && len > 0) + * + * guards (encrypt and decrypt arms), "len > 0" FALSE half: a zero-length + * data update, valid but unused by the tests. + * + * NOT closable, recorded in campaign/db/exclusions.json (three conditions, + * one argument): + * + * :943 if (ret == 0 && aead->state == CHACHA20_POLY1305_STATE_AAD) + * :975 if (ret == 0 && len > 0) (encrypt arm) + * :999 if (ret == 0 && len > 0) (decrypt arm) + * + * all three "ret == 0" operands. ret is initialised to 0 at :904; the only + * statement that can assign it before :943 is at :938, inside the + * WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA block, and it is a wc_Poly1305Update + * on &aead->poly (aead is checked non-NULL at :907) with a local buffer, so + * it cannot fail. :975 and :999 are additionally dominated by the enclosing + * "if (ret == 0)" at :951, and the only assignment between that and them is + * chacha20_poly1305_stitch_chunk() at :968/:992, which returns 0 except + * through SAVE_VECTOR_REGISTERS -- the literal 0 in a userspace build + * (types.h), the same dispatch residual already documented for the + * aes/chacha/sha save-vector guards. + * + * wb_updatedata_pad_fault() below is the evidence for the :975/:999 half of + * that argument rather than a closure attempt: it interposes + * wc_Poly1305_Pad at preprocessing time (the mcdc_fault_hash.h technique -- + * the wrapper is defined against the real declaration, and only then is the + * name #defined to the wrapper, so the involved .c's call sites, and only + * those, are redirected) and makes the AAD padding at :944 fail. That does + * drive ret non-zero, and the guards at :975/:999 are still not reached + * with it, because :951 skips the whole block -- which is exactly why no + * call shape pairs their first operand. + * + * No ciphertext or tag value is checked here: the AEAD KATs own correctness, + * this file only drives the decisions. Every failure is reported as a skip and + * main() always returns 0 -- a non-zero exit makes the campaign discard the + * whole variant. + */ + +/* Must be first, exactly as every wolfcrypt .c does it: establishes + * BUILDING_WOLFSSL / config.h before any wolfSSL declaration is seen, so + * including headers here does not change how the involved .c sees the world. */ +#include + +#include +#include +#include + +#include + +static int wb_fail = 0; +#define WB_NOTE(msg) do { printf(" [wb] %s\n", (msg)); } while (0) + +#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) + +/* ---- wc_Poly1305_Pad() interposer --------------------------------------- + * Declared before the macro below exists, so it reaches the real function. */ +static int wb_pad_armed = 0; + +static int wb_poly1305_pad(Poly1305* ctx, word32 lenToPad) +{ + if (wb_pad_armed) { + return WC_NO_ERR_TRACE(BAD_FUNC_ARG); + } + return wc_Poly1305_Pad(ctx, lenToPad); +} + +#define wc_Poly1305_Pad wb_poly1305_pad + +#endif /* HAVE_CHACHA && HAVE_POLY1305 */ + +#include + +#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) + +#define WB_PT_LEN 64 + +static const byte wb_key[CHACHA20_POLY1305_AEAD_KEYSIZE] = { + 0x80,0x81,0x82,0x83, 0x84,0x85,0x86,0x87, + 0x88,0x89,0x8a,0x8b, 0x8c,0x8d,0x8e,0x8f, + 0x90,0x91,0x92,0x93, 0x94,0x95,0x96,0x97, + 0x98,0x99,0x9a,0x9b, 0x9c,0x9d,0x9e,0x9f +}; +static const byte wb_iv[CHACHA20_POLY1305_AEAD_IV_SIZE] = { + 0x07,0x00,0x00,0x00, 0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47 +}; +static const byte wb_aad[8] = { + 0x50,0x51,0x52,0x53, 0xc0,0xc1,0xc2,0xc3 +}; + +/* ---- 1. one-shot Encrypt/Decrypt AAD argument guards -------------------- */ +static void wb_oneshot_aad_guard(void) +{ + byte pt[WB_PT_LEN]; + byte ct[WB_PT_LEN]; + byte out[WB_PT_LEN]; + byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE]; + word32 i; + + for (i = 0; i < (word32)sizeof(pt); i++) { + pt[i] = (byte)i; + } + + /* Rejecting vector: a non-zero AAD length with no AAD buffer. */ + if (wc_ChaCha20Poly1305_Encrypt(wb_key, wb_iv, NULL, (word32)sizeof(wb_aad), + pt, (word32)sizeof(pt), ct, tag) != + WC_NO_ERR_TRACE(BAD_FUNC_ARG)) { + WB_NOTE("Encrypt(aadLen>0, aad==NULL) not rejected"); + wb_fail = 1; + } + if (wc_ChaCha20Poly1305_Decrypt(wb_key, wb_iv, NULL, (word32)sizeof(wb_aad), + ct, (word32)sizeof(ct), tag, out) != + WC_NO_ERR_TRACE(BAD_FUNC_ARG)) { + WB_NOTE("Decrypt(aadLen>0, aad==NULL) not rejected"); + wb_fail = 1; + } + + /* Accepting vector, AAD present: length operand true, buffer operand + * false. */ + if (wc_ChaCha20Poly1305_Encrypt(wb_key, wb_iv, wb_aad, + (word32)sizeof(wb_aad), pt, (word32)sizeof(pt), ct, tag) != 0) { + WB_NOTE("Encrypt with AAD failed"); + wb_fail = 1; + } + else if (wc_ChaCha20Poly1305_Decrypt(wb_key, wb_iv, wb_aad, + (word32)sizeof(wb_aad), ct, (word32)sizeof(ct), tag, out) != 0) { + WB_NOTE("Decrypt with AAD failed"); + wb_fail = 1; + } + + /* Accepting vector, no AAD at all: length operand false. */ + if (wc_ChaCha20Poly1305_Encrypt(wb_key, wb_iv, NULL, 0, pt, + (word32)sizeof(pt), ct, tag) != 0) { + WB_NOTE("Encrypt without AAD failed"); + wb_fail = 1; + } + else if (wc_ChaCha20Poly1305_Decrypt(wb_key, wb_iv, NULL, 0, ct, + (word32)sizeof(ct), tag, out) != 0) { + WB_NOTE("Decrypt without AAD failed"); + wb_fail = 1; + } + + WB_NOTE("one-shot AAD length/buffer guard pairs exercised"); +} + +/* ---- 2./3. streaming UpdateAad / UpdateData guards ---------------------- */ +static void wb_stream_guards(int isEncrypt) +{ + ChaChaPoly_Aead aead; + byte in[WB_PT_LEN]; + byte out[WB_PT_LEN]; + byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE]; + const char* dir = isEncrypt ? "encrypt" : "decrypt"; + word32 i; + + for (i = 0; i < (word32)sizeof(in); i++) { + in[i] = (byte)(i + 1); + } + + XMEMSET(&aead, 0, sizeof(aead)); + if (wc_ChaCha20Poly1305_Init(&aead, wb_key, wb_iv, isEncrypt) != 0) { + printf(" [wb] Init(%s) failed; stream guards skipped\n", dir); + wb_fail = 1; + return; + } + + /* UpdateAad: length without a buffer is rejected ... */ + if (wc_ChaCha20Poly1305_UpdateAad(&aead, NULL, (word32)sizeof(wb_aad)) != + WC_NO_ERR_TRACE(BAD_FUNC_ARG)) { + printf(" [wb] UpdateAad(NULL, len>0) not rejected (%s)\n", dir); + wb_fail = 1; + } + /* ... while no buffer at zero length is the accepted no-op: it pairs the + * length operand of the guard and the buffer operand of the "is there any + * AAD to hash" test below it. */ + if (wc_ChaCha20Poly1305_UpdateAad(&aead, NULL, 0) != 0) { + printf(" [wb] UpdateAad(NULL, 0) rejected (%s)\n", dir); + wb_fail = 1; + } + /* Real AAD: both operands true. */ + if (wc_ChaCha20Poly1305_UpdateAad(&aead, wb_aad, + (word32)sizeof(wb_aad)) != 0) { + printf(" [wb] UpdateAad(aad, len>0) failed (%s)\n", dir); + wb_fail = 1; + } + + /* UpdateData with nothing to do: the AAD padding still runs (state is + * AAD), so ret == 0 and the length operand is false. */ + if (wc_ChaCha20Poly1305_UpdateData(&aead, in, out, 0) != 0) { + printf(" [wb] UpdateData(len==0) failed (%s)\n", dir); + wb_fail = 1; + } + /* The accepting vector for the same guard, same binary. */ + if (wc_ChaCha20Poly1305_UpdateData(&aead, in, out, + (word32)sizeof(in)) != 0) { + printf(" [wb] UpdateData(len>0) failed (%s)\n", dir); + wb_fail = 1; + } + if (wc_ChaCha20Poly1305_Final(&aead, tag) != 0) { + printf(" [wb] Final failed (%s)\n", dir); + wb_fail = 1; + } + + printf(" [wb] streaming AAD/data guard pairs exercised (%s)\n", dir); +} + +/* ---- 3b. AAD-padding fault: evidence for the "ret == 0" exclusions ------ + * See the file header. Faulting the pad at :944 sets ret non-zero, and the + * guards at :975/:999 are then not evaluated at all because :951 skips the + * block -- so their first operand has no independence pair through any call + * shape. Kept as an executed error path, not as a closure. */ +static void wb_updatedata_pad_fault(int isEncrypt) +{ + ChaChaPoly_Aead aead; + byte in[WB_PT_LEN]; + byte out[WB_PT_LEN]; + const char* dir = isEncrypt ? "encrypt" : "decrypt"; + int ret; + word32 i; + + for (i = 0; i < (word32)sizeof(in); i++) { + in[i] = (byte)(i + 2); + } + + XMEMSET(&aead, 0, sizeof(aead)); + if (wc_ChaCha20Poly1305_Init(&aead, wb_key, wb_iv, isEncrypt) != 0) { + printf(" [wb] Init(%s) failed; pad-fault case skipped\n", dir); + wb_fail = 1; + return; + } + /* State must be AAD so that UpdateData runs the padding step. */ + if (wc_ChaCha20Poly1305_UpdateAad(&aead, wb_aad, + (word32)sizeof(wb_aad)) != 0) { + printf(" [wb] UpdateAad failed; pad-fault case skipped (%s)\n", dir); + wb_fail = 1; + return; + } + + wb_pad_armed = 1; + ret = wc_ChaCha20Poly1305_UpdateData(&aead, in, out, (word32)sizeof(in)); + wb_pad_armed = 0; + if (ret == 0) { + printf(" [wb] faulted UpdateData unexpectedly succeeded (%s)\n", dir); + wb_fail = 1; + } + /* The aead is poisoned: no tag is taken from it, and it is not reused. */ + chacha20_poly1305_aead_zero(&aead); + + printf(" [wb] UpdateData with a failed AAD pad exercised (%s)\n", dir); +} + +static void wb_run(void) +{ + wb_oneshot_aad_guard(); + wb_stream_guards(CHACHA20_POLY1305_AEAD_ENCRYPT); + wb_stream_guards(CHACHA20_POLY1305_AEAD_DECRYPT); + wb_updatedata_pad_fault(CHACHA20_POLY1305_AEAD_ENCRYPT); + wb_updatedata_pad_fault(CHACHA20_POLY1305_AEAD_DECRYPT); +} + +#else + +static void wb_run(void) +{ + WB_NOTE("HAVE_CHACHA/HAVE_POLY1305 off in this variant; skipped"); +} + +#endif /* HAVE_CHACHA && HAVE_POLY1305 */ + +int main(void) +{ + setvbuf(stdout, NULL, _IONBF, 0); + printf("chacha20_poly1305.c white-box MC/DC supplement\n"); + wb_run(); + printf("done (%s)\n", wb_fail ? "with skips" : "ok"); + /* Setup failures are surfaced as skips, not test failures: the campaign + * treats a nonzero exit as a failed variant and discards its coverage. */ + return 0; +} diff --git a/tests/unit-mcdc/test_chacha_whitebox.c b/tests/unit-mcdc/test_chacha_whitebox.c index 03a6f01278..96f14366b1 100644 --- a/tests/unit-mcdc/test_chacha_whitebox.c +++ b/tests/unit-mcdc/test_chacha_whitebox.c @@ -50,6 +50,38 @@ * "AVX1 only" or "neither" and letting the dispatch call the real * chacha_encrypt_avx1/chacha_encrypt_x64 asm is always safe: we never claim * a capability the CPU lacks, only hide one it has. + * + * --------------------------------------------------------------------------- + * AVX-512 dispatch ladder (wb_chacha_avx512_dispatch) + * --------------------------------------------------------------------------- + * The AVX-512 ChaCha work replaced that flat if/else with a length-and-feature + * ladder in wc_Chacha_Process, five compound decisions deep: + * + * msglen <= CHACHA_CHUNK_BYTES && IS_INTEL_AVX512_VL(f) == 0 && + * IS_INTEL_AVX1(f) -> 1-block AVX1 + * IS_INTEL_AVX512_VL(f) == 0 && msglen < 4*CHUNK && + * IS_INTEL_SSSE3(f) -> SSSE3 + * IS_INTEL_AVX512_VL(f) == 0 && msglen < 4*CHUNK -> scalar x64 + * chacha_avx512_beneficial(f) && msglen >= 16*CHUNK -> 512-bit zmm + * IS_INTEL_AVX512_VL(f) && msglen < 8*CHUNK -> AVX-512VL + * + * plus the predicate chacha_avx512_beneficial(): IS_INTEL_AVX512(f) && + * IS_INTEL_VAES(f). + * + * Two inputs decide all of them: the message length and the cpuid mask. One + * host only ever supplies one mask -- on an AVX-512VL machine every + * "IS_INTEL_AVX512_VL(f) == 0" operand is permanently false, which pins three + * of the five decisions to a single outcome no matter what lengths the tests + * use, and on a machine without AVX-512 the last two are pinned the other way. + * wb_chacha_process_case() drives one (mask, length) pair; the table in + * wb_chacha_avx512_dispatch() is the full sweep -- every row in this one + * binary, since llvm-cov derives MC/DC independence pairs per binary. + * + * Crash-safety, same rule as above and enforced by wb_hw(): each row's mask is + * built as (wanted & real cpuid_get_flags()), so a capability bit is only ever + * cleared, never claimed. On a host without AVX-512 the AVX-512 rows degrade + * into the rows below them in the ladder -- coverage is not gained, but no + * unsupported instruction is ever executed. */ #include @@ -127,6 +159,143 @@ static void wb_chacha_dispatch(void) #endif +/* ------------------------------------------------------------------------- + * AVX-512 dispatch ladder sweep. See the file header. + * ---------------------------------------------------------------------- */ +#if defined(HAVE_CHACHA) && defined(USE_INTEL_CHACHA_SPEEDUP) + +/* Capability bits are only ever removed from what the CPU really reports. */ +static cpuid_flags_t wb_hw(cpuid_flags_t wanted) +{ + return wanted & cpuid_get_flags(); +} + +/* Longest row: one 16-block zmm chunk plus a tail. */ +#define WB_CHACHA_MAX_LEN (20 * CHACHA_CHUNK_BYTES) + +static void wb_chacha_process_case(cpuid_flags_t flags, word32 msglen, + const char* what) +{ + static const byte key[CHACHA_MAX_KEY_SZ] = { + 0x00,0x01,0x02,0x03, 0x04,0x05,0x06,0x07, + 0x08,0x09,0x0a,0x0b, 0x0c,0x0d,0x0e,0x0f, + 0x10,0x11,0x12,0x13, 0x14,0x15,0x16,0x17, + 0x18,0x19,0x1a,0x1b, 0x1c,0x1d,0x1e,0x1f + }; + static const byte nonce[CHACHA_IV_BYTES] = { + 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x02 + }; + ChaCha enc; + byte plain[WB_CHACHA_MAX_LEN]; + byte cipher[WB_CHACHA_MAX_LEN]; + word32 i; + + for (i = 0; i < (word32)sizeof(plain); i++) { + plain[i] = (byte)i; + } + + /* A fresh key/IV per row: ctx->left from the previous row must not shift + * which branch of the ladder this row's length reaches. */ + if (wc_Chacha_SetKey(&enc, key, (word32)sizeof(key)) != 0 || + wc_Chacha_SetIV(&enc, nonce, 0) != 0) { + printf(" [wb] SetKey/SetIV failed (%s skipped)\n", what); + wb_fail = 1; + return; + } + cpuidFlags = flags; + if (wc_Chacha_Process(&enc, cipher, plain, msglen) != 0) { + printf(" [wb] wc_Chacha_Process failed (%s)\n", what); + wb_fail = 1; + } +} + +static void wb_chacha_avx512_dispatch(void) +{ + cpuid_flags_t saved_flags = cpuidFlags; + const word32 chunk = CHACHA_CHUNK_BYTES; + const cpuid_flags_t none = 0; + const cpuid_flags_t avx1 = wb_hw(CPUID_AVX1); + const cpuid_flags_t ssse3 = wb_hw(CPUID_SSSE3); + const cpuid_flags_t vl = wb_hw(CPUID_AVX512 | CPUID_AVX512_VL | + CPUID_VAES); + const cpuid_flags_t f512 = wb_hw(CPUID_AVX512); + const cpuid_flags_t zmm = wb_hw(CPUID_AVX512 | CPUID_VAES); + + /* One block or less, no AVX-512VL, AVX1 present: the 1-block AVX1 path. */ + wb_chacha_process_case(avx1, chunk / 2, "<=1 block, AVX1"); + /* Same length, AVX-512VL present: second operand of that test false. */ + wb_chacha_process_case(vl, chunk / 2, "<=1 block, AVX-512VL present"); + /* Same length, no AVX1 either: third operand false, scalar single block. */ + wb_chacha_process_case(none, chunk / 2, "<=1 block, no SIMD"); + /* 65..255 bytes, SSSE3 present, no AVX-512VL: the SSSE3 exact-block path. */ + wb_chacha_process_case(ssse3, 2 * chunk, "<4 blocks, SSSE3"); + /* Same length without SSSE3: falls to the scalar <4-block path. */ + wb_chacha_process_case(none, 2 * chunk, "<4 blocks, no SSSE3"); + /* Same length with AVX-512VL: the two "VL == 0" first operands are false + * and the AVX-512VL kernel handles it. */ + wb_chacha_process_case(vl, 2 * chunk, "<4 blocks, AVX-512VL"); + /* >=4 blocks: the length operand of the SSSE3 and scalar tests is false. */ + wb_chacha_process_case(ssse3, 8 * chunk, ">=4 blocks, SSSE3"); + /* >=4 blocks, nothing available: reaches the zmm and AVX-512VL tests with + * both their feature operands false. */ + wb_chacha_process_case(none, 8 * chunk, ">=4 blocks, no SIMD"); + /* AVX-512 without VAES: chacha_avx512_beneficial()'s second operand false, + * so the zmm kernel is not entered whatever the length. */ + wb_chacha_process_case(f512, 8 * chunk, "AVX-512 without VAES"); + /* AVX-512 + VAES and a >=16-block message: the 512-bit zmm kernel. */ + wb_chacha_process_case(zmm, 17 * chunk, ">=16 blocks, zmm"); + /* AVX-512 + VAES but under 16 blocks: the length operand false. */ + wb_chacha_process_case(zmm, 8 * chunk, "<16 blocks, zmm beneficial"); + + cpuidFlags = saved_flags; + WB_NOTE("chacha AVX-512 dispatch ladder sweep exercised"); +} + +#ifdef HAVE_INTEL_AVX512 +/* chacha_avx512_beneficial() is a pure predicate over a flags word: it runs no + * instruction from the feature it tests, so evaluating it directly over the + * four operand combinations pairs both of its conditions on any host, + * independently of what this machine implements. */ +static void wb_chacha_predicate(void) +{ + static volatile cpuid_flags_t rows[4]; + volatile int sink = 0; + int i; + + rows[0] = 0; + rows[1] = CPUID_AVX512; + rows[2] = CPUID_VAES; + rows[3] = CPUID_AVX512 | CPUID_VAES; + + for (i = 0; i < 4; i++) { + sink += chacha_avx512_beneficial(rows[i]); + } + (void)sink; + WB_NOTE("chacha_avx512_beneficial operand combinations exercised"); +} +#else +static void wb_chacha_predicate(void) +{ + WB_NOTE("HAVE_INTEL_AVX512 off; chacha_avx512_beneficial skipped"); +} +#endif + +#else + +static void wb_chacha_avx512_dispatch(void) +{ + WB_NOTE("USE_INTEL_CHACHA_SPEEDUP not compiled in this variant; " + "AVX-512 ladder sweep skipped"); +} + +static void wb_chacha_predicate(void) +{ + WB_NOTE("USE_INTEL_CHACHA_SPEEDUP not compiled in this variant; " + "predicate sweep skipped"); +} + +#endif + int main(void) { printf("chacha.c white-box supplement\n"); @@ -135,6 +304,8 @@ int main(void) return 0; #else wb_chacha_dispatch(); + wb_chacha_predicate(); + wb_chacha_avx512_dispatch(); printf("done (%s)\n", wb_fail ? "with skips" : "ok"); /* Setup failures are surfaced as skips, not test failures: the campaign * treats a nonzero exit as a failed variant and discards its coverage. */ diff --git a/tests/unit-mcdc/test_poly1305_whitebox.c b/tests/unit-mcdc/test_poly1305_whitebox.c index b567f5bd71..0a77f364ff 100644 --- a/tests/unit-mcdc/test_poly1305_whitebox.c +++ b/tests/unit-mcdc/test_poly1305_whitebox.c @@ -48,6 +48,55 @@ * (see db/modules.json poly1305 notes), so forcing intel_flags to "AVX1 * only" and letting the dispatch call the real poly1305_*_avx asm is always * safe: we never claim a capability the CPU lacks, only hide one it has. + * + * --------------------------------------------------------------------------- + * AVX-512 / IFMA / fused-flag dispatch (wb_poly1305_avx512_dispatch) + * --------------------------------------------------------------------------- + * The AVX-512 Poly1305 work turned the single-condition AVX2 test above into + * six compound decisions -- three in wc_Poly1305Update, three in + * wc_Poly1305Final -- with the same shape in both: + * + * if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) && + * poly1305_use_ifma(intel_flags)) <- 8-way IFMA, radix 2^44 + * if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) && + * poly1305_use_avx512(intel_flags)) <- 8-way vpmuludq + * if (!POLY1305_FORCE_SCALAR(ctx) && IS_INTEL_AVX2(intel_flags)) + * + * plus the two predicates they call: + * + * poly1305_use_avx512: IS_INTEL_AVX512(f) && IS_INTEL_VAES(f) + * poly1305_use_ifma: IS_INTEL_AVX512_IFMA(f) && IS_CPU_INTEL(f) + * + * Three inputs decide all six: the two ctx flags (set only by the fused + * ChaCha20-Poly1305 kernels in chacha20_poly1305.c, so no poly1305-only caller + * ever sets them) and intel_flags. Any one host reaches exactly one + * combination -- on an AMD host IS_CPU_INTEL is false while AVX-512 and VAES + * are present, so every call takes the 8-way vpmuludq path and the other five + * decisions are never evaluated with a differing operand. + * wb_poly1305_dispatch_case() drives one (intel_flags, forceAvx2, forceScalar) + * triple through SetKey/Update/Final; the table in + * wb_poly1305_avx512_dispatch() is the full combination sweep, every row of it + * in this one binary so that each condition's independence pair is derivable + * here (llvm-cov computes MC/DC per binary). + * + * Crash-safety rule, unchanged from above: a forced mask may only CLEAR + * capability bits the CPU has, never claim one it lacks. Each row's mask is + * built as (wanted & real cpuid_get_flags()), so on a host without AVX-512 the + * AVX-512 rows collapse onto the rows below them and are simply not gained -- + * never executed as an unsupported instruction. The one bit forced ON, + * CPUID_INTEL, is a vendor tag rather than a capability: poly1305_use_ifma()'s + * own comment states the gate is a throughput preference between two paths any + * IFMA-capable CPU can run (AMD Zen 4/5 implement IFMA), and the row that lets + * the IFMA kernel actually execute is additionally skipped unless the host + * really reports AVX-512 IFMA. + * + * The ctx-flag rows reproduce, on a poly1305-only context, exactly the state + * the fused kernels set up before calling Update/Final: forceAvx2 -> zero + * ctx->hh, started = 0 (chacha20_poly1305_encrypt_fused); forceScalar -> zero + * ctx->h, finished = 1, leftover = 0, started = 0 + * (chacha20_poly1305_encrypt_fused_ifma, wc_ChaCha20Poly1305_UpdateData). + * No MAC value is checked: which kernel computes the right tag is the KAT + * suite's job, this file only drives the dispatch decisions. */ #include @@ -108,6 +157,167 @@ static void wb_poly1305_dispatch(void) #endif +/* ------------------------------------------------------------------------- + * AVX-512 / IFMA / fused-flag dispatch sweep. See the file header. + * ---------------------------------------------------------------------- */ +#if defined(HAVE_POLY1305) && defined(USE_INTEL_POLY1305_SPEEDUP) + +/* Capability bits are only ever removed from what the CPU really reports, so a + * forced mask can never make the dispatch enter a path the host cannot run. */ +static cpuid_flags_t wb_hw(cpuid_flags_t wanted) +{ + return wanted & cpuid_get_flags(); +} + +/* Reproduce the context state the fused ChaCha20-Poly1305 kernels leave behind + * when they pin Update/Final to one path (chacha20_poly1305.c). */ +static void wb_poly1305_force(Poly1305* ctx, int forceAvx2, int forceScalar) +{ +#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED + if (forceAvx2) { + ctx->forceAvx2 = 1; + XMEMSET(ctx->hh, 0, sizeof(ctx->hh)); + ctx->started = 0; + ctx->leftover = 0; + } +#else + (void)forceAvx2; +#endif +#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA + if (forceScalar) { + ctx->forceScalar = 1; + XMEMSET(ctx->h, 0, sizeof(ctx->h)); + ctx->finished = 1; + ctx->started = 0; + ctx->leftover = 0; + } +#else + (void)forceScalar; +#endif + (void)ctx; +} + +/* One (intel_flags, forceAvx2, forceScalar) row: key, hash more than the + * 8-block buffer so the block kernels run, and finalise. */ +static void wb_poly1305_dispatch_case(cpuid_flags_t flags, int forceAvx2, + int forceScalar, const char* what) +{ + static const byte key[32] = { + 0x85,0xd6,0xbe,0x78,0x57,0x55,0x6d,0x33, + 0x7f,0x44,0x52,0xfe,0x42,0xd5,0x06,0xa8, + 0x01,0x03,0x80,0x8a,0xfb,0x0d,0xb2,0xfd, + 0x4a,0xbf,0xf6,0xaf,0x41,0x49,0xf5,0x1b + }; + Poly1305 ctx; + byte msg[17 * POLY1305_BLOCK_SIZE + 5]; + byte tag[WC_POLY1305_MAC_SZ]; + size_t i; + + for (i = 0; i < sizeof(msg); i++) { + msg[i] = (byte)i; + } + + XMEMSET(&ctx, 0, sizeof(ctx)); + /* Non-initializer value: cpuid_get_flags_ex() in SetKey keeps it. */ + intel_flags = flags; + if (wc_Poly1305SetKey(&ctx, key, (word32)sizeof(key)) != 0) { + printf(" [wb] wc_Poly1305SetKey failed (%s skipped)\n", what); + wb_fail = 1; + return; + } + wb_poly1305_force(&ctx, forceAvx2, forceScalar); + if (wc_Poly1305Update(&ctx, msg, (word32)sizeof(msg)) != 0 || + wc_Poly1305Final(&ctx, tag) != 0) { + printf(" [wb] Update/Final failed (%s)\n", what); + wb_fail = 1; + } +} + +static void wb_poly1305_avx512_dispatch(void) +{ + cpuid_flags_t saved_flags = intel_flags; + cpuid_flags_t real = cpuid_get_flags(); + /* Common base: the scalar and 4-way kernels this host really has. */ + const cpuid_flags_t avx1 = wb_hw(CPUID_AVX1); + const cpuid_flags_t avx2 = wb_hw(CPUID_AVX1 | CPUID_AVX2); + const cpuid_flags_t f512 = wb_hw(CPUID_AVX1 | CPUID_AVX2 | CPUID_AVX512); + const cpuid_flags_t fvaes = wb_hw(CPUID_AVX1 | CPUID_AVX2 | CPUID_AVX512 | + CPUID_VAES | CPUID_AVX512_IFMA); + + /* AVX2 absent: IS_INTEL_AVX2 false, every AVX-512 predicate false. */ + wb_poly1305_dispatch_case(avx1, 0, 0, "scalar (AVX1 only)"); + /* AVX2 present, no AVX-512: the 4-way path. */ + wb_poly1305_dispatch_case(avx2, 0, 0, "4-way AVX2"); + /* AVX-512 without VAES: poly1305_use_avx512() second operand false. */ + wb_poly1305_dispatch_case(f512, 0, 0, "AVX-512 without VAES"); + /* AVX-512 + VAES + IFMA, vendor bit absent: use_avx512 true (8-way + * vpmuludq), use_ifma's IS_CPU_INTEL operand false. */ + wb_poly1305_dispatch_case(fvaes, 0, 0, "8-way AVX-512, non-Intel vendor"); + /* Same, plus the Intel vendor tag: the IFMA path. Only when the host + * really implements AVX-512 IFMA -- the kernel is vpmadd52 asm. */ + if (IS_INTEL_AVX512(real) && IS_INTEL_AVX512_IFMA(real)) { + wb_poly1305_dispatch_case(fvaes | CPUID_INTEL, 0, 0, + "8-way AVX-512 IFMA"); + } + else { + WB_NOTE("host reports no AVX-512 IFMA; IFMA dispatch row skipped"); + } + /* Fused-kernel context flags: each pins the dispatch to one path. */ + wb_poly1305_dispatch_case(fvaes, 1, 0, "forceAvx2 (fused 4-way)"); + wb_poly1305_dispatch_case(fvaes, 0, 1, "forceScalar (fused IFMA stitch)"); + + intel_flags = saved_flags; + WB_NOTE("poly1305 AVX-512/IFMA/fused-flag dispatch sweep exercised"); +} + +#ifdef HAVE_INTEL_AVX512 +/* poly1305_use_avx512()/poly1305_use_ifma() are pure predicates over a flags + * word: no state, no instruction from the feature they test. Evaluating them + * directly over the four operand combinations pairs both conditions of each + * without depending on what this host implements, so the two predicates close + * on any machine; the dispatch sweep above then shows the same rows arising + * from real calls wherever the hardware allows. */ +static void wb_poly1305_predicates(void) +{ + static volatile cpuid_flags_t rows[4]; + volatile int sink = 0; + int i; + + rows[0] = 0; + rows[1] = CPUID_AVX512 | CPUID_AVX512_IFMA; + rows[2] = CPUID_VAES | CPUID_INTEL; + rows[3] = CPUID_AVX512 | CPUID_AVX512_IFMA | CPUID_VAES | CPUID_INTEL; + + for (i = 0; i < 4; i++) { + sink += poly1305_use_avx512(rows[i]); + sink += poly1305_use_ifma(rows[i]); + } + (void)sink; + WB_NOTE("poly1305_use_avx512/use_ifma operand combinations exercised"); +} +#else +static void wb_poly1305_predicates(void) +{ + WB_NOTE("HAVE_INTEL_AVX512 off; use_avx512/use_ifma predicates skipped"); +} +#endif + +#else + +static void wb_poly1305_avx512_dispatch(void) +{ + WB_NOTE("USE_INTEL_POLY1305_SPEEDUP not compiled in this variant; " + "AVX-512 dispatch sweep skipped"); +} + +static void wb_poly1305_predicates(void) +{ + WB_NOTE("USE_INTEL_POLY1305_SPEEDUP not compiled in this variant; " + "predicate sweep skipped"); +} + +#endif + /* ---- wc_Poly1305SetKey() argument guard (~line 850) ---------------------- * * * if ((ctx == NULL) || (key == NULL) || (keySz != 32)) @@ -188,6 +398,8 @@ int main(void) return 0; #else wb_poly1305_dispatch(); + wb_poly1305_predicates(); + wb_poly1305_avx512_dispatch(); wb_poly1305_setkey_guard(); printf("done (%s)\n", wb_fail ? "with skips" : "ok"); /* Setup failures are surfaced as skips, not test failures: the campaign