crypt/wolfssl
1
0
Fork 0
mirror of https://github.com/wolfSSL/wolfssl.git synced 2026-10-24 04:09:07 +03:00
Code Activity

tests: sweep the chacha20/poly1305 AVX-512 dispatch rows

Browse Source
This commit is contained in:
Daniele Lacamera
2026-08-17 09:25:11 +02:00
parent 73522e7096
commit 99a3511376
4 changed files with 710 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/include.am
View File

@@ -137,6 +137,7 @@ EXTRA_DIST += \
tests/unit-mcdc/test_asn_whitebox.c \
tests/unit-mcdc/test_blake2b_whitebox.c \
tests/unit-mcdc/test_blake2s_whitebox.c \
tests/unit-mcdc/test_chacha20_poly1305_whitebox.c \
tests/unit-mcdc/test_chacha_whitebox.c \
tests/unit-mcdc/test_cmac_whitebox.c \
tests/unit-mcdc/test_cryptocb_whitebox.c \

326
tests/unit-mcdc/test_chacha20_poly1305_whitebox.c Normal file
View File

@@ -0,0 +1,326 @@
/* test_chacha20_poly1305_whitebox.c
*
* Copyright (C) 2006-2026 wolfSSL Inc.
*
* This file is part of wolfSSL.
*
* wolfSSL is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* wolfSSL is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
*/
/* White-box MC/DC supplement for wolfcrypt/src/chacha20_poly1305.c.
*
* Three groups of conditions, all in code the AEAD rework added.
*
* 1. AAD argument guards, wc_ChaCha20Poly1305_Encrypt/_Decrypt and
* wc_ChaCha20Poly1305_UpdateAad:
*
* if (!inKey || !inIV || inPlaintext == NULL ||
* (inAADLen > 0 && inAAD == NULL) || !outCiphertext || !outAuthTag)
* if (aead == NULL || (inAAD == NULL && inAADLen > 0))
*
* The AEAD tests pass either a real AAD buffer with a non-zero length or no
* AAD at all, so the "length without a buffer" combination -- the one the
* guard exists for -- is never built, and neither operand of it is paired.
* This file passes inAAD == NULL with a non-zero length, alongside the
* accepting shapes (AAD present, and no AAD at all) in the same binary.
*
* 2. wc_ChaCha20Poly1305_UpdateAad's
*
* if (inAAD && inAADLen > 0)
*
* first operand FALSE. inAAD == NULL only survives the guard above at zero
* length, i.e. exactly the "no AAD this call" no-op the tests never make.
*
* 3. wc_ChaCha20Poly1305_UpdateData's two
*
* if (ret == 0 && len > 0)
*
* guards (encrypt and decrypt arms), "len > 0" FALSE half: a zero-length
* data update, valid but unused by the tests.
*
* NOT closable, recorded in campaign/db/exclusions.json (three conditions,
* one argument):
*
* :943 if (ret == 0 && aead->state == CHACHA20_POLY1305_STATE_AAD)
* :975 if (ret == 0 && len > 0) (encrypt arm)
* :999 if (ret == 0 && len > 0) (decrypt arm)
*
* all three "ret == 0" operands. ret is initialised to 0 at :904; the only
* statement that can assign it before :943 is at :938, inside the
* WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA block, and it is a wc_Poly1305Update
* on &aead->poly (aead is checked non-NULL at :907) with a local buffer, so
* it cannot fail. :975 and :999 are additionally dominated by the enclosing
* "if (ret == 0)" at :951, and the only assignment between that and them is
* chacha20_poly1305_stitch_chunk() at :968/:992, which returns 0 except
* through SAVE_VECTOR_REGISTERS -- the literal 0 in a userspace build
* (types.h), the same dispatch residual already documented for the
* aes/chacha/sha save-vector guards.
*
* wb_updatedata_pad_fault() below is the evidence for the :975/:999 half of
* that argument rather than a closure attempt: it interposes
* wc_Poly1305_Pad at preprocessing time (the mcdc_fault_hash.h technique --
* the wrapper is defined against the real declaration, and only then is the
* name #defined to the wrapper, so the involved .c's call sites, and only
* those, are redirected) and makes the AAD padding at :944 fail. That does
* drive ret non-zero, and the guards at :975/:999 are still not reached
* with it, because :951 skips the whole block -- which is exactly why no
* call shape pairs their first operand.
*
* No ciphertext or tag value is checked here: the AEAD KATs own correctness,
* this file only drives the decisions. Every failure is reported as a skip and
* main() always returns 0 -- a non-zero exit makes the campaign discard the
* whole variant.
*/
/* Must be first, exactly as every wolfcrypt .c does it: establishes
* BUILDING_WOLFSSL / config.h before any wolfSSL declaration is seen, so
* including headers here does not change how the involved .c sees the world. */
#include <wolfssl/wolfcrypt/libwolfssl_sources.h>
#include <wolfssl/wolfcrypt/chacha20_poly1305.h>
#include <wolfssl/wolfcrypt/poly1305.h>
#include <wolfssl/wolfcrypt/error-crypt.h>
#include <stdio.h>
static int wb_fail = 0;
#define WB_NOTE(msg) do { printf(" [wb] %s\n", (msg)); } while (0)
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
/* ---- wc_Poly1305_Pad() interposer ---------------------------------------
* Declared before the macro below exists, so it reaches the real function. */
static int wb_pad_armed = 0;
static int wb_poly1305_pad(Poly1305* ctx, word32 lenToPad)
{
if (wb_pad_armed) {
return WC_NO_ERR_TRACE(BAD_FUNC_ARG);
}
return wc_Poly1305_Pad(ctx, lenToPad);
}
#define wc_Poly1305_Pad wb_poly1305_pad
#endif /* HAVE_CHACHA && HAVE_POLY1305 */
#include <wolfcrypt/src/chacha20_poly1305.c>
#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305)
#define WB_PT_LEN 64
static const byte wb_key[CHACHA20_POLY1305_AEAD_KEYSIZE] = {
0x80,0x81,0x82,0x83, 0x84,0x85,0x86,0x87,
0x88,0x89,0x8a,0x8b, 0x8c,0x8d,0x8e,0x8f,
0x90,0x91,0x92,0x93, 0x94,0x95,0x96,0x97,
0x98,0x99,0x9a,0x9b, 0x9c,0x9d,0x9e,0x9f
};
static const byte wb_iv[CHACHA20_POLY1305_AEAD_IV_SIZE] = {
0x07,0x00,0x00,0x00, 0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47
};
static const byte wb_aad[8] = {
0x50,0x51,0x52,0x53, 0xc0,0xc1,0xc2,0xc3
};
/* ---- 1. one-shot Encrypt/Decrypt AAD argument guards -------------------- */
static void wb_oneshot_aad_guard(void)
{
byte pt[WB_PT_LEN];
byte ct[WB_PT_LEN];
byte out[WB_PT_LEN];
byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
word32 i;
for (i = 0; i < (word32)sizeof(pt); i++) {
pt[i] = (byte)i;
}
/* Rejecting vector: a non-zero AAD length with no AAD buffer. */
if (wc_ChaCha20Poly1305_Encrypt(wb_key, wb_iv, NULL, (word32)sizeof(wb_aad),
pt, (word32)sizeof(pt), ct, tag) !=
WC_NO_ERR_TRACE(BAD_FUNC_ARG)) {
WB_NOTE("Encrypt(aadLen>0, aad==NULL) not rejected");
wb_fail = 1;
}
if (wc_ChaCha20Poly1305_Decrypt(wb_key, wb_iv, NULL, (word32)sizeof(wb_aad),
ct, (word32)sizeof(ct), tag, out) !=
WC_NO_ERR_TRACE(BAD_FUNC_ARG)) {
WB_NOTE("Decrypt(aadLen>0, aad==NULL) not rejected");
wb_fail = 1;
}
/* Accepting vector, AAD present: length operand true, buffer operand
* false. */
if (wc_ChaCha20Poly1305_Encrypt(wb_key, wb_iv, wb_aad,
(word32)sizeof(wb_aad), pt, (word32)sizeof(pt), ct, tag) != 0) {
WB_NOTE("Encrypt with AAD failed");
wb_fail = 1;
}
else if (wc_ChaCha20Poly1305_Decrypt(wb_key, wb_iv, wb_aad,
(word32)sizeof(wb_aad), ct, (word32)sizeof(ct), tag, out) != 0) {
WB_NOTE("Decrypt with AAD failed");
wb_fail = 1;
}
/* Accepting vector, no AAD at all: length operand false. */
if (wc_ChaCha20Poly1305_Encrypt(wb_key, wb_iv, NULL, 0, pt,
(word32)sizeof(pt), ct, tag) != 0) {
WB_NOTE("Encrypt without AAD failed");
wb_fail = 1;
}
else if (wc_ChaCha20Poly1305_Decrypt(wb_key, wb_iv, NULL, 0, ct,
(word32)sizeof(ct), tag, out) != 0) {
WB_NOTE("Decrypt without AAD failed");
wb_fail = 1;
}
WB_NOTE("one-shot AAD length/buffer guard pairs exercised");
}
/* ---- 2./3. streaming UpdateAad / UpdateData guards ---------------------- */
static void wb_stream_guards(int isEncrypt)
{
ChaChaPoly_Aead aead;
byte in[WB_PT_LEN];
byte out[WB_PT_LEN];
byte tag[CHACHA20_POLY1305_AEAD_AUTHTAG_SIZE];
const char* dir = isEncrypt ? "encrypt" : "decrypt";
word32 i;
for (i = 0; i < (word32)sizeof(in); i++) {
in[i] = (byte)(i + 1);
}
XMEMSET(&aead, 0, sizeof(aead));
if (wc_ChaCha20Poly1305_Init(&aead, wb_key, wb_iv, isEncrypt) != 0) {
printf(" [wb] Init(%s) failed; stream guards skipped\n", dir);
wb_fail = 1;
return;
}
/* UpdateAad: length without a buffer is rejected ... */
if (wc_ChaCha20Poly1305_UpdateAad(&aead, NULL, (word32)sizeof(wb_aad)) !=
WC_NO_ERR_TRACE(BAD_FUNC_ARG)) {
printf(" [wb] UpdateAad(NULL, len>0) not rejected (%s)\n", dir);
wb_fail = 1;
}
/* ... while no buffer at zero length is the accepted no-op: it pairs the
* length operand of the guard and the buffer operand of the "is there any
* AAD to hash" test below it. */
if (wc_ChaCha20Poly1305_UpdateAad(&aead, NULL, 0) != 0) {
printf(" [wb] UpdateAad(NULL, 0) rejected (%s)\n", dir);
wb_fail = 1;
}
/* Real AAD: both operands true. */
if (wc_ChaCha20Poly1305_UpdateAad(&aead, wb_aad,
(word32)sizeof(wb_aad)) != 0) {
printf(" [wb] UpdateAad(aad, len>0) failed (%s)\n", dir);
wb_fail = 1;
}
/* UpdateData with nothing to do: the AAD padding still runs (state is
* AAD), so ret == 0 and the length operand is false. */
if (wc_ChaCha20Poly1305_UpdateData(&aead, in, out, 0) != 0) {
printf(" [wb] UpdateData(len==0) failed (%s)\n", dir);
wb_fail = 1;
}
/* The accepting vector for the same guard, same binary. */
if (wc_ChaCha20Poly1305_UpdateData(&aead, in, out,
(word32)sizeof(in)) != 0) {
printf(" [wb] UpdateData(len>0) failed (%s)\n", dir);
wb_fail = 1;
}
if (wc_ChaCha20Poly1305_Final(&aead, tag) != 0) {
printf(" [wb] Final failed (%s)\n", dir);
wb_fail = 1;
}
printf(" [wb] streaming AAD/data guard pairs exercised (%s)\n", dir);
}
/* ---- 3b. AAD-padding fault: evidence for the "ret == 0" exclusions ------
* See the file header. Faulting the pad at :944 sets ret non-zero, and the
* guards at :975/:999 are then not evaluated at all because :951 skips the
* block -- so their first operand has no independence pair through any call
* shape. Kept as an executed error path, not as a closure. */
static void wb_updatedata_pad_fault(int isEncrypt)
{
ChaChaPoly_Aead aead;
byte in[WB_PT_LEN];
byte out[WB_PT_LEN];
const char* dir = isEncrypt ? "encrypt" : "decrypt";
int ret;
word32 i;
for (i = 0; i < (word32)sizeof(in); i++) {
in[i] = (byte)(i + 2);
}
XMEMSET(&aead, 0, sizeof(aead));
if (wc_ChaCha20Poly1305_Init(&aead, wb_key, wb_iv, isEncrypt) != 0) {
printf(" [wb] Init(%s) failed; pad-fault case skipped\n", dir);
wb_fail = 1;
return;
}
/* State must be AAD so that UpdateData runs the padding step. */
if (wc_ChaCha20Poly1305_UpdateAad(&aead, wb_aad,
(word32)sizeof(wb_aad)) != 0) {
printf(" [wb] UpdateAad failed; pad-fault case skipped (%s)\n", dir);
wb_fail = 1;
return;
}
wb_pad_armed = 1;
ret = wc_ChaCha20Poly1305_UpdateData(&aead, in, out, (word32)sizeof(in));
wb_pad_armed = 0;
if (ret == 0) {
printf(" [wb] faulted UpdateData unexpectedly succeeded (%s)\n", dir);
wb_fail = 1;
}
/* The aead is poisoned: no tag is taken from it, and it is not reused. */
chacha20_poly1305_aead_zero(&aead);
printf(" [wb] UpdateData with a failed AAD pad exercised (%s)\n", dir);
}
static void wb_run(void)
{
wb_oneshot_aad_guard();
wb_stream_guards(CHACHA20_POLY1305_AEAD_ENCRYPT);
wb_stream_guards(CHACHA20_POLY1305_AEAD_DECRYPT);
wb_updatedata_pad_fault(CHACHA20_POLY1305_AEAD_ENCRYPT);
wb_updatedata_pad_fault(CHACHA20_POLY1305_AEAD_DECRYPT);
}
#else
static void wb_run(void)
{
WB_NOTE("HAVE_CHACHA/HAVE_POLY1305 off in this variant; skipped");
}
#endif /* HAVE_CHACHA && HAVE_POLY1305 */
int main(void)
{
setvbuf(stdout, NULL, _IONBF, 0);
printf("chacha20_poly1305.c white-box MC/DC supplement\n");
wb_run();
printf("done (%s)\n", wb_fail ? "with skips" : "ok");
/* Setup failures are surfaced as skips, not test failures: the campaign
* treats a nonzero exit as a failed variant and discards its coverage. */
return 0;
}

171
tests/unit-mcdc/test_chacha_whitebox.c
View File

@@ -50,6 +50,38 @@
* "AVX1 only" or "neither" and letting the dispatch call the real
* chacha_encrypt_avx1/chacha_encrypt_x64 asm is always safe: we never claim
* a capability the CPU lacks, only hide one it has.
*
* ---------------------------------------------------------------------------
* AVX-512 dispatch ladder (wb_chacha_avx512_dispatch)
* ---------------------------------------------------------------------------
* The AVX-512 ChaCha work replaced that flat if/else with a length-and-feature
* ladder in wc_Chacha_Process, five compound decisions deep:
*
* msglen <= CHACHA_CHUNK_BYTES && IS_INTEL_AVX512_VL(f) == 0 &&
* IS_INTEL_AVX1(f) -> 1-block AVX1
* IS_INTEL_AVX512_VL(f) == 0 && msglen < 4*CHUNK &&
* IS_INTEL_SSSE3(f) -> SSSE3
* IS_INTEL_AVX512_VL(f) == 0 && msglen < 4*CHUNK -> scalar x64
* chacha_avx512_beneficial(f) && msglen >= 16*CHUNK -> 512-bit zmm
* IS_INTEL_AVX512_VL(f) && msglen < 8*CHUNK -> AVX-512VL
*
* plus the predicate chacha_avx512_beneficial(): IS_INTEL_AVX512(f) &&
* IS_INTEL_VAES(f).
*
* Two inputs decide all of them: the message length and the cpuid mask. One
* host only ever supplies one mask -- on an AVX-512VL machine every
* "IS_INTEL_AVX512_VL(f) == 0" operand is permanently false, which pins three
* of the five decisions to a single outcome no matter what lengths the tests
* use, and on a machine without AVX-512 the last two are pinned the other way.
* wb_chacha_process_case() drives one (mask, length) pair; the table in
* wb_chacha_avx512_dispatch() is the full sweep -- every row in this one
* binary, since llvm-cov derives MC/DC independence pairs per binary.
*
* Crash-safety, same rule as above and enforced by wb_hw(): each row's mask is
* built as (wanted & real cpuid_get_flags()), so a capability bit is only ever
* cleared, never claimed. On a host without AVX-512 the AVX-512 rows degrade
* into the rows below them in the ladder -- coverage is not gained, but no
* unsupported instruction is ever executed.
*/
#include <wolfcrypt/src/chacha.c>
@@ -127,6 +159,143 @@ static void wb_chacha_dispatch(void)
#endif
/* -------------------------------------------------------------------------
* AVX-512 dispatch ladder sweep. See the file header.
* ---------------------------------------------------------------------- */
#if defined(HAVE_CHACHA) && defined(USE_INTEL_CHACHA_SPEEDUP)
/* Capability bits are only ever removed from what the CPU really reports. */
static cpuid_flags_t wb_hw(cpuid_flags_t wanted)
{
return wanted & cpuid_get_flags();
}
/* Longest row: one 16-block zmm chunk plus a tail. */
#define WB_CHACHA_MAX_LEN (20 * CHACHA_CHUNK_BYTES)
static void wb_chacha_process_case(cpuid_flags_t flags, word32 msglen,
const char* what)
{
static const byte key[CHACHA_MAX_KEY_SZ] = {
0x00,0x01,0x02,0x03, 0x04,0x05,0x06,0x07,
0x08,0x09,0x0a,0x0b, 0x0c,0x0d,0x0e,0x0f,
0x10,0x11,0x12,0x13, 0x14,0x15,0x16,0x17,
0x18,0x19,0x1a,0x1b, 0x1c,0x1d,0x1e,0x1f
};
static const byte nonce[CHACHA_IV_BYTES] = {
0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x00, 0x00,0x00,0x00,0x02
};
ChaCha enc;
byte plain[WB_CHACHA_MAX_LEN];
byte cipher[WB_CHACHA_MAX_LEN];
word32 i;
for (i = 0; i < (word32)sizeof(plain); i++) {
plain[i] = (byte)i;
}
/* A fresh key/IV per row: ctx->left from the previous row must not shift
* which branch of the ladder this row's length reaches. */
if (wc_Chacha_SetKey(&enc, key, (word32)sizeof(key)) != 0 ||
wc_Chacha_SetIV(&enc, nonce, 0) != 0) {
printf(" [wb] SetKey/SetIV failed (%s skipped)\n", what);
wb_fail = 1;
return;
}
cpuidFlags = flags;
if (wc_Chacha_Process(&enc, cipher, plain, msglen) != 0) {
printf(" [wb] wc_Chacha_Process failed (%s)\n", what);
wb_fail = 1;
}
}
static void wb_chacha_avx512_dispatch(void)
{
cpuid_flags_t saved_flags = cpuidFlags;
const word32 chunk = CHACHA_CHUNK_BYTES;
const cpuid_flags_t none = 0;
const cpuid_flags_t avx1 = wb_hw(CPUID_AVX1);
const cpuid_flags_t ssse3 = wb_hw(CPUID_SSSE3);
const cpuid_flags_t vl = wb_hw(CPUID_AVX512 | CPUID_AVX512_VL |
CPUID_VAES);
const cpuid_flags_t f512 = wb_hw(CPUID_AVX512);
const cpuid_flags_t zmm = wb_hw(CPUID_AVX512 | CPUID_VAES);
/* One block or less, no AVX-512VL, AVX1 present: the 1-block AVX1 path. */
wb_chacha_process_case(avx1, chunk / 2, "<=1 block, AVX1");
/* Same length, AVX-512VL present: second operand of that test false. */
wb_chacha_process_case(vl, chunk / 2, "<=1 block, AVX-512VL present");
/* Same length, no AVX1 either: third operand false, scalar single block. */
wb_chacha_process_case(none, chunk / 2, "<=1 block, no SIMD");
/* 65..255 bytes, SSSE3 present, no AVX-512VL: the SSSE3 exact-block path. */
wb_chacha_process_case(ssse3, 2 * chunk, "<4 blocks, SSSE3");
/* Same length without SSSE3: falls to the scalar <4-block path. */
wb_chacha_process_case(none, 2 * chunk, "<4 blocks, no SSSE3");
/* Same length with AVX-512VL: the two "VL == 0" first operands are false
* and the AVX-512VL kernel handles it. */
wb_chacha_process_case(vl, 2 * chunk, "<4 blocks, AVX-512VL");
/* >=4 blocks: the length operand of the SSSE3 and scalar tests is false. */
wb_chacha_process_case(ssse3, 8 * chunk, ">=4 blocks, SSSE3");
/* >=4 blocks, nothing available: reaches the zmm and AVX-512VL tests with
* both their feature operands false. */
wb_chacha_process_case(none, 8 * chunk, ">=4 blocks, no SIMD");
/* AVX-512 without VAES: chacha_avx512_beneficial()'s second operand false,
* so the zmm kernel is not entered whatever the length. */
wb_chacha_process_case(f512, 8 * chunk, "AVX-512 without VAES");
/* AVX-512 + VAES and a >=16-block message: the 512-bit zmm kernel. */
wb_chacha_process_case(zmm, 17 * chunk, ">=16 blocks, zmm");
/* AVX-512 + VAES but under 16 blocks: the length operand false. */
wb_chacha_process_case(zmm, 8 * chunk, "<16 blocks, zmm beneficial");
cpuidFlags = saved_flags;
WB_NOTE("chacha AVX-512 dispatch ladder sweep exercised");
}
#ifdef HAVE_INTEL_AVX512
/* chacha_avx512_beneficial() is a pure predicate over a flags word: it runs no
* instruction from the feature it tests, so evaluating it directly over the
* four operand combinations pairs both of its conditions on any host,
* independently of what this machine implements. */
static void wb_chacha_predicate(void)
{
static volatile cpuid_flags_t rows[4];
volatile int sink = 0;
int i;
rows[0] = 0;
rows[1] = CPUID_AVX512;
rows[2] = CPUID_VAES;
rows[3] = CPUID_AVX512 | CPUID_VAES;
for (i = 0; i < 4; i++) {
sink += chacha_avx512_beneficial(rows[i]);
}
(void)sink;
WB_NOTE("chacha_avx512_beneficial operand combinations exercised");
}
#else
static void wb_chacha_predicate(void)
{
WB_NOTE("HAVE_INTEL_AVX512 off; chacha_avx512_beneficial skipped");
}
#endif
#else
static void wb_chacha_avx512_dispatch(void)
{
WB_NOTE("USE_INTEL_CHACHA_SPEEDUP not compiled in this variant; "
"AVX-512 ladder sweep skipped");
}
static void wb_chacha_predicate(void)
{
WB_NOTE("USE_INTEL_CHACHA_SPEEDUP not compiled in this variant; "
"predicate sweep skipped");
}
#endif
int main(void)
{
printf("chacha.c white-box supplement\n");
@@ -135,6 +304,8 @@ int main(void)
return 0;
#else
wb_chacha_dispatch();
wb_chacha_predicate();
wb_chacha_avx512_dispatch();
printf("done (%s)\n", wb_fail ? "with skips" : "ok");
/* Setup failures are surfaced as skips, not test failures: the campaign
* treats a nonzero exit as a failed variant and discards its coverage. */

212
tests/unit-mcdc/test_poly1305_whitebox.c
View File

@@ -48,6 +48,55 @@
* (see db/modules.json poly1305 notes), so forcing intel_flags to "AVX1
* only" and letting the dispatch call the real poly1305_*_avx asm is always
* safe: we never claim a capability the CPU lacks, only hide one it has.
*
* ---------------------------------------------------------------------------
* AVX-512 / IFMA / fused-flag dispatch (wb_poly1305_avx512_dispatch)
* ---------------------------------------------------------------------------
* The AVX-512 Poly1305 work turned the single-condition AVX2 test above into
* six compound decisions -- three in wc_Poly1305Update, three in
* wc_Poly1305Final -- with the same shape in both:
*
* if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) &&
* poly1305_use_ifma(intel_flags)) <- 8-way IFMA, radix 2^44
* if (!POLY1305_FORCE_AVX2(ctx) && !POLY1305_FORCE_SCALAR(ctx) &&
* poly1305_use_avx512(intel_flags)) <- 8-way vpmuludq
* if (!POLY1305_FORCE_SCALAR(ctx) && IS_INTEL_AVX2(intel_flags))
*
* plus the two predicates they call:
*
* poly1305_use_avx512: IS_INTEL_AVX512(f) && IS_INTEL_VAES(f)
* poly1305_use_ifma: IS_INTEL_AVX512_IFMA(f) && IS_CPU_INTEL(f)
*
* Three inputs decide all six: the two ctx flags (set only by the fused
* ChaCha20-Poly1305 kernels in chacha20_poly1305.c, so no poly1305-only caller
* ever sets them) and intel_flags. Any one host reaches exactly one
* combination -- on an AMD host IS_CPU_INTEL is false while AVX-512 and VAES
* are present, so every call takes the 8-way vpmuludq path and the other five
* decisions are never evaluated with a differing operand.
* wb_poly1305_dispatch_case() drives one (intel_flags, forceAvx2, forceScalar)
* triple through SetKey/Update/Final; the table in
* wb_poly1305_avx512_dispatch() is the full combination sweep, every row of it
* in this one binary so that each condition's independence pair is derivable
* here (llvm-cov computes MC/DC per binary).
*
* Crash-safety rule, unchanged from above: a forced mask may only CLEAR
* capability bits the CPU has, never claim one it lacks. Each row's mask is
* built as (wanted & real cpuid_get_flags()), so on a host without AVX-512 the
* AVX-512 rows collapse onto the rows below them and are simply not gained --
* never executed as an unsupported instruction. The one bit forced ON,
* CPUID_INTEL, is a vendor tag rather than a capability: poly1305_use_ifma()'s
* own comment states the gate is a throughput preference between two paths any
* IFMA-capable CPU can run (AMD Zen 4/5 implement IFMA), and the row that lets
* the IFMA kernel actually execute is additionally skipped unless the host
* really reports AVX-512 IFMA.
*
* The ctx-flag rows reproduce, on a poly1305-only context, exactly the state
* the fused kernels set up before calling Update/Final: forceAvx2 -> zero
* ctx->hh, started = 0 (chacha20_poly1305_encrypt_fused); forceScalar -> zero
* ctx->h, finished = 1, leftover = 0, started = 0
* (chacha20_poly1305_encrypt_fused_ifma, wc_ChaCha20Poly1305_UpdateData).
* No MAC value is checked: which kernel computes the right tag is the KAT
* suite's job, this file only drives the dispatch decisions.
*/
#include <wolfcrypt/src/poly1305.c>
@@ -108,6 +157,167 @@ static void wb_poly1305_dispatch(void)
#endif
/* -------------------------------------------------------------------------
* AVX-512 / IFMA / fused-flag dispatch sweep. See the file header.
* ---------------------------------------------------------------------- */
#if defined(HAVE_POLY1305) && defined(USE_INTEL_POLY1305_SPEEDUP)
/* Capability bits are only ever removed from what the CPU really reports, so a
* forced mask can never make the dispatch enter a path the host cannot run. */
static cpuid_flags_t wb_hw(cpuid_flags_t wanted)
{
return wanted & cpuid_get_flags();
}
/* Reproduce the context state the fused ChaCha20-Poly1305 kernels leave behind
* when they pin Update/Final to one path (chacha20_poly1305.c). */
static void wb_poly1305_force(Poly1305* ctx, int forceAvx2, int forceScalar)
{
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED
if (forceAvx2) {
ctx->forceAvx2 = 1;
XMEMSET(ctx->hh, 0, sizeof(ctx->hh));
ctx->started = 0;
ctx->leftover = 0;
}
#else
(void)forceAvx2;
#endif
#ifdef WOLFSSL_CHACHA20_POLY1305_FUSED_IFMA
if (forceScalar) {
ctx->forceScalar = 1;
XMEMSET(ctx->h, 0, sizeof(ctx->h));
ctx->finished = 1;
ctx->started = 0;
ctx->leftover = 0;
}
#else
(void)forceScalar;
#endif
(void)ctx;
}
/* One (intel_flags, forceAvx2, forceScalar) row: key, hash more than the
* 8-block buffer so the block kernels run, and finalise. */
static void wb_poly1305_dispatch_case(cpuid_flags_t flags, int forceAvx2,
int forceScalar, const char* what)
{
static const byte key[32] = {
0x85,0xd6,0xbe,0x78,0x57,0x55,0x6d,0x33,
0x7f,0x44,0x52,0xfe,0x42,0xd5,0x06,0xa8,
0x01,0x03,0x80,0x8a,0xfb,0x0d,0xb2,0xfd,
0x4a,0xbf,0xf6,0xaf,0x41,0x49,0xf5,0x1b
};
Poly1305 ctx;
byte msg[17 * POLY1305_BLOCK_SIZE + 5];
byte tag[WC_POLY1305_MAC_SZ];
size_t i;
for (i = 0; i < sizeof(msg); i++) {
msg[i] = (byte)i;
}
XMEMSET(&ctx, 0, sizeof(ctx));
/* Non-initializer value: cpuid_get_flags_ex() in SetKey keeps it. */
intel_flags = flags;
if (wc_Poly1305SetKey(&ctx, key, (word32)sizeof(key)) != 0) {
printf(" [wb] wc_Poly1305SetKey failed (%s skipped)\n", what);
wb_fail = 1;
return;
}
wb_poly1305_force(&ctx, forceAvx2, forceScalar);
if (wc_Poly1305Update(&ctx, msg, (word32)sizeof(msg)) != 0 ||
wc_Poly1305Final(&ctx, tag) != 0) {
printf(" [wb] Update/Final failed (%s)\n", what);
wb_fail = 1;
}
}
static void wb_poly1305_avx512_dispatch(void)
{
cpuid_flags_t saved_flags = intel_flags;
cpuid_flags_t real = cpuid_get_flags();
/* Common base: the scalar and 4-way kernels this host really has. */
const cpuid_flags_t avx1 = wb_hw(CPUID_AVX1);
const cpuid_flags_t avx2 = wb_hw(CPUID_AVX1 | CPUID_AVX2);
const cpuid_flags_t f512 = wb_hw(CPUID_AVX1 | CPUID_AVX2 | CPUID_AVX512);
const cpuid_flags_t fvaes = wb_hw(CPUID_AVX1 | CPUID_AVX2 | CPUID_AVX512 |
CPUID_VAES | CPUID_AVX512_IFMA);
/* AVX2 absent: IS_INTEL_AVX2 false, every AVX-512 predicate false. */
wb_poly1305_dispatch_case(avx1, 0, 0, "scalar (AVX1 only)");
/* AVX2 present, no AVX-512: the 4-way path. */
wb_poly1305_dispatch_case(avx2, 0, 0, "4-way AVX2");
/* AVX-512 without VAES: poly1305_use_avx512() second operand false. */
wb_poly1305_dispatch_case(f512, 0, 0, "AVX-512 without VAES");
/* AVX-512 + VAES + IFMA, vendor bit absent: use_avx512 true (8-way
* vpmuludq), use_ifma's IS_CPU_INTEL operand false. */
wb_poly1305_dispatch_case(fvaes, 0, 0, "8-way AVX-512, non-Intel vendor");
/* Same, plus the Intel vendor tag: the IFMA path. Only when the host
* really implements AVX-512 IFMA -- the kernel is vpmadd52 asm. */
if (IS_INTEL_AVX512(real) && IS_INTEL_AVX512_IFMA(real)) {
wb_poly1305_dispatch_case(fvaes | CPUID_INTEL, 0, 0,
"8-way AVX-512 IFMA");
}
else {
WB_NOTE("host reports no AVX-512 IFMA; IFMA dispatch row skipped");
}
/* Fused-kernel context flags: each pins the dispatch to one path. */
wb_poly1305_dispatch_case(fvaes, 1, 0, "forceAvx2 (fused 4-way)");
wb_poly1305_dispatch_case(fvaes, 0, 1, "forceScalar (fused IFMA stitch)");
intel_flags = saved_flags;
WB_NOTE("poly1305 AVX-512/IFMA/fused-flag dispatch sweep exercised");
}
#ifdef HAVE_INTEL_AVX512
/* poly1305_use_avx512()/poly1305_use_ifma() are pure predicates over a flags
* word: no state, no instruction from the feature they test. Evaluating them
* directly over the four operand combinations pairs both conditions of each
* without depending on what this host implements, so the two predicates close
* on any machine; the dispatch sweep above then shows the same rows arising
* from real calls wherever the hardware allows. */
static void wb_poly1305_predicates(void)
{
static volatile cpuid_flags_t rows[4];
volatile int sink = 0;
int i;
rows[0] = 0;
rows[1] = CPUID_AVX512 | CPUID_AVX512_IFMA;
rows[2] = CPUID_VAES | CPUID_INTEL;
rows[3] = CPUID_AVX512 | CPUID_AVX512_IFMA | CPUID_VAES | CPUID_INTEL;
for (i = 0; i < 4; i++) {
sink += poly1305_use_avx512(rows[i]);
sink += poly1305_use_ifma(rows[i]);
}
(void)sink;
WB_NOTE("poly1305_use_avx512/use_ifma operand combinations exercised");
}
#else
static void wb_poly1305_predicates(void)
{
WB_NOTE("HAVE_INTEL_AVX512 off; use_avx512/use_ifma predicates skipped");
}
#endif
#else
static void wb_poly1305_avx512_dispatch(void)
{
WB_NOTE("USE_INTEL_POLY1305_SPEEDUP not compiled in this variant; "
"AVX-512 dispatch sweep skipped");
}
static void wb_poly1305_predicates(void)
{
WB_NOTE("USE_INTEL_POLY1305_SPEEDUP not compiled in this variant; "
"predicate sweep skipped");
}
#endif
/* ---- wc_Poly1305SetKey() argument guard (~line 850) ---------------------- *
*
* if ((ctx == NULL) || (key == NULL) || (keySz != 32))
@@ -188,6 +398,8 @@ int main(void)
return 0;
#else
wb_poly1305_dispatch();
wb_poly1305_predicates();
wb_poly1305_avx512_dispatch();
wb_poly1305_setkey_guard();
printf("done (%s)\n", wb_fail ? "with skips" : "ok");
/* Setup failures are surfaced as skips, not test failures: the campaign