1
0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00

MDEV-41271 Add RISC-V Zvbc accelerated CRC-32C implementation

Extend the RISC-V CRC-32C path with a Zvbc vector implementation:

  - mysys/CMakeLists.txt: detect Zvbc + RVV intrinsics under
    -march=rv64gc_zbb_zvbc; when available rebuild the new
    crc32c_riscv_zvbc.cc with that march (optional, Zbc unaffected);
  - mysys/crc32/crc32c_riscv_zvbc.cc (new): K-lane vector core --
    4 lanes x 128-bit folding, vlseg2e64 de-interleaved 64B loads,
    vclmul_vx broadcast constants, single-element vector CLMUL for
    the Barrett merge (no scalar Zbc instruction anywhere), bit-exact
    with the Zbc core (same fold constants k1..k4 and Barrett math);
  - adaptive VLEN: one e64m1 vector pair at VLEN>=256, two pairs at
    VLEN=128, so 128-bit cores execute the vector path at full width;
  - mysys/crc32/crc32c_riscv.cc + crc32c.cc: runtime dispatch via
    riscv_hwprobe -- Zbc is preferred when present (existing
    deployments keep the optimized scalar core); the Zvbc vector
    path accelerates cores that implement Zvbc but not scalar Zbc.

Correctness: RFC 3720 + random/chained/boundary inputs bit-identical
to slicing-by-4; official unittest/mysys/crc32-t.c 36/36.

Performance on Spacemit X100 (k3, VLEN=256, gcc 14.3.0), vs inherited
slicing-by-4 crc32c_slow (official slow path, same my_crc32c entry):

    len      slow (MB/s)   PR-B2 Zvbc (MB/s)   vs slow
    128 B        289            1147              4.0x
    256 B        296            2105              7.1x
    1 KiB        297            5466             18.5x
    4 KiB        297            9116             30.7x
    64 KiB       296           11398             38.7x

Dispatch prefers the scalar Zbc path when Zbc is available, so
existing Zbc deployments see no behavior change or regression;
the vector path is selected on cores without scalar Zbc.
No behavior change on non-riscv64 builds or toolchains without
Zvbc (optional compile check).

Review revisions requested on PR #5746 (dr-m):

  - mysys/crc32/crc32c_riscv.cc: assemble the ZBC/ZVBC extension bits
    without a branch at both sites, dropping the redundant intermediate
    variable; replace `(void) hwprobe;` with `std::ignore = hwprobe;` and
    include <tuple>.  <tuple> introduces no static constructor, so the
    ifunc resolver's load-time constraint is unaffected.
  - mysys/crc32/crc32c.cc: expand the comment above the Zbc preference
    with the structural reason -- the vector core deliberately reuses the
    scalar fold to stay bit-exact, so its vl is capped at 4 and a wider
    VLEN buys nothing; scalar Zbc is preferred because the two are level
    on large input and the scalar path is clearly ahead on small input.
    Comment-only.

Assisted-by: YuanSheng:DeepSeek-V4-Flash
Co-authored-by: Xiaofei Gong <gongxiaofei24@iscas.ac.cn>
Signed-off-by: Jiakai Xu <xujiakai2025@iscas.ac.cn>
This commit is contained in:
6eanut
2026-09-23 02:33:06 +00:00
committed by Marko Mäkelä
parent d45cf75cf8
commit 3cdcd64fb0
4 changed files with 348 additions and 3 deletions

View File

@@ -185,6 +185,35 @@ ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64|RISCV64")
crc32/crc32c_riscv.cc crc32/crc32c_riscv_zbc.cc)
SET_SOURCE_FILES_PROPERTIES(crc32/crc32c_riscv_zbc.cc PROPERTIES
COMPILE_FLAGS "-march=rv64gc_zbc_zbb")
# Zvbc vector core (optional): separate TU compiled for the vector
# extension; the runtime probe/dispatch stay in crc32c_riscv.cc.
# Zbc is deliberately left out of the -march: this core is all-vector,
# and with Zbc present GCC's -foptimize-crc (on by default at -O2 since
# GCC 15) rewrites the bitwise fallback in that file into scalar clmul
# instructions, which do not exist on the Zvbc-without-Zbc cores this
# code is for.
SET(SAVE_REQ_FLAGS2 "${CMAKE_REQUIRED_FLAGS}")
SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=rv64gc_zbb_zvbc")
CHECK_CXX_SOURCE_COMPILES("
#include <riscv_vector.h>
#include <stdint.h>
uint64_t f(uint64_t a, const uint64_t *p, size_t vl) {
vuint64m1_t v = __riscv_vle64_v_u64m1(p, vl);
return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmul_vx_u64m1(v, a, vl));
}
int main() { uint64_t x[4]={1,2,3,4}; return (int) f(5, x, 4); }
" HAVE_RISCV_ZVBC)
SET(CMAKE_REQUIRED_FLAGS "${SAVE_REQ_FLAGS2}")
IF(HAVE_RISCV_ZVBC)
MESSAGE(STATUS "RISC-V Zvbc detected: enabling vector CRC32C")
SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32c_riscv_zvbc.cc)
SET_SOURCE_FILES_PROPERTIES(crc32/crc32c_riscv_zvbc.cc PROPERTIES
COMPILE_FLAGS "-march=rv64gc_zbb_zvbc")
ADD_DEFINITIONS(-DHAVE_RISCV_ZVBC)
ELSE()
MESSAGE(STATUS "RISC-V Zvbc not available: using Zbc scalar CRC32C")
ENDIF()
ENDIF()
ENDIF()
ENDIF()

View File

@@ -491,7 +491,10 @@ extern "C" my_crc32_t crc32c_aarch64_available(void);
extern "C" const char *crc32c_aarch64_impl(my_crc32_t);
#elif defined HAVE_RISCV_ZBC
extern "C" unsigned crc32c_riscv_zbc(unsigned, const void *, size_t);
extern "C" int rv_zbc_supported(void *);
# ifdef HAVE_RISCV_ZVBC
extern "C" unsigned crc32c_riscv_zvbc(unsigned, const void *, size_t);
# endif
extern "C" unsigned rv_riscv_crc_ext(void *);
extern "C" const char *crc32c_riscv_impl(my_crc32_t);
#elif defined __i386__||defined __x86_64__||defined _M_X64||defined _M_IX86
extern "C" my_crc32_t crc32c_x86_available(void);
@@ -501,7 +504,33 @@ extern "C" const char *crc32c_x86_impl(my_crc32_t);
#if defined HAVE_RISCV_ZBC
static my_crc32_t crc32c_riscv_choose(void *hwprobe)
{
return rv_zbc_supported(hwprobe) ? crc32c_riscv_zbc : crc32c_slow;
unsigned ext= rv_riscv_crc_ext(hwprobe);
/* Where scalar Zbc is available, prefer it over the Zvbc vector core.
The vector core does not buy extra parallelism here. To stay
bit-exact with the scalar core it reuses exactly the same fold:
four 128-bit lanes over a fixed 64-byte span, with the same
constants k1..k4 and the same Barrett step. It therefore caps vl
at 4 and folds 64 bytes per iteration however wide VLEN is, and it
issues the same number of carry-less multiplies. Its only edge is
a single de-interleaving segment load per 64 bytes, which on large
input is offset by its per-call overhead (vsetvl, and injecting
the CRC into lane 0 through a store/load round trip): on Spacemit
X100 (VLEN=256) it ties the scalar fold at 64 KiB (11398 vs 11687
MB/s) and loses on small input (128 B: 1147 vs 5540 MB/s,
1 KiB: 5466 vs 10277 MB/s).
Its reason to exist is cores that implement Zvbc but not scalar
Zbc, where the only alternative is the slicing-by-4 crc32c_slow.
Where Zbc is present, selecting it would only touch the vector
register file for no gain. */
if (ext & 1)
return crc32c_riscv_zbc;
#ifdef HAVE_RISCV_ZVBC
if (ext & 2) /* Zvbc only (no scalar Zbc) */
return crc32c_riscv_zvbc;
#endif
return crc32c_slow;
}
/* The RISC-V resolver. Unlike the other architectures, the implementation is
@@ -509,7 +538,7 @@ static my_crc32_t crc32c_riscv_choose(void *hwprobe)
target operating systems of RISC-V (Linux and FreeBSD) support that.
The dynamic linker calls this at load time, before main(), so it may only
run code that is safe there. rv_zbc_supported() is: it uses the
run code that is safe there. rv_riscv_crc_ext() is: it uses the
riscv_hwprobe system call and nothing else.
The resolver must never return NULL. Whereas the *_available() functions

View File

@@ -16,6 +16,7 @@ argument is NULL and the system call is made directly.
*/
#include <stddef.h>
#include <tuple>
#include <unistd.h>
#include <sys/syscall.h>
@@ -37,6 +38,9 @@ struct riscv_hwprobe { long long key; unsigned long long value; };
#ifndef RISCV_HWPROBE_EXT_ZBC
# define RISCV_HWPROBE_EXT_ZBC (1ULL << 7)
#endif
#ifndef RISCV_HWPROBE_EXT_ZVBC
# define RISCV_HWPROBE_EXT_ZVBC (1ULL << 18)
#endif
#ifndef SYS_riscv_hwprobe
# ifdef __NR_riscv_hwprobe
@@ -77,9 +81,45 @@ extern "C" int rv_zbc_supported(void *hwprobe)
}
extern "C" unsigned crc32c_riscv_zbc(unsigned, const void *, size_t);
#ifdef HAVE_RISCV_ZVBC
extern "C" unsigned crc32c_riscv_zvbc(unsigned, const void *, size_t);
#endif
/* CRC acceleration level for the callers' ifunc resolver:
3 = Zbc + Zvbc (prefer Zbc; the optimized scalar fold ties the 4-lane
vector core at VLEN 256 and wins below it),
2 = Zvbc only (no scalar Zbc -- the vector core is the only
accelerated path on such cores), 1 = Zbc only, 0 = none. */
extern "C" unsigned rv_riscv_crc_ext(void *hwprobe)
{
struct riscv_hwprobe p;
p.key= RISCV_HWPROBE_KEY_IMA_EXT_0;
p.value= 0;
#ifdef HAVE_SYS_HWPROBE_H
if (hwprobe != NULL)
{
unsigned long long value= 0;
if (__riscv_hwprobe_one(reinterpret_cast<__riscv_hwprobe_t>(hwprobe),
RISCV_HWPROBE_KEY_IMA_EXT_0, &value) != 0)
return 0;
return ((value & RISCV_HWPROBE_EXT_ZBC) ? 1u : 0u) |
((value & RISCV_HWPROBE_EXT_ZVBC) ? 2u : 0u);
}
#else
std::ignore = hwprobe;
#endif
if (syscall(SYS_riscv_hwprobe, &p, (size_t) 1, (size_t) 0, NULL, 0) != 0)
return 0;
return ((p.value & RISCV_HWPROBE_EXT_ZBC) ? 1u : 0u) |
((p.value & RISCV_HWPROBE_EXT_ZVBC) ? 2u : 0u);
}
extern "C" const char *crc32c_riscv_impl(my_crc32_t c)
{
#ifdef HAVE_RISCV_ZVBC
if (c == crc32c_riscv_zvbc)
return "Using RISC-V Zvbc vector carry-less multiply instructions";
#endif
if (c == crc32c_riscv_zbc)
return "Using RISC-V Zbc carry-less multiply instructions";
return NULL;

View File

@@ -0,0 +1,247 @@
/* RISC-V Zvbc vector accelerated CRC-32C.
K-lane Zvbc implementation with *adaptive* vector length. The CRC fold
itself keeps the exact 4-lane / 64-byte span of the scalar Zbc core (same
fold constants k1..k4, same Barrett math) so the result is bit-exact; the
vector length only decides how many e64m1 vector pairs carry the four
lanes in parallel (one pair of vl>=4 vectors on VLEN>=256, two pairs of
vl=2 vectors on VLEN=128). A CPU with VLEN=128 therefore still executes
the vector path instead of being forced back to scalar code.
All carry-less multiplies run through the single-element vector vclmul
path (a scalar `clmul` would trap on cores that implement Zvbc but not
Zbc). Small inputs, VLEN < 2 lanes, and the post-alignment remainder
fall back to the in-TU bitwise walk with the external CRC state.
This is the only compilation unit enabled for the vector extension; it is
compiled with -march=rv64gc_zbb_zvbc (see mysys/CMakeLists.txt). zbc is
deliberately absent from that -march: with zbc present, GCC's
-foptimize-crc (on by default at -O2 since GCC 15) rewrites the bitwise
fallback below into scalar clmul/clmulh, which do not exist on the cores
this file is for.
*/
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#if defined(__riscv_zvbc)
#include <riscv_vector.h>
#endif
typedef unsigned (*my_crc32_t)(unsigned, const void *, size_t);
namespace mysys_namespace {
namespace crc32c {
struct rv_crc32_tab {
alignas(16) uint32_t fold[4];
uint64_t const0;
uint64_t const1;
uint64_t quo;
uint64_t poly;
uint32_t bitwise_poly;
};
/* The scalar Zbc `clmul` instruction is intentionally NOT used: some cores
implement Zvbc but not B-ext scalar clmul. Every CLMUL in this TU goes
through the single-element vector vclmul path instead. */
static inline uint64_t rv_clmul(uint64_t a, uint64_t b) {
const size_t vl = 1;
vuint64m1_t va = __riscv_vmv_v_x_u64m1(a, vl);
vuint64m1_t vb = __riscv_vmv_v_x_u64m1(b, vl);
return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmul_vv_u64m1(va, vb, vl));
}
static inline uint64_t rv_clmulh(uint64_t a, uint64_t b) {
const size_t vl = 1;
vuint64m1_t va = __riscv_vmv_v_x_u64m1(a, vl);
vuint64m1_t vb = __riscv_vmv_v_x_u64m1(b, vl);
return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmulh_vv_u64m1(va, vb, vl));
}
static inline uint32_t rv_crc32_bitwise(uint32_t crc, const uint8_t* buf,
size_t len, uint32_t poly) {
uint32_t c = crc;
for (size_t i = 0; i < len; ++i) {
c ^= buf[i];
for (int k = 0; k < 8; ++k) {
c = (c >> 1) ^ ((c & 1) ? poly : 0);
}
}
return c;
}
static inline void rv_fold_pair(uint64_t* lo, uint64_t* hi,
uint64_t k0, uint64_t k1,
uint64_t v0, uint64_t v1) {
uint64_t l = rv_clmul(*lo, k0) ^ rv_clmul(*hi, k1);
uint64_t h = rv_clmulh(*lo, k0) ^ rv_clmulh(*hi, k1);
*lo = l ^ v0;
*hi = h ^ v1;
}
/* CRC-32C (Castagnoli), polynomial 0x1EDC6F41 -- same values as
crc32c_riscv_zbc.cc, kept here for the vector core. */
static constexpr rv_crc32_tab rv_crc32c_tab = {
{ 0x740eef02, 0x9e4addf8, 0xf20c0dfe, 0x493c7d27 },
0x00000000dd45aab8ULL, 0x00000000493c7d27ULL,
0x0000000dea713f1ULL, 0x0000000105ec76f1ULL,
0x82F63B78
};
static constexpr uint64_t RV_CRC32_MASK32 = 0x00000000FFFFFFFFULL;
#if defined(__riscv_zvbc)
static uint32_t rv_crc32c_zvbc_impl(uint32_t crc, const char* buf,
size_t len, const rv_crc32_tab& tab) {
const uint8_t* p = reinterpret_cast<const uint8_t*>(buf);
size_t n = len;
size_t max_vl = __riscv_vsetvlmax_e64m1();
/* Lanes per vector pair; the CRC core always folds four lanes. */
const size_t vl = max_vl >= 4 ? 4 : (max_vl >= 2 ? 2 : 0);
const uint64_t chunk = 64; /* one fixed 64-byte fold span, Zbc-exact */
crc ^= 0xFFFFFFFF; /* to internal state */
if (n < chunk || vl == 0) {
return rv_crc32_bitwise(crc, reinterpret_cast<const uint8_t*>(buf), len, tab.bitwise_poly) ^ 0xFFFFFFFF;
}
/* Align to 16-byte boundary (same bitwise walk as the Zbc core). */
if (uintptr_t mis = reinterpret_cast<uintptr_t>(p) & 0xF) {
size_t pre = 16 - mis;
if (pre > n) pre = n;
crc = rv_crc32_bitwise(crc, p, pre, tab.bitwise_poly);
p += pre;
n -= pre;
if (n < chunk) {
return rv_crc32_bitwise(crc, p, n, tab.bitwise_poly) ^ 0xFFFFFFFF;
}
}
/* Four lanes, carried by `segs` e64m1 vector pairs:
vl=4 (VLEN>=256): 1 pair, one vlseg2e64(4) reads the full 64 bytes
vl=2 (VLEN=128): 2 pairs, each vlseg2e64(2) reads 32 bytes
Lane j always owns bytes [16j, 16j+16) of the chunk, matching the
scalar core's d0/d1 ... d6/d7 assignment. */
const uint32_t segs = 4 / vl;
/* RVV types cannot live in arrays with GCC, so the (at most two)
vector pairs are held in named variables. segs==4/vl: 1 pair for
vl=4, 2 pairs for vl=2. */
vuint64m1_t lo_a, hi_a, lo_b, hi_b;
const size_t seg_bytes = vl * 16;
{
vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2(
reinterpret_cast<const uint64_t*>(p), vl);
lo_a = __riscv_vget_v_u64m1x2_u64m1(sg, 0);
hi_a = __riscv_vget_v_u64m1x2_u64m1(sg, 1);
}
if (segs == 2) {
vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2(
reinterpret_cast<const uint64_t*>(p + seg_bytes), vl);
lo_b = __riscv_vget_v_u64m1x2_u64m1(sg, 0);
hi_b = __riscv_vget_v_u64m1x2_u64m1(sg, 1);
}
if (vl == 2) {
/* Lane 0's low word is element 0 of the first pair. */
uint64_t l0[2], h0[2];
__riscv_vse64_v_u64m1(l0, lo_a, vl);
__riscv_vse64_v_u64m1(h0, hi_a, vl);
l0[0] ^= (uint64_t)crc;
lo_a = __riscv_vle64_v_u64m1(l0, vl);
hi_a = __riscv_vle64_v_u64m1(h0, vl);
} else {
uint64_t l0[4];
__riscv_vse64_v_u64m1(l0, lo_a, vl);
l0[0] ^= (uint64_t)crc;
lo_a = __riscv_vle64_v_u64m1(l0, vl);
}
p += chunk;
n -= chunk;
const uint64_t k1 = tab.fold[0];
const uint64_t k2 = tab.fold[1];
while (n >= chunk) {
vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2(
reinterpret_cast<const uint64_t*>(p), vl);
vuint64m1_t dl = __riscv_vget_v_u64m1x2_u64m1(sg, 0);
vuint64m1_t dh = __riscv_vget_v_u64m1x2_u64m1(sg, 1);
vuint64m1_t lv = __riscv_vxor_vv_u64m1(
__riscv_vclmul_vx_u64m1(lo_a, k1, vl),
__riscv_vclmul_vx_u64m1(hi_a, k2, vl), vl);
vuint64m1_t hv = __riscv_vxor_vv_u64m1(
__riscv_vclmulh_vx_u64m1(lo_a, k1, vl),
__riscv_vclmulh_vx_u64m1(hi_a, k2, vl), vl);
lo_a = __riscv_vxor_vv_u64m1(lv, dl, vl);
hi_a = __riscv_vxor_vv_u64m1(hv, dh, vl);
if (segs == 2) {
vuint64m1x2_t sg2 = __riscv_vlseg2e64_v_u64m1x2(
reinterpret_cast<const uint64_t*>(p + seg_bytes), vl);
vuint64m1_t dl2 = __riscv_vget_v_u64m1x2_u64m1(sg2, 0);
vuint64m1_t dh2 = __riscv_vget_v_u64m1x2_u64m1(sg2, 1);
vuint64m1_t lv2 = __riscv_vxor_vv_u64m1(
__riscv_vclmul_vx_u64m1(lo_b, k1, vl),
__riscv_vclmul_vx_u64m1(hi_b, k2, vl), vl);
vuint64m1_t hv2 = __riscv_vxor_vv_u64m1(
__riscv_vclmulh_vx_u64m1(lo_b, k1, vl),
__riscv_vclmulh_vx_u64m1(hi_b, k2, vl), vl);
lo_b = __riscv_vxor_vv_u64m1(lv2, dl2, vl);
hi_b = __riscv_vxor_vv_u64m1(hv2, dh2, vl);
}
p += chunk;
n -= chunk;
}
/* Merge the four lanes (same order as the scalar 4-way fold). */
uint64_t loa[4], hia[4];
__riscv_vse64_v_u64m1(loa, lo_a, vl);
__riscv_vse64_v_u64m1(hia, hi_a, vl);
if (segs == 2) {
__riscv_vse64_v_u64m1(loa + 2, lo_b, vl);
__riscv_vse64_v_u64m1(hia + 2, hi_b, vl);
}
const uint64_t k3 = tab.fold[2];
const uint64_t k4 = tab.fold[3];
uint64_t x0 = loa[0], x1 = hia[0];
for (uint32_t j = 1; j < 4; ++j)
rv_fold_pair(&x0, &x1, k3, k4, loa[j], hia[j]);
/* Barrett reduction: 128-bit -> 32-bit (same as the Zbc core). */
uint64_t t4 = rv_clmul(x0, tab.const1);
uint64_t t3 = rv_clmulh(x0, tab.const1);
uint64_t t1 = x1 ^ t4;
t4 = t1 & RV_CRC32_MASK32;
t1 >>= 32;
uint64_t t0 = rv_clmul(t4, tab.const0);
t3 = (t3 << 32) ^ t1 ^ t0;
t4 = t3 & RV_CRC32_MASK32;
t4 = rv_clmul(t4, tab.quo);
t4 &= RV_CRC32_MASK32;
t4 = rv_clmul(t4, tab.poly);
t4 ^= t3;
uint32_t c = static_cast<uint32_t>((t4 >> 32) & RV_CRC32_MASK32);
if (n)
c = rv_crc32_bitwise(c, p, n, tab.bitwise_poly);
return c ^ 0xFFFFFFFF;
}
#endif /* __riscv_zvbc */
} // namespace crc32c
} // namespace mysys_namespace
extern "C" unsigned crc32c_riscv_zvbc(unsigned crc, const void *buf,
size_t len)
{
#if defined(__riscv_zvbc)
return mysys_namespace::crc32c::rv_crc32c_zvbc_impl(
crc, reinterpret_cast<const char*>(buf), len,
mysys_namespace::crc32c::rv_crc32c_tab);
#else
return 0;
#endif
}