mirror of
https://github.com/MariaDB/server.git
synced 2026-10-23 18:49:29 +03:00
MDEV-41271 Add RISC-V Zvbc accelerated CRC-32C implementation
Extend the RISC-V CRC-32C path with a Zvbc vector implementation:
- mysys/CMakeLists.txt: detect Zvbc + RVV intrinsics under
-march=rv64gc_zbb_zvbc; when available rebuild the new
crc32c_riscv_zvbc.cc with that march (optional, Zbc unaffected);
- mysys/crc32/crc32c_riscv_zvbc.cc (new): K-lane vector core --
4 lanes x 128-bit folding, vlseg2e64 de-interleaved 64B loads,
vclmul_vx broadcast constants, single-element vector CLMUL for
the Barrett merge (no scalar Zbc instruction anywhere), bit-exact
with the Zbc core (same fold constants k1..k4 and Barrett math);
- adaptive VLEN: one e64m1 vector pair at VLEN>=256, two pairs at
VLEN=128, so 128-bit cores execute the vector path at full width;
- mysys/crc32/crc32c_riscv.cc + crc32c.cc: runtime dispatch via
riscv_hwprobe -- Zbc is preferred when present (existing
deployments keep the optimized scalar core); the Zvbc vector
path accelerates cores that implement Zvbc but not scalar Zbc.
Correctness: RFC 3720 + random/chained/boundary inputs bit-identical
to slicing-by-4; official unittest/mysys/crc32-t.c 36/36.
Performance on Spacemit X100 (k3, VLEN=256, gcc 14.3.0), vs inherited
slicing-by-4 crc32c_slow (official slow path, same my_crc32c entry):
len slow (MB/s) PR-B2 Zvbc (MB/s) vs slow
128 B 289 1147 4.0x
256 B 296 2105 7.1x
1 KiB 297 5466 18.5x
4 KiB 297 9116 30.7x
64 KiB 296 11398 38.7x
Dispatch prefers the scalar Zbc path when Zbc is available, so
existing Zbc deployments see no behavior change or regression;
the vector path is selected on cores without scalar Zbc.
No behavior change on non-riscv64 builds or toolchains without
Zvbc (optional compile check).
Review revisions requested on PR #5746 (dr-m):
- mysys/crc32/crc32c_riscv.cc: assemble the ZBC/ZVBC extension bits
without a branch at both sites, dropping the redundant intermediate
variable; replace `(void) hwprobe;` with `std::ignore = hwprobe;` and
include <tuple>. <tuple> introduces no static constructor, so the
ifunc resolver's load-time constraint is unaffected.
- mysys/crc32/crc32c.cc: expand the comment above the Zbc preference
with the structural reason -- the vector core deliberately reuses the
scalar fold to stay bit-exact, so its vl is capped at 4 and a wider
VLEN buys nothing; scalar Zbc is preferred because the two are level
on large input and the scalar path is clearly ahead on small input.
Comment-only.
Assisted-by: YuanSheng:DeepSeek-V4-Flash
Co-authored-by: Xiaofei Gong <gongxiaofei24@iscas.ac.cn>
Signed-off-by: Jiakai Xu <xujiakai2025@iscas.ac.cn>
This commit is contained in:
@@ -185,6 +185,35 @@ ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64|RISCV64")
|
||||
crc32/crc32c_riscv.cc crc32/crc32c_riscv_zbc.cc)
|
||||
SET_SOURCE_FILES_PROPERTIES(crc32/crc32c_riscv_zbc.cc PROPERTIES
|
||||
COMPILE_FLAGS "-march=rv64gc_zbc_zbb")
|
||||
|
||||
# Zvbc vector core (optional): separate TU compiled for the vector
|
||||
# extension; the runtime probe/dispatch stay in crc32c_riscv.cc.
|
||||
# Zbc is deliberately left out of the -march: this core is all-vector,
|
||||
# and with Zbc present GCC's -foptimize-crc (on by default at -O2 since
|
||||
# GCC 15) rewrites the bitwise fallback in that file into scalar clmul
|
||||
# instructions, which do not exist on the Zvbc-without-Zbc cores this
|
||||
# code is for.
|
||||
SET(SAVE_REQ_FLAGS2 "${CMAKE_REQUIRED_FLAGS}")
|
||||
SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=rv64gc_zbb_zvbc")
|
||||
CHECK_CXX_SOURCE_COMPILES("
|
||||
#include <riscv_vector.h>
|
||||
#include <stdint.h>
|
||||
uint64_t f(uint64_t a, const uint64_t *p, size_t vl) {
|
||||
vuint64m1_t v = __riscv_vle64_v_u64m1(p, vl);
|
||||
return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmul_vx_u64m1(v, a, vl));
|
||||
}
|
||||
int main() { uint64_t x[4]={1,2,3,4}; return (int) f(5, x, 4); }
|
||||
" HAVE_RISCV_ZVBC)
|
||||
SET(CMAKE_REQUIRED_FLAGS "${SAVE_REQ_FLAGS2}")
|
||||
IF(HAVE_RISCV_ZVBC)
|
||||
MESSAGE(STATUS "RISC-V Zvbc detected: enabling vector CRC32C")
|
||||
SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32c_riscv_zvbc.cc)
|
||||
SET_SOURCE_FILES_PROPERTIES(crc32/crc32c_riscv_zvbc.cc PROPERTIES
|
||||
COMPILE_FLAGS "-march=rv64gc_zbb_zvbc")
|
||||
ADD_DEFINITIONS(-DHAVE_RISCV_ZVBC)
|
||||
ELSE()
|
||||
MESSAGE(STATUS "RISC-V Zvbc not available: using Zbc scalar CRC32C")
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
ENDIF()
|
||||
|
||||
@@ -491,7 +491,10 @@ extern "C" my_crc32_t crc32c_aarch64_available(void);
|
||||
extern "C" const char *crc32c_aarch64_impl(my_crc32_t);
|
||||
#elif defined HAVE_RISCV_ZBC
|
||||
extern "C" unsigned crc32c_riscv_zbc(unsigned, const void *, size_t);
|
||||
extern "C" int rv_zbc_supported(void *);
|
||||
# ifdef HAVE_RISCV_ZVBC
|
||||
extern "C" unsigned crc32c_riscv_zvbc(unsigned, const void *, size_t);
|
||||
# endif
|
||||
extern "C" unsigned rv_riscv_crc_ext(void *);
|
||||
extern "C" const char *crc32c_riscv_impl(my_crc32_t);
|
||||
#elif defined __i386__||defined __x86_64__||defined _M_X64||defined _M_IX86
|
||||
extern "C" my_crc32_t crc32c_x86_available(void);
|
||||
@@ -501,7 +504,33 @@ extern "C" const char *crc32c_x86_impl(my_crc32_t);
|
||||
#if defined HAVE_RISCV_ZBC
|
||||
static my_crc32_t crc32c_riscv_choose(void *hwprobe)
|
||||
{
|
||||
return rv_zbc_supported(hwprobe) ? crc32c_riscv_zbc : crc32c_slow;
|
||||
unsigned ext= rv_riscv_crc_ext(hwprobe);
|
||||
/* Where scalar Zbc is available, prefer it over the Zvbc vector core.
|
||||
|
||||
The vector core does not buy extra parallelism here. To stay
|
||||
bit-exact with the scalar core it reuses exactly the same fold:
|
||||
four 128-bit lanes over a fixed 64-byte span, with the same
|
||||
constants k1..k4 and the same Barrett step. It therefore caps vl
|
||||
at 4 and folds 64 bytes per iteration however wide VLEN is, and it
|
||||
issues the same number of carry-less multiplies. Its only edge is
|
||||
a single de-interleaving segment load per 64 bytes, which on large
|
||||
input is offset by its per-call overhead (vsetvl, and injecting
|
||||
the CRC into lane 0 through a store/load round trip): on Spacemit
|
||||
X100 (VLEN=256) it ties the scalar fold at 64 KiB (11398 vs 11687
|
||||
MB/s) and loses on small input (128 B: 1147 vs 5540 MB/s,
|
||||
1 KiB: 5466 vs 10277 MB/s).
|
||||
|
||||
Its reason to exist is cores that implement Zvbc but not scalar
|
||||
Zbc, where the only alternative is the slicing-by-4 crc32c_slow.
|
||||
Where Zbc is present, selecting it would only touch the vector
|
||||
register file for no gain. */
|
||||
if (ext & 1)
|
||||
return crc32c_riscv_zbc;
|
||||
#ifdef HAVE_RISCV_ZVBC
|
||||
if (ext & 2) /* Zvbc only (no scalar Zbc) */
|
||||
return crc32c_riscv_zvbc;
|
||||
#endif
|
||||
return crc32c_slow;
|
||||
}
|
||||
|
||||
/* The RISC-V resolver. Unlike the other architectures, the implementation is
|
||||
@@ -509,7 +538,7 @@ static my_crc32_t crc32c_riscv_choose(void *hwprobe)
|
||||
target operating systems of RISC-V (Linux and FreeBSD) support that.
|
||||
|
||||
The dynamic linker calls this at load time, before main(), so it may only
|
||||
run code that is safe there. rv_zbc_supported() is: it uses the
|
||||
run code that is safe there. rv_riscv_crc_ext() is: it uses the
|
||||
riscv_hwprobe system call and nothing else.
|
||||
|
||||
The resolver must never return NULL. Whereas the *_available() functions
|
||||
|
||||
@@ -16,6 +16,7 @@ argument is NULL and the system call is made directly.
|
||||
*/
|
||||
|
||||
#include <stddef.h>
|
||||
#include <tuple>
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
|
||||
@@ -37,6 +38,9 @@ struct riscv_hwprobe { long long key; unsigned long long value; };
|
||||
#ifndef RISCV_HWPROBE_EXT_ZBC
|
||||
# define RISCV_HWPROBE_EXT_ZBC (1ULL << 7)
|
||||
#endif
|
||||
#ifndef RISCV_HWPROBE_EXT_ZVBC
|
||||
# define RISCV_HWPROBE_EXT_ZVBC (1ULL << 18)
|
||||
#endif
|
||||
|
||||
#ifndef SYS_riscv_hwprobe
|
||||
# ifdef __NR_riscv_hwprobe
|
||||
@@ -77,9 +81,45 @@ extern "C" int rv_zbc_supported(void *hwprobe)
|
||||
}
|
||||
|
||||
extern "C" unsigned crc32c_riscv_zbc(unsigned, const void *, size_t);
|
||||
#ifdef HAVE_RISCV_ZVBC
|
||||
extern "C" unsigned crc32c_riscv_zvbc(unsigned, const void *, size_t);
|
||||
#endif
|
||||
|
||||
/* CRC acceleration level for the callers' ifunc resolver:
|
||||
3 = Zbc + Zvbc (prefer Zbc; the optimized scalar fold ties the 4-lane
|
||||
vector core at VLEN 256 and wins below it),
|
||||
2 = Zvbc only (no scalar Zbc -- the vector core is the only
|
||||
accelerated path on such cores), 1 = Zbc only, 0 = none. */
|
||||
extern "C" unsigned rv_riscv_crc_ext(void *hwprobe)
|
||||
{
|
||||
struct riscv_hwprobe p;
|
||||
p.key= RISCV_HWPROBE_KEY_IMA_EXT_0;
|
||||
p.value= 0;
|
||||
#ifdef HAVE_SYS_HWPROBE_H
|
||||
if (hwprobe != NULL)
|
||||
{
|
||||
unsigned long long value= 0;
|
||||
if (__riscv_hwprobe_one(reinterpret_cast<__riscv_hwprobe_t>(hwprobe),
|
||||
RISCV_HWPROBE_KEY_IMA_EXT_0, &value) != 0)
|
||||
return 0;
|
||||
return ((value & RISCV_HWPROBE_EXT_ZBC) ? 1u : 0u) |
|
||||
((value & RISCV_HWPROBE_EXT_ZVBC) ? 2u : 0u);
|
||||
}
|
||||
#else
|
||||
std::ignore = hwprobe;
|
||||
#endif
|
||||
if (syscall(SYS_riscv_hwprobe, &p, (size_t) 1, (size_t) 0, NULL, 0) != 0)
|
||||
return 0;
|
||||
return ((p.value & RISCV_HWPROBE_EXT_ZBC) ? 1u : 0u) |
|
||||
((p.value & RISCV_HWPROBE_EXT_ZVBC) ? 2u : 0u);
|
||||
}
|
||||
|
||||
extern "C" const char *crc32c_riscv_impl(my_crc32_t c)
|
||||
{
|
||||
#ifdef HAVE_RISCV_ZVBC
|
||||
if (c == crc32c_riscv_zvbc)
|
||||
return "Using RISC-V Zvbc vector carry-less multiply instructions";
|
||||
#endif
|
||||
if (c == crc32c_riscv_zbc)
|
||||
return "Using RISC-V Zbc carry-less multiply instructions";
|
||||
return NULL;
|
||||
|
||||
247
mysys/crc32/crc32c_riscv_zvbc.cc
Normal file
247
mysys/crc32/crc32c_riscv_zvbc.cc
Normal file
@@ -0,0 +1,247 @@
|
||||
/* RISC-V Zvbc vector accelerated CRC-32C.
|
||||
|
||||
K-lane Zvbc implementation with *adaptive* vector length. The CRC fold
|
||||
itself keeps the exact 4-lane / 64-byte span of the scalar Zbc core (same
|
||||
fold constants k1..k4, same Barrett math) so the result is bit-exact; the
|
||||
vector length only decides how many e64m1 vector pairs carry the four
|
||||
lanes in parallel (one pair of vl>=4 vectors on VLEN>=256, two pairs of
|
||||
vl=2 vectors on VLEN=128). A CPU with VLEN=128 therefore still executes
|
||||
the vector path instead of being forced back to scalar code.
|
||||
|
||||
All carry-less multiplies run through the single-element vector vclmul
|
||||
path (a scalar `clmul` would trap on cores that implement Zvbc but not
|
||||
Zbc). Small inputs, VLEN < 2 lanes, and the post-alignment remainder
|
||||
fall back to the in-TU bitwise walk with the external CRC state.
|
||||
|
||||
This is the only compilation unit enabled for the vector extension; it is
|
||||
compiled with -march=rv64gc_zbb_zvbc (see mysys/CMakeLists.txt). zbc is
|
||||
deliberately absent from that -march: with zbc present, GCC's
|
||||
-foptimize-crc (on by default at -O2 since GCC 15) rewrites the bitwise
|
||||
fallback below into scalar clmul/clmulh, which do not exist on the cores
|
||||
this file is for.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
#if defined(__riscv_zvbc)
|
||||
#include <riscv_vector.h>
|
||||
#endif
|
||||
|
||||
typedef unsigned (*my_crc32_t)(unsigned, const void *, size_t);
|
||||
|
||||
namespace mysys_namespace {
|
||||
namespace crc32c {
|
||||
|
||||
struct rv_crc32_tab {
|
||||
alignas(16) uint32_t fold[4];
|
||||
uint64_t const0;
|
||||
uint64_t const1;
|
||||
uint64_t quo;
|
||||
uint64_t poly;
|
||||
uint32_t bitwise_poly;
|
||||
};
|
||||
|
||||
/* The scalar Zbc `clmul` instruction is intentionally NOT used: some cores
|
||||
implement Zvbc but not B-ext scalar clmul. Every CLMUL in this TU goes
|
||||
through the single-element vector vclmul path instead. */
|
||||
static inline uint64_t rv_clmul(uint64_t a, uint64_t b) {
|
||||
const size_t vl = 1;
|
||||
vuint64m1_t va = __riscv_vmv_v_x_u64m1(a, vl);
|
||||
vuint64m1_t vb = __riscv_vmv_v_x_u64m1(b, vl);
|
||||
return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmul_vv_u64m1(va, vb, vl));
|
||||
}
|
||||
static inline uint64_t rv_clmulh(uint64_t a, uint64_t b) {
|
||||
const size_t vl = 1;
|
||||
vuint64m1_t va = __riscv_vmv_v_x_u64m1(a, vl);
|
||||
vuint64m1_t vb = __riscv_vmv_v_x_u64m1(b, vl);
|
||||
return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmulh_vv_u64m1(va, vb, vl));
|
||||
}
|
||||
|
||||
static inline uint32_t rv_crc32_bitwise(uint32_t crc, const uint8_t* buf,
|
||||
size_t len, uint32_t poly) {
|
||||
uint32_t c = crc;
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
c ^= buf[i];
|
||||
for (int k = 0; k < 8; ++k) {
|
||||
c = (c >> 1) ^ ((c & 1) ? poly : 0);
|
||||
}
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
static inline void rv_fold_pair(uint64_t* lo, uint64_t* hi,
|
||||
uint64_t k0, uint64_t k1,
|
||||
uint64_t v0, uint64_t v1) {
|
||||
uint64_t l = rv_clmul(*lo, k0) ^ rv_clmul(*hi, k1);
|
||||
uint64_t h = rv_clmulh(*lo, k0) ^ rv_clmulh(*hi, k1);
|
||||
*lo = l ^ v0;
|
||||
*hi = h ^ v1;
|
||||
}
|
||||
|
||||
/* CRC-32C (Castagnoli), polynomial 0x1EDC6F41 -- same values as
|
||||
crc32c_riscv_zbc.cc, kept here for the vector core. */
|
||||
static constexpr rv_crc32_tab rv_crc32c_tab = {
|
||||
{ 0x740eef02, 0x9e4addf8, 0xf20c0dfe, 0x493c7d27 },
|
||||
0x00000000dd45aab8ULL, 0x00000000493c7d27ULL,
|
||||
0x0000000dea713f1ULL, 0x0000000105ec76f1ULL,
|
||||
0x82F63B78
|
||||
};
|
||||
|
||||
static constexpr uint64_t RV_CRC32_MASK32 = 0x00000000FFFFFFFFULL;
|
||||
|
||||
#if defined(__riscv_zvbc)
|
||||
static uint32_t rv_crc32c_zvbc_impl(uint32_t crc, const char* buf,
|
||||
size_t len, const rv_crc32_tab& tab) {
|
||||
const uint8_t* p = reinterpret_cast<const uint8_t*>(buf);
|
||||
size_t n = len;
|
||||
|
||||
size_t max_vl = __riscv_vsetvlmax_e64m1();
|
||||
/* Lanes per vector pair; the CRC core always folds four lanes. */
|
||||
const size_t vl = max_vl >= 4 ? 4 : (max_vl >= 2 ? 2 : 0);
|
||||
const uint64_t chunk = 64; /* one fixed 64-byte fold span, Zbc-exact */
|
||||
|
||||
crc ^= 0xFFFFFFFF; /* to internal state */
|
||||
|
||||
if (n < chunk || vl == 0) {
|
||||
return rv_crc32_bitwise(crc, reinterpret_cast<const uint8_t*>(buf), len, tab.bitwise_poly) ^ 0xFFFFFFFF;
|
||||
}
|
||||
|
||||
/* Align to 16-byte boundary (same bitwise walk as the Zbc core). */
|
||||
if (uintptr_t mis = reinterpret_cast<uintptr_t>(p) & 0xF) {
|
||||
size_t pre = 16 - mis;
|
||||
if (pre > n) pre = n;
|
||||
crc = rv_crc32_bitwise(crc, p, pre, tab.bitwise_poly);
|
||||
p += pre;
|
||||
n -= pre;
|
||||
if (n < chunk) {
|
||||
return rv_crc32_bitwise(crc, p, n, tab.bitwise_poly) ^ 0xFFFFFFFF;
|
||||
}
|
||||
}
|
||||
|
||||
/* Four lanes, carried by `segs` e64m1 vector pairs:
|
||||
vl=4 (VLEN>=256): 1 pair, one vlseg2e64(4) reads the full 64 bytes
|
||||
vl=2 (VLEN=128): 2 pairs, each vlseg2e64(2) reads 32 bytes
|
||||
Lane j always owns bytes [16j, 16j+16) of the chunk, matching the
|
||||
scalar core's d0/d1 ... d6/d7 assignment. */
|
||||
const uint32_t segs = 4 / vl;
|
||||
|
||||
/* RVV types cannot live in arrays with GCC, so the (at most two)
|
||||
vector pairs are held in named variables. segs==4/vl: 1 pair for
|
||||
vl=4, 2 pairs for vl=2. */
|
||||
vuint64m1_t lo_a, hi_a, lo_b, hi_b;
|
||||
const size_t seg_bytes = vl * 16;
|
||||
{
|
||||
vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2(
|
||||
reinterpret_cast<const uint64_t*>(p), vl);
|
||||
lo_a = __riscv_vget_v_u64m1x2_u64m1(sg, 0);
|
||||
hi_a = __riscv_vget_v_u64m1x2_u64m1(sg, 1);
|
||||
}
|
||||
if (segs == 2) {
|
||||
vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2(
|
||||
reinterpret_cast<const uint64_t*>(p + seg_bytes), vl);
|
||||
lo_b = __riscv_vget_v_u64m1x2_u64m1(sg, 0);
|
||||
hi_b = __riscv_vget_v_u64m1x2_u64m1(sg, 1);
|
||||
}
|
||||
if (vl == 2) {
|
||||
/* Lane 0's low word is element 0 of the first pair. */
|
||||
uint64_t l0[2], h0[2];
|
||||
__riscv_vse64_v_u64m1(l0, lo_a, vl);
|
||||
__riscv_vse64_v_u64m1(h0, hi_a, vl);
|
||||
l0[0] ^= (uint64_t)crc;
|
||||
lo_a = __riscv_vle64_v_u64m1(l0, vl);
|
||||
hi_a = __riscv_vle64_v_u64m1(h0, vl);
|
||||
} else {
|
||||
uint64_t l0[4];
|
||||
__riscv_vse64_v_u64m1(l0, lo_a, vl);
|
||||
l0[0] ^= (uint64_t)crc;
|
||||
lo_a = __riscv_vle64_v_u64m1(l0, vl);
|
||||
}
|
||||
p += chunk;
|
||||
n -= chunk;
|
||||
|
||||
const uint64_t k1 = tab.fold[0];
|
||||
const uint64_t k2 = tab.fold[1];
|
||||
|
||||
while (n >= chunk) {
|
||||
vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2(
|
||||
reinterpret_cast<const uint64_t*>(p), vl);
|
||||
vuint64m1_t dl = __riscv_vget_v_u64m1x2_u64m1(sg, 0);
|
||||
vuint64m1_t dh = __riscv_vget_v_u64m1x2_u64m1(sg, 1);
|
||||
vuint64m1_t lv = __riscv_vxor_vv_u64m1(
|
||||
__riscv_vclmul_vx_u64m1(lo_a, k1, vl),
|
||||
__riscv_vclmul_vx_u64m1(hi_a, k2, vl), vl);
|
||||
vuint64m1_t hv = __riscv_vxor_vv_u64m1(
|
||||
__riscv_vclmulh_vx_u64m1(lo_a, k1, vl),
|
||||
__riscv_vclmulh_vx_u64m1(hi_a, k2, vl), vl);
|
||||
lo_a = __riscv_vxor_vv_u64m1(lv, dl, vl);
|
||||
hi_a = __riscv_vxor_vv_u64m1(hv, dh, vl);
|
||||
if (segs == 2) {
|
||||
vuint64m1x2_t sg2 = __riscv_vlseg2e64_v_u64m1x2(
|
||||
reinterpret_cast<const uint64_t*>(p + seg_bytes), vl);
|
||||
vuint64m1_t dl2 = __riscv_vget_v_u64m1x2_u64m1(sg2, 0);
|
||||
vuint64m1_t dh2 = __riscv_vget_v_u64m1x2_u64m1(sg2, 1);
|
||||
vuint64m1_t lv2 = __riscv_vxor_vv_u64m1(
|
||||
__riscv_vclmul_vx_u64m1(lo_b, k1, vl),
|
||||
__riscv_vclmul_vx_u64m1(hi_b, k2, vl), vl);
|
||||
vuint64m1_t hv2 = __riscv_vxor_vv_u64m1(
|
||||
__riscv_vclmulh_vx_u64m1(lo_b, k1, vl),
|
||||
__riscv_vclmulh_vx_u64m1(hi_b, k2, vl), vl);
|
||||
lo_b = __riscv_vxor_vv_u64m1(lv2, dl2, vl);
|
||||
hi_b = __riscv_vxor_vv_u64m1(hv2, dh2, vl);
|
||||
}
|
||||
p += chunk;
|
||||
n -= chunk;
|
||||
}
|
||||
|
||||
/* Merge the four lanes (same order as the scalar 4-way fold). */
|
||||
uint64_t loa[4], hia[4];
|
||||
__riscv_vse64_v_u64m1(loa, lo_a, vl);
|
||||
__riscv_vse64_v_u64m1(hia, hi_a, vl);
|
||||
if (segs == 2) {
|
||||
__riscv_vse64_v_u64m1(loa + 2, lo_b, vl);
|
||||
__riscv_vse64_v_u64m1(hia + 2, hi_b, vl);
|
||||
}
|
||||
const uint64_t k3 = tab.fold[2];
|
||||
const uint64_t k4 = tab.fold[3];
|
||||
uint64_t x0 = loa[0], x1 = hia[0];
|
||||
for (uint32_t j = 1; j < 4; ++j)
|
||||
rv_fold_pair(&x0, &x1, k3, k4, loa[j], hia[j]);
|
||||
|
||||
/* Barrett reduction: 128-bit -> 32-bit (same as the Zbc core). */
|
||||
uint64_t t4 = rv_clmul(x0, tab.const1);
|
||||
uint64_t t3 = rv_clmulh(x0, tab.const1);
|
||||
uint64_t t1 = x1 ^ t4;
|
||||
t4 = t1 & RV_CRC32_MASK32;
|
||||
t1 >>= 32;
|
||||
uint64_t t0 = rv_clmul(t4, tab.const0);
|
||||
t3 = (t3 << 32) ^ t1 ^ t0;
|
||||
|
||||
t4 = t3 & RV_CRC32_MASK32;
|
||||
t4 = rv_clmul(t4, tab.quo);
|
||||
t4 &= RV_CRC32_MASK32;
|
||||
t4 = rv_clmul(t4, tab.poly);
|
||||
t4 ^= t3;
|
||||
|
||||
uint32_t c = static_cast<uint32_t>((t4 >> 32) & RV_CRC32_MASK32);
|
||||
if (n)
|
||||
c = rv_crc32_bitwise(c, p, n, tab.bitwise_poly);
|
||||
return c ^ 0xFFFFFFFF;
|
||||
}
|
||||
#endif /* __riscv_zvbc */
|
||||
|
||||
} // namespace crc32c
|
||||
} // namespace mysys_namespace
|
||||
|
||||
extern "C" unsigned crc32c_riscv_zvbc(unsigned crc, const void *buf,
|
||||
size_t len)
|
||||
{
|
||||
#if defined(__riscv_zvbc)
|
||||
return mysys_namespace::crc32c::rv_crc32c_zvbc_impl(
|
||||
crc, reinterpret_cast<const char*>(buf), len,
|
||||
mysys_namespace::crc32c::rv_crc32c_tab);
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
Reference in New Issue
Block a user