diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 5cd230d592c..137612c4c80 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -185,6 +185,35 @@ ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64|RISCV64") crc32/crc32c_riscv.cc crc32/crc32c_riscv_zbc.cc) SET_SOURCE_FILES_PROPERTIES(crc32/crc32c_riscv_zbc.cc PROPERTIES COMPILE_FLAGS "-march=rv64gc_zbc_zbb") + + # Zvbc vector core (optional): separate TU compiled for the vector + # extension; the runtime probe/dispatch stay in crc32c_riscv.cc. + # Zbc is deliberately left out of the -march: this core is all-vector, + # and with Zbc present GCC's -foptimize-crc (on by default at -O2 since + # GCC 15) rewrites the bitwise fallback in that file into scalar clmul + # instructions, which do not exist on the Zvbc-without-Zbc cores this + # code is for. + SET(SAVE_REQ_FLAGS2 "${CMAKE_REQUIRED_FLAGS}") + SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=rv64gc_zbb_zvbc") + CHECK_CXX_SOURCE_COMPILES(" + #include + #include + uint64_t f(uint64_t a, const uint64_t *p, size_t vl) { + vuint64m1_t v = __riscv_vle64_v_u64m1(p, vl); + return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmul_vx_u64m1(v, a, vl)); + } + int main() { uint64_t x[4]={1,2,3,4}; return (int) f(5, x, 4); } + " HAVE_RISCV_ZVBC) + SET(CMAKE_REQUIRED_FLAGS "${SAVE_REQ_FLAGS2}") + IF(HAVE_RISCV_ZVBC) + MESSAGE(STATUS "RISC-V Zvbc detected: enabling vector CRC32C") + SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32c_riscv_zvbc.cc) + SET_SOURCE_FILES_PROPERTIES(crc32/crc32c_riscv_zvbc.cc PROPERTIES + COMPILE_FLAGS "-march=rv64gc_zbb_zvbc") + ADD_DEFINITIONS(-DHAVE_RISCV_ZVBC) + ELSE() + MESSAGE(STATUS "RISC-V Zvbc not available: using Zbc scalar CRC32C") + ENDIF() ENDIF() ENDIF() ENDIF() diff --git a/mysys/crc32/crc32c.cc b/mysys/crc32/crc32c.cc index f72e011ec11..c28daaed3ed 100644 --- a/mysys/crc32/crc32c.cc +++ b/mysys/crc32/crc32c.cc @@ -491,7 +491,10 @@ extern "C" my_crc32_t crc32c_aarch64_available(void); extern "C" const char *crc32c_aarch64_impl(my_crc32_t); #elif defined HAVE_RISCV_ZBC extern "C" unsigned crc32c_riscv_zbc(unsigned, const void *, size_t); -extern "C" int rv_zbc_supported(void *); +# ifdef HAVE_RISCV_ZVBC +extern "C" unsigned crc32c_riscv_zvbc(unsigned, const void *, size_t); +# endif +extern "C" unsigned rv_riscv_crc_ext(void *); extern "C" const char *crc32c_riscv_impl(my_crc32_t); #elif defined __i386__||defined __x86_64__||defined _M_X64||defined _M_IX86 extern "C" my_crc32_t crc32c_x86_available(void); @@ -501,7 +504,33 @@ extern "C" const char *crc32c_x86_impl(my_crc32_t); #if defined HAVE_RISCV_ZBC static my_crc32_t crc32c_riscv_choose(void *hwprobe) { - return rv_zbc_supported(hwprobe) ? crc32c_riscv_zbc : crc32c_slow; + unsigned ext= rv_riscv_crc_ext(hwprobe); + /* Where scalar Zbc is available, prefer it over the Zvbc vector core. + + The vector core does not buy extra parallelism here. To stay + bit-exact with the scalar core it reuses exactly the same fold: + four 128-bit lanes over a fixed 64-byte span, with the same + constants k1..k4 and the same Barrett step. It therefore caps vl + at 4 and folds 64 bytes per iteration however wide VLEN is, and it + issues the same number of carry-less multiplies. Its only edge is + a single de-interleaving segment load per 64 bytes, which on large + input is offset by its per-call overhead (vsetvl, and injecting + the CRC into lane 0 through a store/load round trip): on Spacemit + X100 (VLEN=256) it ties the scalar fold at 64 KiB (11398 vs 11687 + MB/s) and loses on small input (128 B: 1147 vs 5540 MB/s, + 1 KiB: 5466 vs 10277 MB/s). + + Its reason to exist is cores that implement Zvbc but not scalar + Zbc, where the only alternative is the slicing-by-4 crc32c_slow. + Where Zbc is present, selecting it would only touch the vector + register file for no gain. */ + if (ext & 1) + return crc32c_riscv_zbc; +#ifdef HAVE_RISCV_ZVBC + if (ext & 2) /* Zvbc only (no scalar Zbc) */ + return crc32c_riscv_zvbc; +#endif + return crc32c_slow; } /* The RISC-V resolver. Unlike the other architectures, the implementation is @@ -509,7 +538,7 @@ static my_crc32_t crc32c_riscv_choose(void *hwprobe) target operating systems of RISC-V (Linux and FreeBSD) support that. The dynamic linker calls this at load time, before main(), so it may only - run code that is safe there. rv_zbc_supported() is: it uses the + run code that is safe there. rv_riscv_crc_ext() is: it uses the riscv_hwprobe system call and nothing else. The resolver must never return NULL. Whereas the *_available() functions diff --git a/mysys/crc32/crc32c_riscv.cc b/mysys/crc32/crc32c_riscv.cc index 01eccc9df97..241a209c5e5 100644 --- a/mysys/crc32/crc32c_riscv.cc +++ b/mysys/crc32/crc32c_riscv.cc @@ -16,6 +16,7 @@ argument is NULL and the system call is made directly. */ #include +#include #include #include @@ -37,6 +38,9 @@ struct riscv_hwprobe { long long key; unsigned long long value; }; #ifndef RISCV_HWPROBE_EXT_ZBC # define RISCV_HWPROBE_EXT_ZBC (1ULL << 7) #endif +#ifndef RISCV_HWPROBE_EXT_ZVBC +# define RISCV_HWPROBE_EXT_ZVBC (1ULL << 18) +#endif #ifndef SYS_riscv_hwprobe # ifdef __NR_riscv_hwprobe @@ -77,9 +81,45 @@ extern "C" int rv_zbc_supported(void *hwprobe) } extern "C" unsigned crc32c_riscv_zbc(unsigned, const void *, size_t); +#ifdef HAVE_RISCV_ZVBC +extern "C" unsigned crc32c_riscv_zvbc(unsigned, const void *, size_t); +#endif + +/* CRC acceleration level for the callers' ifunc resolver: + 3 = Zbc + Zvbc (prefer Zbc; the optimized scalar fold ties the 4-lane + vector core at VLEN 256 and wins below it), + 2 = Zvbc only (no scalar Zbc -- the vector core is the only + accelerated path on such cores), 1 = Zbc only, 0 = none. */ +extern "C" unsigned rv_riscv_crc_ext(void *hwprobe) +{ + struct riscv_hwprobe p; + p.key= RISCV_HWPROBE_KEY_IMA_EXT_0; + p.value= 0; +#ifdef HAVE_SYS_HWPROBE_H + if (hwprobe != NULL) + { + unsigned long long value= 0; + if (__riscv_hwprobe_one(reinterpret_cast<__riscv_hwprobe_t>(hwprobe), + RISCV_HWPROBE_KEY_IMA_EXT_0, &value) != 0) + return 0; + return ((value & RISCV_HWPROBE_EXT_ZBC) ? 1u : 0u) | + ((value & RISCV_HWPROBE_EXT_ZVBC) ? 2u : 0u); + } +#else + std::ignore = hwprobe; +#endif + if (syscall(SYS_riscv_hwprobe, &p, (size_t) 1, (size_t) 0, NULL, 0) != 0) + return 0; + return ((p.value & RISCV_HWPROBE_EXT_ZBC) ? 1u : 0u) | + ((p.value & RISCV_HWPROBE_EXT_ZVBC) ? 2u : 0u); +} extern "C" const char *crc32c_riscv_impl(my_crc32_t c) { +#ifdef HAVE_RISCV_ZVBC + if (c == crc32c_riscv_zvbc) + return "Using RISC-V Zvbc vector carry-less multiply instructions"; +#endif if (c == crc32c_riscv_zbc) return "Using RISC-V Zbc carry-less multiply instructions"; return NULL; diff --git a/mysys/crc32/crc32c_riscv_zvbc.cc b/mysys/crc32/crc32c_riscv_zvbc.cc new file mode 100644 index 00000000000..33512d9ee69 --- /dev/null +++ b/mysys/crc32/crc32c_riscv_zvbc.cc @@ -0,0 +1,247 @@ +/* RISC-V Zvbc vector accelerated CRC-32C. + +K-lane Zvbc implementation with *adaptive* vector length. The CRC fold +itself keeps the exact 4-lane / 64-byte span of the scalar Zbc core (same +fold constants k1..k4, same Barrett math) so the result is bit-exact; the +vector length only decides how many e64m1 vector pairs carry the four +lanes in parallel (one pair of vl>=4 vectors on VLEN>=256, two pairs of +vl=2 vectors on VLEN=128). A CPU with VLEN=128 therefore still executes +the vector path instead of being forced back to scalar code. + +All carry-less multiplies run through the single-element vector vclmul +path (a scalar `clmul` would trap on cores that implement Zvbc but not +Zbc). Small inputs, VLEN < 2 lanes, and the post-alignment remainder +fall back to the in-TU bitwise walk with the external CRC state. + +This is the only compilation unit enabled for the vector extension; it is +compiled with -march=rv64gc_zbb_zvbc (see mysys/CMakeLists.txt). zbc is +deliberately absent from that -march: with zbc present, GCC's +-foptimize-crc (on by default at -O2 since GCC 15) rewrites the bitwise +fallback below into scalar clmul/clmulh, which do not exist on the cores +this file is for. +*/ + +#include +#include +#include + +#if defined(__riscv_zvbc) +#include +#endif + +typedef unsigned (*my_crc32_t)(unsigned, const void *, size_t); + +namespace mysys_namespace { +namespace crc32c { + +struct rv_crc32_tab { + alignas(16) uint32_t fold[4]; + uint64_t const0; + uint64_t const1; + uint64_t quo; + uint64_t poly; + uint32_t bitwise_poly; +}; + +/* The scalar Zbc `clmul` instruction is intentionally NOT used: some cores + implement Zvbc but not B-ext scalar clmul. Every CLMUL in this TU goes + through the single-element vector vclmul path instead. */ +static inline uint64_t rv_clmul(uint64_t a, uint64_t b) { + const size_t vl = 1; + vuint64m1_t va = __riscv_vmv_v_x_u64m1(a, vl); + vuint64m1_t vb = __riscv_vmv_v_x_u64m1(b, vl); + return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmul_vv_u64m1(va, vb, vl)); +} +static inline uint64_t rv_clmulh(uint64_t a, uint64_t b) { + const size_t vl = 1; + vuint64m1_t va = __riscv_vmv_v_x_u64m1(a, vl); + vuint64m1_t vb = __riscv_vmv_v_x_u64m1(b, vl); + return __riscv_vmv_x_s_u64m1_u64(__riscv_vclmulh_vv_u64m1(va, vb, vl)); +} + +static inline uint32_t rv_crc32_bitwise(uint32_t crc, const uint8_t* buf, + size_t len, uint32_t poly) { + uint32_t c = crc; + for (size_t i = 0; i < len; ++i) { + c ^= buf[i]; + for (int k = 0; k < 8; ++k) { + c = (c >> 1) ^ ((c & 1) ? poly : 0); + } + } + return c; +} + +static inline void rv_fold_pair(uint64_t* lo, uint64_t* hi, + uint64_t k0, uint64_t k1, + uint64_t v0, uint64_t v1) { + uint64_t l = rv_clmul(*lo, k0) ^ rv_clmul(*hi, k1); + uint64_t h = rv_clmulh(*lo, k0) ^ rv_clmulh(*hi, k1); + *lo = l ^ v0; + *hi = h ^ v1; +} + +/* CRC-32C (Castagnoli), polynomial 0x1EDC6F41 -- same values as + crc32c_riscv_zbc.cc, kept here for the vector core. */ +static constexpr rv_crc32_tab rv_crc32c_tab = { + { 0x740eef02, 0x9e4addf8, 0xf20c0dfe, 0x493c7d27 }, + 0x00000000dd45aab8ULL, 0x00000000493c7d27ULL, + 0x0000000dea713f1ULL, 0x0000000105ec76f1ULL, + 0x82F63B78 +}; + +static constexpr uint64_t RV_CRC32_MASK32 = 0x00000000FFFFFFFFULL; + +#if defined(__riscv_zvbc) +static uint32_t rv_crc32c_zvbc_impl(uint32_t crc, const char* buf, + size_t len, const rv_crc32_tab& tab) { + const uint8_t* p = reinterpret_cast(buf); + size_t n = len; + + size_t max_vl = __riscv_vsetvlmax_e64m1(); + /* Lanes per vector pair; the CRC core always folds four lanes. */ + const size_t vl = max_vl >= 4 ? 4 : (max_vl >= 2 ? 2 : 0); + const uint64_t chunk = 64; /* one fixed 64-byte fold span, Zbc-exact */ + + crc ^= 0xFFFFFFFF; /* to internal state */ + + if (n < chunk || vl == 0) { + return rv_crc32_bitwise(crc, reinterpret_cast(buf), len, tab.bitwise_poly) ^ 0xFFFFFFFF; + } + + /* Align to 16-byte boundary (same bitwise walk as the Zbc core). */ + if (uintptr_t mis = reinterpret_cast(p) & 0xF) { + size_t pre = 16 - mis; + if (pre > n) pre = n; + crc = rv_crc32_bitwise(crc, p, pre, tab.bitwise_poly); + p += pre; + n -= pre; + if (n < chunk) { + return rv_crc32_bitwise(crc, p, n, tab.bitwise_poly) ^ 0xFFFFFFFF; + } + } + + /* Four lanes, carried by `segs` e64m1 vector pairs: + vl=4 (VLEN>=256): 1 pair, one vlseg2e64(4) reads the full 64 bytes + vl=2 (VLEN=128): 2 pairs, each vlseg2e64(2) reads 32 bytes + Lane j always owns bytes [16j, 16j+16) of the chunk, matching the + scalar core's d0/d1 ... d6/d7 assignment. */ + const uint32_t segs = 4 / vl; + + /* RVV types cannot live in arrays with GCC, so the (at most two) + vector pairs are held in named variables. segs==4/vl: 1 pair for + vl=4, 2 pairs for vl=2. */ + vuint64m1_t lo_a, hi_a, lo_b, hi_b; + const size_t seg_bytes = vl * 16; + { + vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2( + reinterpret_cast(p), vl); + lo_a = __riscv_vget_v_u64m1x2_u64m1(sg, 0); + hi_a = __riscv_vget_v_u64m1x2_u64m1(sg, 1); + } + if (segs == 2) { + vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2( + reinterpret_cast(p + seg_bytes), vl); + lo_b = __riscv_vget_v_u64m1x2_u64m1(sg, 0); + hi_b = __riscv_vget_v_u64m1x2_u64m1(sg, 1); + } + if (vl == 2) { + /* Lane 0's low word is element 0 of the first pair. */ + uint64_t l0[2], h0[2]; + __riscv_vse64_v_u64m1(l0, lo_a, vl); + __riscv_vse64_v_u64m1(h0, hi_a, vl); + l0[0] ^= (uint64_t)crc; + lo_a = __riscv_vle64_v_u64m1(l0, vl); + hi_a = __riscv_vle64_v_u64m1(h0, vl); + } else { + uint64_t l0[4]; + __riscv_vse64_v_u64m1(l0, lo_a, vl); + l0[0] ^= (uint64_t)crc; + lo_a = __riscv_vle64_v_u64m1(l0, vl); + } + p += chunk; + n -= chunk; + + const uint64_t k1 = tab.fold[0]; + const uint64_t k2 = tab.fold[1]; + + while (n >= chunk) { + vuint64m1x2_t sg = __riscv_vlseg2e64_v_u64m1x2( + reinterpret_cast(p), vl); + vuint64m1_t dl = __riscv_vget_v_u64m1x2_u64m1(sg, 0); + vuint64m1_t dh = __riscv_vget_v_u64m1x2_u64m1(sg, 1); + vuint64m1_t lv = __riscv_vxor_vv_u64m1( + __riscv_vclmul_vx_u64m1(lo_a, k1, vl), + __riscv_vclmul_vx_u64m1(hi_a, k2, vl), vl); + vuint64m1_t hv = __riscv_vxor_vv_u64m1( + __riscv_vclmulh_vx_u64m1(lo_a, k1, vl), + __riscv_vclmulh_vx_u64m1(hi_a, k2, vl), vl); + lo_a = __riscv_vxor_vv_u64m1(lv, dl, vl); + hi_a = __riscv_vxor_vv_u64m1(hv, dh, vl); + if (segs == 2) { + vuint64m1x2_t sg2 = __riscv_vlseg2e64_v_u64m1x2( + reinterpret_cast(p + seg_bytes), vl); + vuint64m1_t dl2 = __riscv_vget_v_u64m1x2_u64m1(sg2, 0); + vuint64m1_t dh2 = __riscv_vget_v_u64m1x2_u64m1(sg2, 1); + vuint64m1_t lv2 = __riscv_vxor_vv_u64m1( + __riscv_vclmul_vx_u64m1(lo_b, k1, vl), + __riscv_vclmul_vx_u64m1(hi_b, k2, vl), vl); + vuint64m1_t hv2 = __riscv_vxor_vv_u64m1( + __riscv_vclmulh_vx_u64m1(lo_b, k1, vl), + __riscv_vclmulh_vx_u64m1(hi_b, k2, vl), vl); + lo_b = __riscv_vxor_vv_u64m1(lv2, dl2, vl); + hi_b = __riscv_vxor_vv_u64m1(hv2, dh2, vl); + } + p += chunk; + n -= chunk; + } + + /* Merge the four lanes (same order as the scalar 4-way fold). */ + uint64_t loa[4], hia[4]; + __riscv_vse64_v_u64m1(loa, lo_a, vl); + __riscv_vse64_v_u64m1(hia, hi_a, vl); + if (segs == 2) { + __riscv_vse64_v_u64m1(loa + 2, lo_b, vl); + __riscv_vse64_v_u64m1(hia + 2, hi_b, vl); + } + const uint64_t k3 = tab.fold[2]; + const uint64_t k4 = tab.fold[3]; + uint64_t x0 = loa[0], x1 = hia[0]; + for (uint32_t j = 1; j < 4; ++j) + rv_fold_pair(&x0, &x1, k3, k4, loa[j], hia[j]); + + /* Barrett reduction: 128-bit -> 32-bit (same as the Zbc core). */ + uint64_t t4 = rv_clmul(x0, tab.const1); + uint64_t t3 = rv_clmulh(x0, tab.const1); + uint64_t t1 = x1 ^ t4; + t4 = t1 & RV_CRC32_MASK32; + t1 >>= 32; + uint64_t t0 = rv_clmul(t4, tab.const0); + t3 = (t3 << 32) ^ t1 ^ t0; + + t4 = t3 & RV_CRC32_MASK32; + t4 = rv_clmul(t4, tab.quo); + t4 &= RV_CRC32_MASK32; + t4 = rv_clmul(t4, tab.poly); + t4 ^= t3; + + uint32_t c = static_cast((t4 >> 32) & RV_CRC32_MASK32); + if (n) + c = rv_crc32_bitwise(c, p, n, tab.bitwise_poly); + return c ^ 0xFFFFFFFF; +} +#endif /* __riscv_zvbc */ + +} // namespace crc32c +} // namespace mysys_namespace + +extern "C" unsigned crc32c_riscv_zvbc(unsigned crc, const void *buf, + size_t len) +{ +#if defined(__riscv_zvbc) + return mysys_namespace::crc32c::rv_crc32c_zvbc_impl( + crc, reinterpret_cast(buf), len, + mysys_namespace::crc32c::rv_crc32c_tab); +#else + return 0; +#endif +}