From fa8dc815a74ed285354b6131c005dbf383fc4962 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Thu, 15 Jul 2021 21:40:20 +0300 Subject: [PATCH] MCOL-4814 Add a cmake build option to enable LZ4 compression. This patch adds an option for cmake flags to enable lz4 compression. --- CMakeLists.txt | 22 +++++++++++++++++----- dbcon/mysql/ha_mcs_sysvars.cpp | 3 +++ dbcon/mysql/ha_mcs_sysvars.h | 3 +++ mcsconfig.h.in | 3 +++ utils/compress/CMakeLists.txt | 7 +++++-- utils/compress/idbcompress.cpp | 15 +++++++++++++++ 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 08e54533b..2fdde8afc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ SET(CMAKE_POSITION_INDEPENDENT_CODE TRUE) SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj) SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) +SET(WITH_COLUMNSTORE_LZ4 AUTO CACHE STRING "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") SET (ENGINE_SYSCONFDIR "/etc") SET (ENGINE_DATADIR "/var/lib/columnstore") @@ -163,11 +164,22 @@ if(NOT AWK_EXECUTABLE) return() endif() -FIND_PACKAGE(LZ4) -if (NOT LZ4_FOUND) - MESSAGE_ONCE(CS_NO_LZ4 "lz4 not found") - return() -endif() +SET(HAVE_LZ4 0 CACHE INTERNAL "") +IF (WITH_COLUMNSTORE_LZ4 STREQUAL "ON" OR WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") + FIND_PACKAGE(LZ4) + IF (NOT LZ4_FOUND) + IF (WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO") + MESSAGE_ONCE(STATUS "LZ4 not found, building without LZ4") + ELSE() + MESSAGE_ONCE(FATAL_ERROR "LZ4 not found.") + ENDIF() + ELSE() + MESSAGE_ONCE(STATUS "Building with LZ4") + SET(HAVE_LZ4 1 CACHE INTERNAL "") + ENDIF() +ELSE() + MESSAGE_ONCE(STATUS "Building without LZ4") +ENDIF() IF (NOT INSTALL_LAYOUT) INCLUDE(check_compiler_flag) diff --git a/dbcon/mysql/ha_mcs_sysvars.cpp b/dbcon/mysql/ha_mcs_sysvars.cpp index 55e4b14d7..517163ce5 100644 --- a/dbcon/mysql/ha_mcs_sysvars.cpp +++ b/dbcon/mysql/ha_mcs_sysvars.cpp @@ -19,12 +19,15 @@ #include #include "idb_mysql.h" #include "ha_mcs_sysvars.h" +#include "mcsconfig.h" const char* mcs_compression_type_names[] = { "SNAPPY", // 0 "SNAPPY", // 1 "SNAPPY", // 2 +#ifdef HAVE_LZ4 "LZ4", // 3 +#endif NullS }; diff --git a/dbcon/mysql/ha_mcs_sysvars.h b/dbcon/mysql/ha_mcs_sysvars.h index e74585862..a1a3a8d2a 100644 --- a/dbcon/mysql/ha_mcs_sysvars.h +++ b/dbcon/mysql/ha_mcs_sysvars.h @@ -21,6 +21,7 @@ #include #include "idb_mysql.h" +#include "mcsconfig.h" extern st_mysql_sys_var* mcs_system_variables[]; extern st_mysql_show_var mcs_status_variables[]; @@ -31,7 +32,9 @@ extern char cs_commit_hash[]; enum mcs_compression_type_t { NO_COMPRESSION = 0, SNAPPY = 2, +#ifdef HAVE_LZ4 LZ4 = 3 +#endif }; // use_import_for_batchinsert mode diff --git a/mcsconfig.h.in b/mcsconfig.h.in index 9f6ba06e6..07cbc6257 100644 --- a/mcsconfig.h.in +++ b/mcsconfig.h.in @@ -268,6 +268,9 @@ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_ZLIB_H 1 +/* Define to 1 if you have lz4 library. */ +#cmakedefine HAVE_LZ4 1 + /* Define to 1 if the system has the type `_Bool'. */ #cmakedefine HAVE__BOOL 1 diff --git a/utils/compress/CMakeLists.txt b/utils/compress/CMakeLists.txt index b3dc2b068..16bb503b5 100644 --- a/utils/compress/CMakeLists.txt +++ b/utils/compress/CMakeLists.txt @@ -10,7 +10,10 @@ add_definitions(-DNDEBUG) add_library(compress SHARED ${compress_LIB_SRCS}) -target_link_libraries(compress ${SNAPPY_LIBRARIES} ${LZ4_LIBRARIES}) +target_link_libraries(compress ${SNAPPY_LIBRARIES}) +IF(HAVE_LZ4) + MESSAGE_ONCE(STATUS "LINK WITH LZ4") + target_link_libraries(compress ${LZ4_LIBRARIES}) +ENDIF() install(TARGETS compress DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - diff --git a/utils/compress/idbcompress.cpp b/utils/compress/idbcompress.cpp index 79f812fbd..43b9041ea 100644 --- a/utils/compress/idbcompress.cpp +++ b/utils/compress/idbcompress.cpp @@ -29,7 +29,14 @@ using namespace std; #include "logger.h" #include "snappy.h" #include "hasher.h" +#include "mcsconfig.h" +#ifdef HAVE_LZ4 #include "lz4.h" +#else +// Taken from lz4.h. +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) +#endif #define IDBCOMP_DLLEXPORT #include "idbcompress.h" @@ -590,6 +597,7 @@ CompressInterfaceLZ4::CompressInterfaceLZ4(uint32_t numUserPaddingBytes) int32_t CompressInterfaceLZ4::compress(const char* in, size_t inLen, char* out, size_t* outLen) const { +#ifdef HAVE_LZ4 auto compressedLen = LZ4_compress_default(in, out, inLen, *outLen); if (!compressedLen) @@ -606,11 +614,15 @@ int32_t CompressInterfaceLZ4::compress(const char* in, size_t inLen, char* out, *outLen = compressedLen; return ERR_OK; +#else + return ERR_COMPRESS; +#endif } int32_t CompressInterfaceLZ4::uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const { +#ifdef HAVE_LZ4 auto decompressedLen = LZ4_decompress_safe(in, out, inLen, *outLen); if (decompressedLen < 0) @@ -629,6 +641,9 @@ int32_t CompressInterfaceLZ4::uncompress(const char* in, size_t inLen, #endif return ERR_OK; +#else + return ERR_DECOMPRESS; +#endif } size_t CompressInterfaceLZ4::maxCompressedSize(size_t uncompSize) const