From 864223166668ab9eeeff5cb83712f0cd48a004f5 Mon Sep 17 00:00:00 2001 From: "david.hall" Date: Thu, 17 Nov 2022 11:29:21 -0600 Subject: [PATCH 1/8] Changes to compile local --- CMakeLists.txt | 2 +- utils/rowgroup/rowstorage.cpp | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9911b520..b85bae3e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,7 +187,7 @@ ENDIF() IF (NOT INSTALL_LAYOUT) MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) + MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2 -DBOOST_BIND_GLOBAL_PLACEHOLDERS" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) # enable security hardening features, like most distributions do # in our benchmarks that costs about ~1% of performance, depending on the load diff --git a/utils/rowgroup/rowstorage.cpp b/utils/rowgroup/rowstorage.cpp index ad0144b36..4690f6cf5 100644 --- a/utils/rowgroup/rowstorage.cpp +++ b/utils/rowgroup/rowstorage.cpp @@ -789,7 +789,6 @@ class RowGroupStorage void putKeyRow(uint64_t idx, Row& row) { uint64_t rgid = idx / fMaxRows; - uint64_t rid = idx % fMaxRows; while (rgid >= fRGDatas.size()) { @@ -819,7 +818,7 @@ class RowGroupStorage fLRU->add(rgid); - assert(rid == fRowGroupOut->getRowCount()); + assert(idx % fMaxRows == fRowGroupOut->getRowCount()); fRowGroupOut->getRow(fRowGroupOut->getRowCount(), &row); fRowGroupOut->incRowCount(); } From 9b84bf57c91463482d9e4c9fa63081c0eadd30aa Mon Sep 17 00:00:00 2001 From: "david.hall" Date: Thu, 17 Nov 2022 11:46:10 -0600 Subject: [PATCH 2/8] From serg: add dependency for generated header files errorids.h messageids.h --- datatypes/CMakeLists.txt | 2 +- dbcon/joblist/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt index fe8fa7142..3526d6184 100644 --- a/datatypes/CMakeLists.txt +++ b/datatypes/CMakeLists.txt @@ -5,7 +5,7 @@ set(datatypes_LIB_SRCS mcs_decimal.cpp) add_library(datatypes SHARED ${datatypes_LIB_SRCS}) -add_dependencies(datatypes external_boost) +add_dependencies(datatypes loggingcpp external_boost) install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 67261842f..db1e12a32 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -63,6 +63,7 @@ set(joblist_LIB_SRCS ${ENGINE_SRC_DIR}/tools/passwd/secrets.cpp) add_library(joblist SHARED ${joblist_LIB_SRCS}) +add_dependencies(joblist_server loggingcpp) target_include_directories(joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) add_dependencies(joblist loggingcpp) From ae88ef182324f79738bee5519387fe13ae7b25b2 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 10 Nov 2022 00:22:15 +0100 Subject: [PATCH 3/8] use system boost, if possible boost 1.71.0 is what ubuntu focal has, so let's start with that version. boost 1.77.0 is the first that supports c++20 --- CMakeLists.txt | 7 ++++++- cmake/boost.cmake | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b85bae3e1..c21a83c7b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,12 @@ INCLUDE(check_compiler_flag) INCLUDE(ExternalProject) INCLUDE(CheckCXXSourceCompiles) -MY_CHECK_CXX_COMPILER_FLAG("-std=c++20") +FIND_PACKAGE(Boost 1.71.0 COMPONENTS chrono filesystem program_options regex system thread) + +IF (NOT Boost_FOUND OR Boost_VERSION VERSION_GREATER 1.76.99) + MY_CHECK_CXX_COMPILER_FLAG("-std=c++20") +ENDIF() + IF (have_CXX__std_c__20) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20") ELSE() diff --git a/cmake/boost.cmake b/cmake/boost.cmake index 534e72110..66a26670c 100644 --- a/cmake/boost.cmake +++ b/cmake/boost.cmake @@ -1,3 +1,8 @@ +if (Boost_FOUND) + add_custom_target(external_boost) + return() +endif() + include(ExternalProject) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") From 21c3bbce162df18e943377eda5eb318849c4a298 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 10 Nov 2022 22:12:07 +0100 Subject: [PATCH 4/8] do *not* link ha_columnstore.so with libmariadb.so this means some libraries have to be compiled twice - for tools with libmariadb.so and for plugin, without. --- CMakeLists.txt | 8 +++++--- dbcon/joblist/CMakeLists.txt | 18 ++++++++---------- dbcon/mysql/CMakeLists.txt | 4 +++- utils/libmysql_client/libmysql_client.h | 4 ++++ utils/querystats/querystats.cpp | 1 - 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c21a83c7b..88a598ab5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,11 +239,13 @@ MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-copy" DEBUG RELEASE RELWITHDEBIN MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") SET (ENGINE_DT_LIB datatypes) -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt libmysql_client ${ENGINE_DT_LIB}) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt ${ENGINE_DT_LIB}) SET (ENGINE_OAM_LIBS oamcpp) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) -SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) -SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils ${ENGINE_EXEC_LIBS}) +SET (PLUGIN_EXEC_LIBS execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querytele thrift threadpool ${ENGINE_BRM_LIBS}) +SET (ENGINE_EXEC_LIBS joblist querystats libmysql_client ${PLUGIN_EXEC_LIBS}) +SET (PLUGIN_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils) +SET (ENGINE_WRITE_LIBS ${PLUGIN_WRITE_LIBS} ${ENGINE_EXEC_LIBS}) SET (ENGINE_COMMON_LDFLAGS "") diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index db1e12a32..0e1e998c8 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -1,13 +1,5 @@ -include_directories(BEFORE - ${CMAKE_BINARY_DIR}/libmariadb/include - ${CMAKE_SOURCE_DIR}/libmariadb/include) -add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) - include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_SRC_DIR}/tools/passwd) - -########### next target ############### - set(joblist_LIB_SRCS anydatalist.cpp batchprimitiveprocessor-jl.cpp @@ -62,9 +54,15 @@ set(joblist_LIB_SRCS windowfunctionstep.cpp ${ENGINE_SRC_DIR}/tools/passwd/secrets.cpp) +########### next target ############### + +add_library(joblist_server STATIC ${joblist_LIB_SRCS}) + +########### next target ############### + +add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) + add_library(joblist SHARED ${joblist_LIB_SRCS}) -add_dependencies(joblist_server loggingcpp) -target_include_directories(joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) add_dependencies(joblist loggingcpp) install(TARGETS joblist DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index eff43d84a..7af81922f 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -11,6 +11,8 @@ SET(S3API_DEPS marias3 curl CACHE INTERNAL "S3API_DEPS") SET ( libcalmysql_SRCS ../../datatypes/mcs_datatype.cpp + ../../utils/libmysql_client/libmysql_client.cpp + ../../utils/querystats/querystats.cpp ha_mcs_sysvars.cpp ha_mcs_client_udfs.cpp ha_mcs_opt_rewrites.cpp @@ -45,7 +47,7 @@ if (COMMAND mysql_add_plugin) ENDIF() mysql_add_plugin(columnstore ${libcalmysql_SRCS} STORAGE_ENGINE MODULE_ONLY ${disabled} - LINK_LIBRARIES ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${S3API_DEPS} threadpool + LINK_LIBRARIES ${ENGINE_LDFLAGS} ${PLUGIN_EXEC_LIBS} ${PLUGIN_WRITE_LIBS} joblist_server ${NETSNMP_LIBRARIES} ${S3API_DEPS} threadpool VERSION ${PACKAGE_VERSION} COMPONENT columnstore-engine CONFIG columnstore.cnf) else () diff --git a/utils/libmysql_client/libmysql_client.h b/utils/libmysql_client/libmysql_client.h index 931886b78..5e04304d0 100644 --- a/utils/libmysql_client/libmysql_client.h +++ b/utils/libmysql_client/libmysql_client.h @@ -19,7 +19,11 @@ #define UTILS LIBMYSQL_CL_H #include +#ifdef MYSQL_DYNAMIC_PLUGIN +#include +#else #include +#endif #include diff --git a/utils/querystats/querystats.cpp b/utils/querystats/querystats.cpp index 360e774ce..ac9ed79e6 100644 --- a/utils/querystats/querystats.cpp +++ b/utils/querystats/querystats.cpp @@ -23,7 +23,6 @@ #define PREFER_MY_CONFIG_H #include -#include #include using namespace std; From 246a4db8dec7c6d4b21c9d7583b188e98d6a31e2 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 4 Sep 2022 20:26:41 +0200 Subject: [PATCH 5/8] fix C API includes ColumnStore used to include server's mysql.h but link all tools with libmariadb.so There's no guarantee that this would work, even with workarounds it had in dbcon/mysql/sm.cpp Fix: * tools (linked with libmariadb.so) *must* include libmariadb's mysql.h * as a hack prevent service_thd_timezone.h from being loaded into tools, as it conflicts with libmariadb's mysql.h * server plugin *must* include server's mysql.h * also don't link every tool with libmariadb.so, link the helper library (liblibmysqlclient.so) that actually needs it, tools use this helper library, not libmariadb.so directly --- dbcon/joblist/CMakeLists.txt | 17 +++++++++-------- dbcon/mysql/sm.cpp | 3 +-- primitives/primproc/CMakeLists.txt | 2 +- utils/libmysql_client/CMakeLists.txt | 5 +++++ utils/querystats/CMakeLists.txt | 4 ++-- versioning/BRM/CMakeLists.txt | 1 - 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 0e1e998c8..134a11489 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -1,5 +1,13 @@ +include_directories(BEFORE + ${CMAKE_BINARY_DIR}/libmariadb/include + ${CMAKE_SOURCE_DIR}/libmariadb/include) +add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) + include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_SRC_DIR}/tools/passwd) + +########### next target ############### + set(joblist_LIB_SRCS anydatalist.cpp batchprimitiveprocessor-jl.cpp @@ -54,15 +62,8 @@ set(joblist_LIB_SRCS windowfunctionstep.cpp ${ENGINE_SRC_DIR}/tools/passwd/secrets.cpp) -########### next target ############### - -add_library(joblist_server STATIC ${joblist_LIB_SRCS}) - -########### next target ############### - -add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) - add_library(joblist SHARED ${joblist_LIB_SRCS}) +target_include_directories(joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) add_dependencies(joblist loggingcpp) install(TARGETS joblist DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/dbcon/mysql/sm.cpp b/dbcon/mysql/sm.cpp index e5b53d883..a7a3bfa0c 100644 --- a/dbcon/mysql/sm.cpp +++ b/dbcon/mysql/sm.cpp @@ -506,5 +506,4 @@ void cpsm_conhdl_t::write(ByteStream bs) #endif } -} // namespace sm - +} diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index bb0902027..cf7c6575d 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -32,4 +32,4 @@ add_dependencies(PrimProc loggingcpp) target_include_directories(PrimProc PRIVATE ${Boost_INCLUDE_DIRS}) target_link_libraries(PrimProc ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} threadpool cacheutils dbbc processor) -install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) \ No newline at end of file +install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/utils/libmysql_client/CMakeLists.txt b/utils/libmysql_client/CMakeLists.txt index e37be62a4..4f41b428b 100644 --- a/utils/libmysql_client/CMakeLists.txt +++ b/utils/libmysql_client/CMakeLists.txt @@ -3,6 +3,11 @@ include_directories(BEFORE ${CMAKE_SOURCE_DIR}/libmariadb/include) add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) +include_directories(BEFORE + ${CMAKE_BINARY_DIR}/libmariadb/include + ${CMAKE_SOURCE_DIR}/libmariadb/include) +add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) + include_directories( ${ENGINE_COMMON_INCLUDES} ) ########### next target ############### diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index 9c134280b..72291789b 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -1,6 +1,6 @@ include_directories(BEFORE - ${CMAKE_BINARY_DIR}/libmariadb/include - ${CMAKE_SOURCE_DIR}/libmariadb/include) + ${CMAKE_BINARY_DIR}/libmariadb/include + ${CMAKE_SOURCE_DIR}/libmariadb/include) add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) include_directories( ${ENGINE_COMMON_INCLUDES} ) diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 7e419cd97..b9745e3e2 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -113,4 +113,3 @@ add_executable(load_brm ${load_brm_SRCS}) target_link_libraries(load_brm ${ENGINE_LDFLAGS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) install(TARGETS load_brm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) - From 9935431b31e03b77835ff44815e5085ef7c71a7a Mon Sep 17 00:00:00 2001 From: "david.hall" Date: Thu, 17 Nov 2022 12:02:31 -0600 Subject: [PATCH 6/8] Remove blank lines --- storage-manager/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 75d4cff10..a874a93d4 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -105,28 +105,24 @@ add_executable(smcat src/smcat.cpp) target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} - ) add_executable(smput src/smput.cpp) target_link_libraries(smput storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} - ) add_executable(smls src/smls.cpp) target_link_libraries(smls storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} - ) add_executable(smrm src/smrm.cpp) target_link_libraries(smrm storagemanager cloudio ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} - ) install(TARGETS storagemanager From 6113a6e0e8c4247578cfea20e54053c26b203e8a Mon Sep 17 00:00:00 2001 From: "david.hall" Date: Wed, 30 Nov 2022 11:23:02 -0600 Subject: [PATCH 7/8] Serg Changes from 22.08.5 --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 88a598ab5..227790ec8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -192,7 +192,8 @@ ENDIF() IF (NOT INSTALL_LAYOUT) MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) - MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2 -DBOOST_BIND_GLOBAL_PLACEHOLDERS" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) + MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) + MY_CHECK_AND_SET_COMPILER_FLAG("-DBOOST_BIND_GLOBAL_PLACEHOLDERS" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG) # enable security hardening features, like most distributions do # in our benchmarks that costs about ~1% of performance, depending on the load From 7e3ad2443784cc966102aebea5c750253b877e7a Mon Sep 17 00:00:00 2001 From: "david.hall" Date: Wed, 30 Nov 2022 12:46:26 -0600 Subject: [PATCH 8/8] Serge changes -- Add static joblist lib --- dbcon/joblist/CMakeLists.txt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index 134a11489..6c52bd95c 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -1,11 +1,4 @@ -include_directories(BEFORE - ${CMAKE_BINARY_DIR}/libmariadb/include - ${CMAKE_SOURCE_DIR}/libmariadb/include) -add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) - include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_SRC_DIR}/tools/passwd) - - ########### next target ############### set(joblist_LIB_SRCS @@ -62,8 +55,19 @@ set(joblist_LIB_SRCS windowfunctionstep.cpp ${ENGINE_SRC_DIR}/tools/passwd/secrets.cpp) +########### next target ############### + +add_library(joblist_server STATIC ${joblist_LIB_SRCS}) +add_dependencies(joblist_server loggingcpp) + +########### next target ############### + +add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED) + add_library(joblist SHARED ${joblist_LIB_SRCS}) -target_include_directories(joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR}) +target_include_directories(joblist BEFORE PUBLIC ${OPENSSL_INCLUDE_DIR} + ${CMAKE_BINARY_DIR}/libmariadb/include + ${CMAKE_SOURCE_DIR}/libmariadb/include) add_dependencies(joblist loggingcpp) install(TARGETS joblist DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)