database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-08 14:22:09 +03:00
Code Activity

chore(ci): backport CI from stable to develop-23.02 (#3586)

Browse Source 
* core(ci): try to backport CI from stable to develop-23.02
* chore(build): no Werror
* chore(build): buildfiles from stable
* restore old boost
* no unused param warning
* Some final build fix
This commit is contained in:
Leonid Fedorov
2025-06-09 20:14:22 +04:00
committed by GitHub
parent 5c067ce489
commit 3800dfda38
33 changed files with 3560 additions and 1958 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1097
.drone.jsonnet
View File

File diff suppressed because it is too large Load Diff

503
CMakeLists.txt
View File

@@ -1,440 +1,87 @@
if (CMAKE_VERSION VERSION_LESS 3.12.0)
MESSAGE_ONCE(CS_CMAKE "ColumnStore needs cmake >= 3.12")
RETURN()
ENDIF()
cmake_minimum_required(VERSION 3.13)
CMAKE_MINIMUM_REQUIRED(VERSION 3.12)
project(Columnstore)
PROJECT(Columnstore)
INCLUDE (CheckCXXSourceCompiles)
# MariaDB server adds this flags by default for Debug builds, and this makes our external projects
# ABI incompatible with debug STL containers
STRING(REPLACE -D_GLIBCXX_DEBUG "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
STRING(REPLACE -D_GLIBCXX_ASSERTIONS "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
FOREACH(p CMP0135)
IF(POLICY ${p})
CMAKE_POLICY(SET ${p} NEW)
ENDIF()
ENDFOREACH()
IF(NOT INSTALL_LAYOUT)
IF(NOT CMAKE_BUILD_TYPE)
SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING
"Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE)
ENDIF(NOT CMAKE_BUILD_TYPE)
SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
ENDIF()
if(SERVER_BUILD_DIR)
if (NOT IS_ABSOLUTE ${SERVER_BUILD_DIR})
set(SERVER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SERVER_BUILD_DIR})
foreach(p CMP0135 CMP0077)
if(POLICY ${p})
cmake_policy(SET ${p} NEW)
endif()
if(NOT EXISTS ${SERVER_BUILD_DIR}/CMakeCache.txt)
message(FATAL_ERROR "SERVER_BUILD_DIR parameter supplied but CMakeCache.txt not found in ${SERVER_BUILD_DIR}")
endif()
load_cache("${SERVER_BUILD_DIR}" READ_WITH_PREFIX SERVER_ MySQL_SOURCE_DIR MySQL_BINARY_DIR CMAKE_BUILD_TYPE CMAKE_INSTALL_PREFIX)
endforeach()
set(SERVER_BUILD_INCLUDE_DIR "${SERVER_MySQL_BINARY_DIR}/include" CACHE PATH "Location of server build include folder" FORCE)
set(SERVER_SOURCE_ROOT_DIR "${SERVER_MySQL_SOURCE_DIR}" CACHE PATH "Location of the server source folder" FORCE)
set(CMAKE_INSTALL_PREFIX "${SERVER_CMAKE_INSTALL_PREFIX}" CACHE PATH "Installation prefix" FORCE)
set(CMAKE_BUILD_TYPE ${SERVER_CMAKE_BUILD_TYPE} CACHE STRING "Build configuration type" FORCE)
endif()
set(PACKAGE columnstore)
set(PACKAGE_NAME columnstore)
set(PACKAGE_TARNAME columnstore)
set(PACKAGE_BUGREPORT support@mariadb.com)
set(PACKAGE_URL "")
set(PACKAGE_STRING columnstore-${PACKAGE_VERSION})
SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external)
LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake)
set_property(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake)
INCLUDE(check_compiler_flag)
INCLUDE(ExternalProject)
INCLUDE(CheckCXXSourceCompiles)
set(CMAKE_CXX_EXTENSIONS FALSE)
set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
FIND_PACKAGE(Boost 1.84.0 COMPONENTS chrono filesystem program_options regex system thread)
# take a note, that install scripts are and services configurations are located oam/install_scripts/CMakeLists.txt
IF (NOT Boost_FOUND OR Boost_VERSION VERSION_GREATER 1.76.99)
MY_CHECK_CXX_COMPILER_FLAG("-std=c++20")
ENDIF()
set(WITH_COLUMNSTORE_LZ4
AUTO
CACHE STRING "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'"
)
IF (have_CXX__std_c__20)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20")
ELSE()
MY_CHECK_CXX_COMPILER_FLAG("-std=c++2a")
IF (have_CXX__std_c__2a)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
ELSE()
MESSAGE_ONCE(CS_NO_CXX20 "C++ Compiler does not understand -std=c++20")
return()
ENDIF()
ENDIF()
# order of includes is important
include(check_compiler_flag)
include(ExternalProject)
include(CheckCXXSourceCompiles)
include(packages)
include(boost)
include(thrift)
include(dirs)
include(includes)
include(libs)
include(ColumnstoreLibrary)
include(columnstore_version)
include(configureEngine)
include(compiler_flags)
include(misc)
# There is an inconsistency b/w default char signedness at ARM and x86.
# This flag explicitly sets char as signed.
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsigned-char")
set(COMPONENTS
dbcon/mysql
utils
oam/oamcpp
dbcon/execplan
dbcon/joblist
versioning
oam
writeengine/wrapper
writeengine/client
writeengine/xml
writeengine/redistribute
dbcon/ddlpackage
dbcon/ddlpackageproc
dbcon/dmlpackage
dbcon/dmlpackageproc
ddlproc
dmlproc
oamapps
primitives
tools
writeengine/server
writeengine/bulk
writeengine/splitter
storage-manager
datatypes
tests
)
UNSET(CMAKE_CXX_STANDARD)
SET(CMAKE_CXX_EXTENSIONS FALSE)
SET(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
SET(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obj)
SET(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
SET(WITH_COLUMNSTORE_LZ4 AUTO CACHE STRING "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'")
foreach(component ${COMPONENTS})
add_subdirectory(${component})
endforeach()
SET (ENGINE_SYSCONFDIR "/etc")
SET (ENGINE_DATADIR "/var/lib/columnstore")
SET (ENGINE_LOGDIR "/var/log/mariadb/columnstore")
IF (INSTALL_LAYOUT)
# We are building from MariaDB server submodule if this is set
SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR}")
IF (INSTALL_SYSCONF2DIR)
SET(MARIADB_MYCNFDIR "${INSTALL_SYSCONF2DIR}")
ELSE ()
SET(MARIADB_MYCNFDIR "/etc/mysql")
ENDIF ()
SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR}")
SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR}")
SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include")
SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR}")
SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR}")
SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore")
ELSE ()
IF (RPM)
SET(MARIADB_PLUGINDIR "/usr/lib64/mysql/plugin")
SET(MARIADB_MYCNFDIR "/etc/my.cnf.d")
SET (ENGINE_LIBDIR "/usr/lib64")
SET (ENGINE_BINDIR "/usr/bin")
SET (ENGINE_INCDIR "/usr/include")
SET (ENGINE_MANDIR "/usr/share/man")
SET (ENGINE_SBINDIR "/usr/sbin")
SET (ENGINE_SUPPORTDIR "/usr/share/columnstore")
ELSEIF (DEB)
SET(MARIADB_PLUGINDIR "/usr/lib/mysql/plugin")
SET(MARIADB_MYCNFDIR "/etc/mysql/conf.d")
SET (ENGINE_LIBDIR "/usr/lib")
SET (ENGINE_BINDIR "/usr/bin")
SET (ENGINE_INCDIR "/usr/include")
SET (ENGINE_MANDIR "/usr/share/man")
SET (ENGINE_SBINDIR "/usr/sbin")
SET (ENGINE_SUPPORTDIR "/usr/share/columnstore")
ELSE ()
# TODO: prefix should probably apply here
SET(MARIADB_PLUGINDIR "/usr/local/lib/mysql/plugin")
SET(MARIADB_MYCNFDIR "/etc/mysql")
SET (ENGINE_LIBDIR "/usr/local/lib")
SET (ENGINE_BINDIR "/usr/local/bin")
SET (ENGINE_INCDIR "/usr/local/include")
SET (ENGINE_MANDIR "/usr/local/man")
SET (ENGINE_SBINDIR "/usr/local/sbin")
SET (ENGINE_SUPPORTDIR "/usr/local/share/columnstore")
ENDIF ()
ENDIF ()
SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
INCLUDE(columnstore_version)
INCLUDE(misc)
INCLUDE(boost)
INCLUDE(thrift)
FIND_PACKAGE(BISON)
IF (NOT BISON_FOUND)
MESSAGE_ONCE(CS_NO_BISON "bison not found!")
return()
ENDIF()
check_cxx_source_compiles("#include <filesystem>\n void main(){}" HAS_STD_FILESYSTEM)
check_cxx_source_compiles("#include <experimental/filesystem>\n void main(){}" HAS_STD_EXPERIMENTAL_FILESYSTEM)
SET (PACKAGE columnstore)
SET (PACKAGE_NAME columnstore)
SET (PACKAGE_TARNAME columnstore)
SET (PACKAGE_BUGREPORT support@mariadb.com)
SET (PACKAGE_URL "")
SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION})
INCLUDE (configureEngine)
FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable")
if(NOT LEX_EXECUTABLE)
FIND_PROGRAM(LEX_EXECUTABLE lex DOC "path to the lex executable")
if(NOT LEX_EXECUTABLE)
MESSAGE_ONCE(CS_NO_LEX "flex/lex not found!")
return()
endif()
endif()
FIND_PACKAGE(LibXml2)
if (NOT LIBXML2_FOUND)
MESSAGE_ONCE(CS_NO_LIBXML "Could not find a usable libxml2 development environment!")
return()
endif()
find_package(Snappy)
if (NOT SNAPPY_FOUND)
MESSAGE_ONCE(CS_NO_SNAPPY "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian")
return()
endif()
FIND_PACKAGE(CURL)
if (NOT CURL_FOUND)
MESSAGE_ONCE(CS_NO_CURL "libcurl development headers not found")
return()
endif()
FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable")
if(NOT AWK_EXECUTABLE)
MESSAGE_ONCE(CS_NO_AWK "awk not found!")
return()
endif()
SET(HAVE_LZ4 0 CACHE INTERNAL "")
IF (WITH_COLUMNSTORE_LZ4 STREQUAL "ON" OR WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO")
FIND_PACKAGE(LZ4)
IF (NOT LZ4_FOUND)
IF (WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO")
MESSAGE_ONCE(CS_LZ4 "LZ4 not found, building without LZ4")
ELSE()
MESSAGE(FATAL_ERROR "LZ4 not found.")
ENDIF()
ELSE()
MESSAGE_ONCE(CS_LZ4 "Building with LZ4")
SET(HAVE_LZ4 1 CACHE INTERNAL "")
ENDIF()
ELSE()
MESSAGE_ONCE(CS_LZ4 "Building without LZ4")
ENDIF()
IF (NOT INSTALL_LAYOUT)
MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL)
MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG)
MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG)
MY_CHECK_AND_SET_COMPILER_FLAG("-DBOOST_BIND_GLOBAL_PLACEHOLDERS" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG)
# enable security hardening features, like most distributions do
# in our benchmarks that costs about ~1% of performance, depending on the load
IF(CMAKE_C_COMPILER_VERSION VERSION_LESS "4.6")
SET(security_default OFF)
ELSE()
SET(security_default ON)
ENDIF()
OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default})
OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF)
IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN AND NOT WITH_GPROF)
# security-enhancing flags
MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC")
MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now")
MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4")
MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO)
MY_CHECK_AND_SET_COMPILER_FLAG("-fexceptions")
IF(SECURITY_HARDENED_NEW)
MY_CHECK_AND_SET_COMPILER_FLAG("-mcet -fcf-protection")
MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector-strong")
MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-clash-protection")
ENDIF()
ENDIF()
ELSE ()
# Remove visibility flag for now as it breaks Ubuntu 18.05 and we need to
# fix our libraries anyway
STRING(REPLACE "-fvisibility=hidden" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
MY_CHECK_AND_SET_COMPILER_FLAG("-msse4.2" RELEASE RELWITHDEBINFO MINSIZEREL DEBUG)
MY_CHECK_AND_SET_COMPILER_FLAG("-D_DEBUG -O0" DEBUG)
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-vla" DEBUG)
ENDIF()
IF (MASK_LONGDOUBLE)
MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE")
ENDIF()
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-enum-enum-conversion -Wno-register -Wno-typedef-redefinition")
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register")
endif()
IF (WITH_COLUMNSTORE_ASAN)
MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO)
MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC")
ENDIF()
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-copy" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-declarations" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL)
MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall -Wextra")
SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed")
SET (ENGINE_DT_LIB datatypes)
SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt ${ENGINE_DT_LIB})
SET (ENGINE_OAM_LIBS oamcpp)
SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS})
SET (PLUGIN_EXEC_LIBS execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querytele threadpool ${ENGINE_BRM_LIBS})
SET (ENGINE_EXEC_LIBS joblist querystats libmysql_client ${PLUGIN_EXEC_LIBS})
SET (PLUGIN_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils)
SET (ENGINE_WRITE_LIBS ${PLUGIN_WRITE_LIBS} ${ENGINE_EXEC_LIBS})
IF (NOT INSTALL_LAYOUT)
SET (LIBMARIADB_BININC_DIR ${SERVER_BUILD_DIR}/libmariadb/include)
SET (LIBMARIADB_SRCINC_DIR ${SERVER_SOURCE_ROOT_DIR}/libmariadb/include)
ELSE()
SET (LIBMARIADB_BININC_DIR ${CMAKE_BINARY_DIR}/libmariadb/include)
SET (LIBMARIADB_SRCINC_DIR ${CMAKE_SOURCE_DIR}/libmariadb/include)
ENDIF()
SET (ENGINE_COMMON_LDFLAGS "")
IF (SERVER_BUILD_INCLUDE_DIR)
IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR})
SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR})
ENDIF()
ELSEIF (INSTALL_LAYOUT)
SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include)
ELSE()
SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include)
ENDIF()
IF (SERVER_SOURCE_ROOT_DIR)
IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR})
SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR})
ENDIF()
ELSEIF (INSTALL_LAYOUT)
SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR})
ELSE()
SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..)
ENDIF()
IF (NOT SERVER_BUILD_DIR)
SET (SERVER_BUILD_DIR ${SERVER_SOURCE_ROOT_DIR})
ENDIF()
MESSAGE_ONCE(SERVER_BUILD_INCLUDE_DIR "SERVER_BUILD_INCLUDE_DIR = ${SERVER_BUILD_INCLUDE_DIR}")
MESSAGE_ONCE(SERVER_SOURCE_ROOT_DIR "SERVER_SOURCE_ROOT_DIR = ${SERVER_SOURCE_ROOT_DIR}")
IF (INSTALL_LAYOUT)
SET (MARIADB_CLIENT_LIBS libmariadb)
ELSE()
SET (MARIADB_CLIENT_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so)
ENDIF()
IF (INSTALL_LAYOUT)
SET (MARIADB_STRING_LIBS dbug strings mysys)
ELSE()
SET (MARIADB_STRING_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../mysys/ libmysys.a -L${SERVER_BUILD_INCLUDE_DIR}/../strings/ libstrings.a -L${SERVER_BUILD_INCLUDE_DIR}/../dbug/ libdbug.a)
ENDIF()
#SET (ENGINE_UTILS_BOOSTIDB_INCLUDE "{CMAKE_CURRENT_SOURCE_DIR}/utils/boost_idb")
SET (ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp")
SET (ENGINE_WE_SHARED_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/shared")
SET (ENGINE_UTILS_IDBDATAFILE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/idbdatafile")
SET (ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp" "${CMAKE_CURRENT_BINARY_DIR}/utils/loggingcpp")
SET (ENGINE_UTILS_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/configcpp")
SET (ENGINE_UTILS_COMPRESS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/compress")
SET (ENGINE_VERSIONING_BRM_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/versioning/BRM")
SET (ENGINE_UTILS_ROWGROUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rowgroup")
SET (ENGINE_UTILS_COMMON_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/common")
SET (ENGINE_UTILS_DATACONVERT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/dataconvert")
SET (ENGINE_UTILS_RWLOCK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rwlock")
SET (ENGINE_UTILS_FUNCEXP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/funcexp")
SET (ENGINE_UTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils")
SET (ENGINE_OAM_OAMCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/oam/oamcpp")
SET (ENGINE_DBCON_DDLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackageproc")
SET (ENGINE_DBCON_DDLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackage")
SET (ENGINE_DBCON_EXECPLAN_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/execplan")
SET (ENGINE_UTILS_STARTUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/startup")
SET (ENGINE_DBCON_JOBLIST_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/joblist")
SET (ENGINE_WE_WRAPPER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/wrapper")
SET (ENGINE_WE_SERVER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/server")
SET (ENGINE_DBCON_DMLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackage")
SET (ENGINE_WE_CLIENT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/client")
SET (ENGINE_DBCON_DMLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackageproc")
SET (ENGINE_UTILS_CACHEUTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/cacheutils")
SET (ENGINE_UTILS_MYSQLCL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/mysqlcl_idb")
SET (ENGINE_UTILS_QUERYTELE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querytele")
SET (ENGINE_UTILS_JOINER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/joiner")
SET (ENGINE_UTILS_THREADPOOL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/threadpool")
SET (ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchloader")
SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup")
SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats")
SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client")
SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml")
SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes")
SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache")
SET (ENGINE_PRIMPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/primproc")
SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql")
SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include")
IF (PCRE_INCLUDES)
SET (ENGINE_SERVER_PCRE_INCLUDE "${PCRE_INCLUDES}")
ELSE ()
SET (ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre")
ENDIF ()
SET (ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include")
SET (ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/")
SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk")
SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR})
SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE})
ADD_SUBDIRECTORY(dbcon/mysql)
IF(NOT TARGET columnstore)
RETURN()
ENDIF()
ADD_SUBDIRECTORY(utils)
ADD_SUBDIRECTORY(oam/oamcpp)
ADD_SUBDIRECTORY(dbcon/execplan)
ADD_SUBDIRECTORY(dbcon/joblist)
ADD_SUBDIRECTORY(versioning)
ADD_SUBDIRECTORY(oam)
ADD_SUBDIRECTORY(writeengine/wrapper)
ADD_SUBDIRECTORY(writeengine/client)
ADD_SUBDIRECTORY(writeengine/xml)
ADD_SUBDIRECTORY(writeengine/redistribute)
ADD_SUBDIRECTORY(dbcon/ddlpackage)
ADD_SUBDIRECTORY(dbcon/ddlpackageproc)
ADD_SUBDIRECTORY(dbcon/dmlpackage)
ADD_SUBDIRECTORY(dbcon/dmlpackageproc)
ADD_SUBDIRECTORY(ddlproc)
ADD_SUBDIRECTORY(dmlproc)
ADD_SUBDIRECTORY(oamapps)
ADD_SUBDIRECTORY(primitives)
ADD_SUBDIRECTORY(tools)
ADD_SUBDIRECTORY(writeengine/server)
ADD_SUBDIRECTORY(writeengine/bulk)
ADD_SUBDIRECTORY(writeengine/splitter)
ADD_SUBDIRECTORY(storage-manager)
ADD_SUBDIRECTORY(datatypes)
ADD_SUBDIRECTORY(tests)
find_package(Git QUIET)
IF (GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git)
exec_program("git"
${CMAKE_CURRENT_SOURCE_DIR}
ARGS "describe --match=NeVeRmAtCh --always --dirty"
OUTPUT_VARIABLE GIT_VERSION)
ELSE ()
SET(GIT_VERSION "source")
ENDIF ()
# releasenum is used by external scripts for various tasks. Leave it alone.
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine)
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h)
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h)
CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE)
INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine)
IF (INSTALL_LAYOUT)
# Do this or when MariaDB builds us we don't have GenError which is required for these
ADD_DEPENDENCIES(udf_mysql GenError)
ADD_DEPENDENCIES(funcexp GenError)
ADD_DEPENDENCIES(oamcpp GenError)
ADD_DEPENDENCIES(regr_mysql GenError)
ELSE ()
# MariaDB has its own packaging routines
INCLUDE(cpackEngineRPM)
INCLUDE(cpackEngineDEB)
ENDIF ()
add_dependencies(udf_mysql GenError)
add_dependencies(funcexp GenError)
add_dependencies(oamcpp GenError)
add_dependencies(regr_mysql GenError)

689
build/bootstrap_mcs.sh
View File

@@ -4,153 +4,171 @@
# - the server's source code is two directories above the MCS engine source.
# - the script is to be run under root.
set -o pipefail
export CLICOLOR_FORCE=1 #cmake output
INSTALL_PREFIX="/usr/"
DATA_DIR="/var/lib/mysql/data"
CMAKE_BIN_NAME=cmake
CTEST_BIN_NAME=ctest
RPM_CONFIG_DIR="/etc/my.cnf.d"
DEB_CONFIG_DIR="/etc/mysql/mariadb.conf.d"
CONFIG_DIR=$RPM_CONFIG_DIR
SCRIPT_LOCATION=$(dirname "$0")
MDB_SOURCE_PATH=$(realpath $SCRIPT_LOCATION/../../../..)
MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
source $SCRIPT_LOCATION/utils.sh
if [ "$EUID" -ne 0 ]
then error "Please run script as root to install MariaDb to system paths"
exit 1
fi
message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky")
DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo))
cd - > /dev/null
GCC_VERSION="11"
MDB_CMAKE_FLAGS=()
source "$SCRIPT_LOCATION"/utils.sh
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
echo "Arguments received: $@"
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH
optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true
optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
optparse.define short=d long=distro desc="Choose your OS: ${DISTRO_OPTIONS[*]}" variable=OS
optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true
optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true
optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH
optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true
optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core"
optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true
optparse.define short=F long=custom-cmake-flags desc="Add custom cmake flags" variable=CUSTOM_CMAKE_FLAGS
optparse.define short=f long=do-not-freeze-revision desc="Disable revision freezing, or do not set 'update none' for columnstore submodule in MDB repository" variable=DO_NOT_FREEZE_REVISION default=false value=true
optparse.define short=g long=alien desc="Turn off maintainer mode (ex. -Werror)" variable=MAINTAINER_MODE default=true value=false
optparse.define short=G long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true
optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true
optparse.define short=n long=no-clean-install desc="Do not perform a clean install (keep existing db files)" variable=NO_CLEAN default=false value=true
optparse.define short=j long=parallel desc="Number of paralles for build" variable=CPUS default=$(getconf _NPROCESSORS_ONLN)
optparse.define short=F long=show-build-flags desc="Print CMake flags, while build" variable=PRINT_CMAKE_FLAGS default=false
optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true
optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true
optparse.define short=n long=no-clean-install desc="Do not perform a clean install (keep existing db files)" variable=NO_CLEAN default=false value=true
optparse.define short=o long=recompile-only variable=RECOMPILE_ONLY default=false value=true
optparse.define short=O long=static desc="Build all with static libraries" variable=STATIC_BUILD default=false value=true
optparse.define short=p long=build-packages desc="Build packages" variable=BUILD_PACKAGES default=false value=true
optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core"
optparse.define short=r long=restart-services variable=RESTART_SERVICES default=true value=false
optparse.define short=s long=sccache desc="Build with sccache" variable=SCCACHE default=false value=true
optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true
optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE
optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true
optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true
optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
source $(optparse.build)
message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
MCS_BUILD_TYPE=$selectedChoice
fi
if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " || $OS = "CentOS" ]]; then
if [[ ! " ${DISTRO_OPTIONS[*]} " =~ " ${OS} " ]]; then
echo "OS is empty, trying to detect..."
detect_distro
fi
INSTALL_PREFIX="/usr/"
DATA_DIR="/var/lib/mysql/data"
CMAKE_BIN_NAME=cmake
CTEST_BIN_NAME=ctest
CONFIG_DIR="/etc/my.cnf.d"
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
CONFIG_DIR="/etc/mysql/mariadb.conf.d"
pkg_format="deb"
if [[ "$OS" == *"rocky"* ]]; then
pkg_format="rpm"
fi
select_branch()
{
cd $SCRIPT_LOCATION
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then
if [[ $BRANCH = "" ]]; then
getChoice -q "Select your branch" -o BRANCHES
BRANCH=$selectedChoice
install_deps() {
if [[ $INSTALL_DEPS = false ]]; then
return
fi
if [[ $BRANCH != $CURRENT_BRANCH ]]; then
message "Selecting $BRANCH branch for Columnstore"
git checkout $BRANCH
message_split
prereq=""
RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
graphviz devscripts ccache equivs eatmydata curl"
if [[ "$OS" == *"rockylinux:8"* || "$OS" == *"rocky:8"* ]]; then
command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
${RPM_BUILD_DEPS}"
elif
[[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]
then
command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
command="${DEB_BUILD_DEPS}"
else
echo "Unsupported OS: $OS"
exit 17
fi
message "Turning off Columnstore submodule auto update via gitconfig"
if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
if [ -f /.dockerenv ]; then
change_ubuntu_mirror us
fi
command="${command} lto-disabled-list"
fi
eval "$prereq"
message "Installing dependencies for $OS"
retry_eval 5 "$command"
}
install_deps
cd $COLUMSNTORE_SOURCE_PATH
COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
cd $MDB_SOURCE_PATH
MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD)
message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch"
cd - >/dev/null
if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}"
fi
disable_git_restore_frozen_revision() {
cd $MDB_SOURCE_PATH
git config submodule.storage/columnstore/columnstore.update none
cd - >/dev/null
fi
cd - > /dev/null
message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch"
}
install_deps()
{
message_split
message "Installing deps"
if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then
apt-get -y update
apt-get -y install build-essential automake libboost-all-dev bison cmake \
libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev \
libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev git \
libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest libsnappy-dev libjemalloc-dev \
liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev graphviz
DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"
elif [[ $OS = 'CentOS' || $OS = 'Rocky' || $OS = 'Fedora' ]]; then
if [[ "$OS_VERSION" == "7" ]]; then
yum -y install cmake3 epel-release centos-release-scl
CMAKE_BIN_NAME=cmake3
CTEST_BIN_NAME=ctest3
else
yum -y install cmake
fi
if [ $OS = 'Rocky' ]; then
yum -y groupinstall "Development Tools" && yum config-manager --set-enabled powertools
yum install -y checkpolicy
fi
if [[ $OS != 'Fedora' ]]; then
yum -y install epel-release
fi
yum install -y bison ncurses-devel readline-devel perl-devel openssl-devel libxml2-devel gperf libaio-devel libevent-devel tree wget pam-devel snappy-devel libicu \
vim wget strace ltrace gdb rsyslog net-tools openssh-server expect boost perl-DBI libicu boost-devel initscripts \
jemalloc-devel libcurl-devel gtest-devel cppunit-devel systemd-devel lzo-devel xz-devel lz4-devel bzip2-devel \
pcre2-devel flex graphviz libaio-devel openssl-devel flex
else
error "Unsupported OS $OS"
exit 17
fi
}
stop_service()
{
stop_service() {
message_split
message "Stopping MariaDB services"
systemctl stop mariadb
systemctl stop mariadb-columnstore
systemctl stop mcs-storagemanager
}
check_service()
{
if systemctl is-active --quiet $1; then
check_service() {
if systemctl is-active --quiet "$1"; then
message "$1 $color_normal[$color_green OK $color_normal]"
else
message "$1 $color_normal[$color_red Fail $color_normal]"
service $1 status
service "$1" status
fi
}
start_service()
{
start_service() {
message_split
message "Starting MariaDB services"
systemctl start mariadb-columnstore
@@ -166,150 +184,235 @@ start_service()
check_service mcs-writeengineserver
}
clean_old_installation()
{
start_storage_manager_if_needed() {
if [[ $CLOUD_STORAGE_ENABLED = true ]]; then
export MCS_USE_S3_STORAGE=1
message_split
message "Starting Storage Manager service"
systemctl start mcs-storagemanager
check_service mcs-storagemanager
fi
}
clean_old_installation() {
message_split
message "Cleaning old installation"
rm -rf /var/lib/columnstore/data1/*
rm -rf /var/lib/columnstore/data/
rm -rf /var/lib/columnstore/local/
rm -f /var/lib/columnstore/storagemanager/storagemanager-lock
rm -f /var/lib/columnstore/storagemanager/cs-initialized
rm -rf /var/lib/columnstore/storagemanager/*
rm -rf /var/log/mariadb/columnstore/*
rm -rf /tmp/*
rm -rf $REPORT_PATH
rm -rf "$REPORT_PATH"
rm -rf /var/lib/mysql
rm -rf /var/run/mysqld
rm -rf $DATA_DIR
rm -rf /etc/mysql
rm -rf $CONFIG_DIR
}
build()
{
message_split
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_normal mode"
modify_packaging() {
echo "Modifying_packaging..."
cd $MDB_SOURCE_PATH
local MDB_CMAKE_FLAGS="-DWITH_SYSTEMD=yes
-DPLUGIN_COLUMNSTORE=YES
-DPLUGIN_MROONGA=NO
-DPLUGIN_ROCKSDB=NO
-DPLUGIN_TOKUDB=NO
-DPLUGIN_CONNECT=NO
-DPLUGIN_SPIDER=NO
-DPLUGIN_OQGRAPH=NO
-DPLUGIN_SPHINX=NO
-DWITH_EMBEDDED_SERVER=OFF
if [[ $pkg_format == "deb" ]]; then
sed -i 's|.*-d storage/columnstore.*|elif [[ -d storage/columnstore/columnstore/debian ]]|' debian/autobake-deb.sh
fi
#disable LTO for 22.04 for now
if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list
done &&
grep mariadb /usr/share/lto-disabled-list/lto-disabled-list
fi
if [[ $pkg_format == "deb" ]]; then
apt-cache madison liburing-dev | grep liburing-dev || {
sed 's/liburing-dev/libaio-dev/g' -i debian/control &&
sed '/-DIGNORE_AIO_CHECK=YES/d' -i debian/rules &&
sed '/-DWITH_URING=yes/d' -i debian/rules
}
apt-cache madison libpmem-dev | grep 'libpmem-dev' || {
sed '/libpmem-dev/d' -i debian/control &&
sed '/-DWITH_PMEM/d' -i debian/rules
}
sed '/libfmt-dev/d' -i debian/control
# Remove Debian build flags that could prevent ColumnStore from building
sed -i '/-DPLUGIN_COLUMNSTORE=NO/d' debian/rules
# Disable dh_missing strict check for missing files
sed -i 's/--fail-missing/--list-missing/' debian/rules
# Tweak debian packaging stuff
for i in mariadb-plugin libmariadbd; do
sed -i "/Package: $i.*/,/^$/d" debian/control
done
sed -i 's/Depends: galera.*/Depends:/' debian/control
for i in galera wsrep ha_sphinx embedded; do
sed -i "/$i/d" debian/*.install
done
fi
}
construct_cmake_flags() {
MDB_CMAKE_FLAGS=(
-DBUILD_CONFIG=mysql_release
-DWITH_WSREP=OFF
-DWITH_SSL=system
-DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX
-DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE
-DCMAKE_EXPORT_COMPILE_COMMANDS=1
"
-DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX
-DMYSQL_MAINTAINER_MODE=NO
-DPLUGIN_COLUMNSTORE=YES
-DPLUGIN_CONNECT=NO
-DPLUGIN_GSSAPI=NO
-DPLUGIN_MROONGA=NO
-DPLUGIN_OQGRAPH=NO
-DPLUGIN_ROCKSDB=NO
-DPLUGIN_SPHINX=NO
-DPLUGIN_SPIDER=NO
-DPLUGIN_TOKUDB=NO
-DWITH_EMBEDDED_SERVER=NO
-DWITH_SSL=system
-DWITH_SYSTEMD=yes
-DWITH_WSREP=NO
)
if [[ $MAINTAINER_MODE = true ]]; then
MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
message "Columnstore maintainer mode on"
else
warn "Maintainer mode is disabled, be careful, alien"
fi
if [[ $SKIP_UNIT_TESTS = true ]]; then
warn "Unittests are not build"
else
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UNITTESTS=YES"
MDB_CMAKE_FLAGS+=(-DWITH_UNITTESTS=YES)
message "Buiding with unittests"
fi
if [[ $DRAW_DEPS = true ]]; then
warn "Generating dependendies graph to mariadb.dot"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} --graphviz=mariadb.dot"
MDB_CMAKE_FLAGS+=(--graphviz=$DEP_GRAPH_PATH)
fi
if [[ $USE_NINJA = true ]]; then
warn "Using Ninja instead of Makefiles"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja"
MDB_CMAKE_FLAGS+=(-GNinja)
fi
if [[ $STATIC_BUILD = true ]]; then
warn "Building all with static linkage"
MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_STATIC_LIBRARIES:BOOL=ON)
fi
if [[ $ASAN = true ]]; then
warn "Building with Address Sanitizer "
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
MDB_CMAKE_FLAGS+=(-DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH})
fi
if [[ $TSAN = true ]]; then
warn "Building with Thread Sanitizer"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
MDB_CMAKE_FLAGS+=(-DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH})
message "Setting vm.mmap_rnd_bits=30 for TSAN support"
sysctl vm.mmap_rnd_bits=30
fi
if [[ $UBSAN = true ]]; then
warn "Building with UB Sanitizer"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
MDB_CMAKE_FLAGS+=(-DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH})
fi
if [[ $WITHOUT_COREDUMPS = true ]]; then
warn "Cores are not dumped"
else
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
warn Building with CoreDumps: /proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p
warn "Building with CoreDumps"
MDB_CMAKE_FLAGS+=(-DWITH_COREDUMPS=ON)
if [ -f /.dockerenv ]; then
warn "Build is executed in Docker, core dumps saving path /proc/sys/kernel/core_pattern will not be configured!"
else
warn "/proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p"
echo "${REPORT_PATH}/core_%e.%p" >/proc/sys/kernel/core_pattern
fi
fi
if [[ $MAKEFILE_VERBOSE = true ]]; then
warn "Verbosing Makefile Commands"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
MDB_CMAKE_FLAGS+=(-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON)
fi
if [[ $SCCACHE = true ]]; then
warn "Use sccache"
MDB_CMAKE_FLAGS+=(-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache)
fi
if [[ $RUN_BENCHMARKS = true ]]; then
if [[ $MCS_BUILD_TYPE = 'Debug' ]]; then
error "Benchmarks will not be build in run in Debug build Mode"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
$RUN_BENCHMARKS = false
elif [[ $OS != 'Ubuntu' && $OS != 'Debian' ]] ; then
MDB_CMAKE_FLAGS+=(-DWITH_MICROBENCHMARKS=NO)
RUN_BENCHMARKS=false
elif [[ $OS != *"ubuntu"* && $OS != *"debian"* ]]; then
error "Benchmarks are now avaliable only at Ubuntu or Debian"
MAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
$RUN_BENCHMARKS = false
MAKE_FLAGS="${MDB_CMAKE_FLAGS[@]} -DWITH_MICROBENCHMARKS=NO"
RUN_BENCHMARKS=false
else
message "Compile with microbenchmarks"
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=YES"
MDB_CMAKE_FLAGS+=(-DWITH_MICROBENCHMARKS=YES)
fi
else
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_MICROBENCHMARKS=NO"
MDB_CMAKE_FLAGS+=(-DWITH_MICROBENCHMARKS=NO)
message "Buiding without microbenchmarks"
fi
cd $MDB_SOURCE_PATH
if [[ "$OS" == *"rocky"* ]]; then
OS_VERSION=${OS//[^0-9]/}
MDB_CMAKE_FLAGS+=(-DRPM=rockylinux${OS_VERSION})
elif [[ "$OS" == "debian:11" ]]; then
CODENAME="bullseye"
elif [[ "$OS" == "debian:12" ]]; then
CODENAME="bookworm"
elif [[ "$OS" == "ubuntu:20.04" ]]; then
CODENAME="focal"
elif [[ "$OS" == "ubuntu:22.04" ]]; then
CODENAME="jammy"
elif [[ "$OS" == "ubuntu:24.04" ]]; then
CODENAME="noble"
else
echo "Unsupported OS: $OS"
exit 17
fi
if [[ -n "$CODENAME" ]]; then
MDB_CMAKE_FLAGS+=(-DDEB=${CODENAME})
fi
if [[ "$NO_CLEAN" == true ]]; then
message "Skipping Columnstore.etc config installation"
MDB_CMAKE_FLAGS+=(-DSKIP_CONFIG_INSTALLATION=ON)
fi
MDB_CMAKE_FLAGS+=($CUSTOM_CMAKE_FLAGS)
message "Building with flags"
newline_array "${MDB_CMAKE_FLAGS[@]}"
}
init_submodules() {
if [[ $SKIP_SUBMODULES = true ]]; then
warn "Skipping initialization of columnstore submodules"
else
message "Initialization of columnstore submodules"
cd storage/columnstore/columnstore
git submodule update --init
cd $MDB_SOURCE_PATH
git submodule update --init --recursive
cd - >/dev/null
fi
}
if [[ $FORCE_CMAKE_CONFIG = true ]] ; then
warn "Erasing cmake cache"
rm -f "$MDB_SOURCE_PATH/CMakeCache.txt"
rm -rf "$MDB_SOURCE_PATH/CMakeFiles"
fi
if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DDEB=bionic"
elif [ $OS = 'CentOS' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7"
elif [ $OS = 'Rocky' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=CentOS7"
elif [ $OS = 'openSUSE' ]; then
MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DRPM=sles15"
fi
if [[ $PRINT_CMAKE_FLAGS = true ]] ; then
message "Building with flags"
newline_array ${MDB_CMAKE_FLAGS[@]}
fi
message "Configuring cmake silently"
${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS . | spinner
message_split
${CMAKE_BIN_NAME} --build . -j $CPUS && \
message "Installing silently" &&
${CMAKE_BIN_NAME} --install . | spinner 30
check_errorcode() {
if [ $? -ne 0 ]; then
message_split
error "!!!! BUILD FAILED !!!!"
@@ -319,59 +422,124 @@ build()
cd - >/dev/null
}
check_user_and_group()
{
if [ -z "$(grep mysql /etc/passwd)" ]; then
message "Adding user mysql into /etc/passwd"
useradd -r -U mysql -d /var/lib/mysql
fi
if [ -z "$(grep mysql /etc/group)" ]; then
GroupID = `awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group`
message "Adding group mysql with id $GroupID"
groupadd -g GroupID mysql
generate_svgs() {
if [[ $DRAW_DEPS = true ]]; then
message_split
warn "Generating svgs with dependency graph to $DEP_GRAPH_PATH"
for f in $(ls "$DEP_GRAPH_PATH".* | grep -v ".svg"); do
dot -Tsvg -o "$f.svg" "$f"
done
fi
}
run_unit_tests()
{
build_package() {
cd $MDB_SOURCE_PATH
if [[ $pkg_format == "rpm" ]]; then
command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package"
else
export DEBIAN_FRONTEND="noninteractive"
export DEB_BUILD_OPTIONS="parallel=$(nproc)"
export DH_BUILD_DDEBS="1"
export BUILDPACKAGE_FLAGS="-b"
command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh"
fi
echo "Building a package for $OS"
echo "Build command: $command"
eval "$command"
check_errorcode
}
check_debian_install_file() {
message "checking debian/mariadb-plugin-columnstore.install"
message_split
python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \
${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \
$MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated
}
build_binary() {
MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH")
message_split
message "Building sources in $color_yellow$MCS_BUILD_TYPE$color_cyan mode"
message "Compiled artifacts will be written to $color_yellow$MARIA_BUILD_PATH$color_cyan"
mkdir -p "$MARIA_BUILD_PATH"
cd $MDB_SOURCE_PATH
if [[ $FORCE_CMAKE_CONFIG = true ]]; then
warn "Erasing cmake cache"
rm -f "$MARIA_BUILD_PATH/CMakeCache.txt"
rm -rf "$MARIA_BUILD_PATH/CMakeFiles"
fi
message "Configuring cmake silently"
${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
message_split
check_debian_install_file
generate_svgs
${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
message "Installing silently" &&
${CMAKE_BIN_NAME} --install "$MARIA_BUILD_PATH" | spinner 30
check_errorcode
message "Adding symbol link to compile_commands.json to the source root"
ln -sf "$MARIA_BUILD_PATH/compile_commands.json" "$MDB_SOURCE_PATH"
}
check_user_and_group() {
user=$1
if [ -z "$(grep "$user" /etc/passwd)" ]; then
message "Adding user $user into /etc/passwd"
useradd -r -U "$user" -d /var/lib/mysql
fi
if [ -z "$(grep "$user" /etc/group)" ]; then
GroupID=$(awk -F: '{uid[$3]=1}END{for(x=100; x<=999; x++) {if(uid[x] != ""){}else{print x; exit;}}}' /etc/group)
message "Adding group $user with id $GroupID"
groupadd -g "$GroupID" "$user"
fi
}
run_unit_tests() {
message_split
if [[ $SKIP_UNIT_TESTS = true ]]; then
warn "Skipping unittests"
else
message "Running unittests"
cd $MARIA_BUILD_PATH
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress
${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress --output-on-failure
cd - >/dev/null
fi
}
run_microbenchmarks_tests()
{
run_microbenchmarks_tests() {
message_split
if [[ $RUN_BENCHMARKS = false ]]; then
warn "Skipping microbenchmarks"
else
message "Runnning microbenchmarks"
cd $MDB_SOURCE_PATH
cd $MARIA_BUILD_PATH
${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress
cd - >/dev/null
fi
}
disable_plugins_for_bootstrap()
{
disable_plugins_for_bootstrap() {
find /etc -type f -exec sed -i 's/plugin-load-add=auth_gssapi.so//g' {} +
find /etc -type f -exec sed -i 's/plugin-load-add=ha_columnstore.so//g' {} +
}
enable_columnstore_back()
{
echo plugin-load-add=ha_columnstore.so >> $CONFIG_DIR/columnstore.cnf
enable_columnstore_back() {
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/dbcon/mysql/columnstore.cnf $CONFIG_DIR
}
fix_config_files()
{
fix_config_files() {
message Fixing config files
THREAD_STACK_SIZE="20M"
@@ -413,99 +581,102 @@ fix_config_files()
message "UBSAN options were added to $MDB_SERVICE_FILE"
fi
fi
if [[ $RECOMPILE_ONLY = false ]]; then
message Reloading systemd
systemctl daemon-reload
fi
}
install()
{
make_dir() {
mkdir -p "$1"
chown mysql:mysql "$1"
}
install() {
if [[ $RECOMPILE_ONLY = false ]]; then
if [ "$EUID" -ne 0 ]; then
error "Please run script as root to install MariaDb to system paths"
exit 1
fi
message_split
message "Installing MariaDB"
disable_plugins_for_bootstrap
mkdir -p $REPORT_PATH
chmod 777 $REPORT_PATH
make_dir "$REPORT_PATH"
chmod 777 "$REPORT_PATH"
check_user_and_group
check_user_and_group mysql
check_user_and_group syslog
mkdir -p /etc/my.cnf.d
make_dir $CONFIG_DIR
bash -c 'echo "[client-server]
socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
echo "[client-server]
socket=/run/mysqld/mysqld.sock" >$CONFIG_DIR/socket.cnf
mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so
mkdir -p /var/lib/mysql
chown mysql:mysql /var/lib/mysql
make_dir /var/lib/mysql
message "Running mysql_install_db"
sudo -u mysql mysql_install_db --rpm --user=mysql >/dev/null
mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so
enable_columnstore_back
mkdir -p /etc/columnstore
make_dir /etc/columnstore
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
if [[ "$NO_CLEAN" == false ]]; then
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
fi
cp $MDB_SOURCE_PATH/support-files/*.service /lib/systemd/system/
cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
if [[ "$OS" = 'Ubuntu' || "$OS" = 'Debian' ]]; then
mkdir -p /usr/share/mysql
mkdir -p /etc/mysql/
cp $MDB_SOURCE_PATH/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
cp $MDB_SOURCE_PATH/debian/additions/debian-start /etc/mysql/debian-start
if [[ "$OS" = *"ubuntu"* || "$OS" = *"debian"* ]]; then
make_dir /usr/share/mysql
make_dir /etc/mysql/
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start.inc.sh /usr/share/mysql/debian-start.inc.sh
cp "$MDB_SOURCE_PATH"/debian/additions/debian-start /etc/mysql/debian-start
>/etc/mysql/debian.cnf
fi
fix_config_files
if [ -d "/etc/mysql/mariadb.conf.d/" ]; then
message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d"
cp -rp /etc/mysql/mariadb.conf.d/* /etc/my.cnf.d
if [ -d "$DEBCONFIG_DIR" ]; then
message "Copying configs from $DEBCONFIG_DIR to $CONFIG_DIR"
cp -rp "$DEBCONFIG_DIR"/* "$CONFIG_DIR"
fi
if [ -d "/etc/mysql/conf.d/" ]; then
message "Copying configs from /etc/mysql/conf.d/ to /etc/my.cnf.d"
cp -rp /etc/mysql/conf.d/* /etc/my.cnf.d
fi
mkdir -p /var/lib/columnstore/data1
mkdir -p /var/lib/columnstore/data1/systemFiles
mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm
mkdir -p /run/mysqld/
mkdir -p $DATA_DIR
chown -R mysql:mysql $DATA_DIR
chown -R mysql:mysql /var/lib/columnstore/
chown -R mysql:mysql /run/mysqld/
make_dir /var/lib/columnstore/data1
make_dir /var/lib/columnstore/data1/systemFiles
make_dir /var/lib/columnstore/data1/systemFiles/dbrm
make_dir /run/mysqld/
make_dir $DATA_DIR
chmod +x $INSTALL_PREFIX/bin/mariadb*
ldconfig
start_storage_manager_if_needed
message "Running columnstore-post-install"
mkdir -p /var/lib/columnstore/local
make_dir /var/lib/columnstore/local
columnstore-post-install --rpmmode=install
message "Running install_mcs_mysql"
install_mcs_mysql.sh
fi
chown -R syslog:syslog /var/log/mariadb/
chmod 777 /var/log/mariadb/
chmod 777 /var/log/mariadb/columnstore
}
smoke()
{
smoke() {
if [[ $SKIP_SMOKE = false ]]; then
message_split
message "Creating test database"
mariadb -e "create database if not exists test;"
message "Selecting magic numbers"
MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql`
MAGIC=$(mysql -N test <"$MDB_SOURCE_PATH"/storage/columnstore/columnstore/tests/scripts/smoke.sql)
if [[ $MAGIC == '42' ]]; then
message "Great answer correct!"
else
@@ -514,36 +685,30 @@ smoke()
fi
}
generate_svgs()
{
if [[ $DRAW_DEPS = true ]] ; then
message_split
warn "Generating svgs with dependency graph to $REPORT_PATH"
for f in $MDB_SOURCE_PATH/mariadb.dot.*;
do dot -Tsvg -o $REPORT_PATH/`basename $f`.svg $f;
done
fi
}
select_branch
if [[ $INSTALL_DEPS = true ]] ; then
install_deps
if [[ $DO_NOT_FREEZE_REVISION = false ]]; then
disable_git_restore_frozen_revision
fi
construct_cmake_flags
init_submodules
if [[ $BUILD_PACKAGES = false ]]; then
stop_service
if [[ $NO_CLEAN = false ]]; then
clean_old_installation
fi
build
build_binary
run_unit_tests
run_microbenchmarks_tests
install
if [[ $RESTART_SERVICES = true ]]; then
start_service
smoke
generate_svgs
fi
else
modify_packaging
build_package
fi
message_splitted "FINISHED"

90
build/build_cmapi.sh Executable file
View File

@@ -0,0 +1,90 @@
#!/bin/bash
# This script installs dependencies and builds cmapi package
# Should be executed by root
# Should reside in build/ (or storage/columnstore/columnstore/build), as all the paths a relative to script's location
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
source "$SCRIPT_LOCATION"/utils.sh
optparse.define short=d long=distro desc="distro" variable=OS
optparse.define short=a long=arch desc="architecture" variable=ARCH
source $(optparse.build)
echo "Arguments received: $@"
if [ "$EUID" -ne 0 ]; then
error "Please run script as root"
exit 1
fi
if [[ -z "${OS:-}" || -z "${ARCH:-}" ]]; then
echo "Please provide provide --distro and --arch parameters, e.g. ./build_cmapi.sh --distro ubuntu:22.04 --arch amd64"
exit 1
fi
pkg_format="deb"
if [[ "$OS" == *"rocky"* ]]; then
pkg_format="rpm"
fi
if [[ "$ARCH" == "arm64" ]]; then
export CC=gcc #TODO: what it is for?
fi
on_exit() {
if [[ $? -eq 0 ]]; then
echo "Cmapi package has been build successfully."
else
echo "Cmapi package build failed!"
fi
}
trap on_exit EXIT
install_deps() {
echo "Installing dependencies..."
cd "$COLUMNSTORE_SOURCE_PATH"/cmapi
if [[ "$OS" == "rockylinux:9" ]]; then
retry_eval 5 "dnf install -q -y libxcrypt-compat yum-utils"
retry_eval 5 "dnf config-manager --set-enabled devel && dnf update -q -y" #to make redhat-lsb-core available for rocky 9
fi
if [[ "$pkg_format" == "rpm" ]]; then
retry_eval 5 "dnf update -q -y && dnf install -q -y epel-release wget zstd findutils gcc cmake make rpm-build redhat-lsb-core libarchive"
else
retry_eval 5 "apt-get update -qq -o Dpkg::Use-Pty=0 && apt-get install -qq -o Dpkg::Use-Pty=0 wget zstd findutils gcc cmake make dpkg-dev lsb-release"
fi
if [ "$ARCH" == "amd64" ]; then
PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-x86_64_v2-unknown-linux-gnu-pgo+lto-full.tar.zst"
elif [ "$ARCH" == "arm64" ]; then
PYTHON_URL="https://github.com/indygreg/python-build-standalone/releases/download/20220802/cpython-3.9.13+20220802-aarch64-unknown-linux-gnu-noopt-full.tar.zst"
else
echo "Unsupported architecture: $ARCH"
exit 1
fi
rm -rf python pp
wget -qO- "$PYTHON_URL" | tar --use-compress-program=unzstd -xf - -C ./
mv python pp
mv pp/install python
chown -R root:root python
python/bin/pip3 install -t deps --only-binary :all -r requirements.txt
cp cmapi_server/cmapi_server.conf cmapi_server/cmapi_server.conf.default
}
build_cmapi() {
cd "$COLUMNSTORE_SOURCE_PATH"/cmapi
./cleanup.sh
cmake -D"${pkg_format^^}"=1 -DSERVER_DIR="$MDB_SOURCE_PATH" . && make package
}
install_deps
build_cmapi

69
build/createrepo.sh Executable file
View File

@@ -0,0 +1,69 @@
#!/bin/bash
# This script creates a repository from packages in result directory
# Should be executed by root
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
optparse.define short=R long=result desc="Directory with packages" variable=RESULT
source $(optparse.build)
echo "Arguments received: $@"
BUILDDIR="verylongdirnameforverystrangecpackbehavior"
COLUMNSTORE_RPM_PACKAGES_PATH="/mdb/${BUILDDIR}/*.rpm"
CMAPI_RPM_PACKAGES_PATH="/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.rpm"
COLUMNSTORE_DEB_PACKAGES_PATH="/mdb/*.deb"
CMAPI_DEB_PACKAGES_PATH="/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.deb"
if [ "$EUID" -ne 0 ]; then
error "Please run script as root"
exit 1
fi
if [[ -z "${RESULT:-}" ]]; then
echo "Please provide provide --result parameter, e.g. ./createrepo.sh --result ubuntu24.04"
exit 1
fi
cd "/mdb/${BUILDDIR}"
if ! compgen -G "/mdb/${BUILDDIR}/*.rpm" >/dev/null &&
! compgen -G "/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.rpm" >/dev/null &&
! compgen -G "/mdb/${BUILDDIR}/*.deb" >/dev/null &&
! compgen -G "/mdb/${BUILDDIR}/storage/columnstore/columnstore/cmapi/*.deb" >/dev/null; then
echo "None of the cmapi or columnstore packages found. Failing!"
exit 1
fi
echo "Adding columnstore packages to repository..."
if [[ $(compgen -G "$COLUMNSTORE_RPM_PACKAGES_PATH") ]]; then
mv -v $COLUMNSTORE_RPM_PACKAGES_PATH "./${RESULT}/"
elif [[ $(compgen -G "$COLUMNSTORE_DEB_PACKAGES_PATH") ]]; then
mv -v $COLUMNSTORE_DEB_PACKAGES_PATH "./${RESULT}/"
else
echo "Columnstore packages are not found!"
fi
echo "Adding cmapi packages to repository..."
if [[ $(compgen -G "$CMAPI_RPM_PACKAGES_PATH") ]]; then
mv -v $CMAPI_RPM_PACKAGES_PATH "./${RESULT}/"
elif [[ $(compgen -G "$CMAPI_DEB_PACKAGES_PATH") ]]; then
mv -v $CMAPI_DEB_PACKAGES_PATH "./${RESULT}/"
else
echo "Cmapi packages are not found!"
fi
if [[ $(compgen -G "./${RESULT}/*.rpm") ]]; then
retry_eval 5 "dnf install -q -y createrepo"
createrepo "./${RESULT}"
else
retry_eval 5 "apt update && apt install -y dpkg-dev"
dpkg-scanpackages "${RESULT}" | gzip >"./${RESULT}/Packages.gz"
fi
mkdir -p "/drone/src/${RESULT}"
cp -vrf "./${RESULT}/." "/drone/src/${RESULT}"

81
build/debian_install_file_compare.py Normal file
View File

@@ -0,0 +1,81 @@
#!/usr/bin/env python3
import argparse
import fnmatch
import sys
def read_patterns(filename):
with open(filename, 'r') as f:
return [line.strip() for line in f if line.strip()]
def read_paths_with_comments(filename):
paths = []
with open(filename, 'r') as f:
for line in f:
raw = line.rstrip('\n')
if not raw.strip():
continue
# Split into path and comment
if '#' in raw:
path, comment = raw.split('#', 1)
path = path.strip()
comment = comment.strip()
else:
path = raw.strip()
comment = ''
if path:
paths.append((raw, path, comment))
return paths
def print_red(msg):
if sys.stdout.isatty():
print("\033[31m" + msg + "\033[0m")
else:
print(msg)
def main():
parser = argparse.ArgumentParser(description="Compare wildcard patterns to file paths.")
parser.add_argument('patterns_file', help='File with wildcard patterns (one per line)')
parser.add_argument('paths_file', help='File with file paths (one per line)')
args = parser.parse_args()
patterns = read_patterns(args.patterns_file)
paths_with_comments = read_paths_with_comments(args.paths_file)
# Track which paths are matched (by index)
path_matched = [False] * len(paths_with_comments)
# Collect patterns that don't match any path
unmatched_patterns = []
for pattern in patterns:
matched = False
for i, (raw, path, comment) in enumerate(paths_with_comments):
if fnmatch.fnmatch(path, pattern):
matched = True
path_matched[i] = True
if not matched:
unmatched_patterns.append(pattern)
if unmatched_patterns:
print_red("The files declared in debian/mariadb-plugin-columnstore.install "
" are not added to CMakeLists via columnstore_* statements see cmake/ColumnstoreLibrary.cmake")
for pattern in unmatched_patterns:
print(f"- {pattern}")
# Collect paths that weren't matched by any pattern
unmatched_paths = []
for (raw, path, comment), matched in zip(paths_with_comments, path_matched):
if not matched:
unmatched_paths.append((path, comment))
if unmatched_paths:
print_red("The files added via columnstore_* statements from cmake/ColumnstoreLibrary.cmake "
"are missing in debian/mariadb-plugin-columnstore.install file")
for path, comment in unmatched_paths:
if comment:
print(f"- {path} {comment}")
else:
print(f"- {path}")
if __name__ == "__main__":
main()

125
build/prepare_test_stage.sh Executable file
View File

@@ -0,0 +1,125 @@
#!/bin/bash
# Should be executed by root
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
COLUMNSTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
source "$SCRIPT_LOCATION"/utils.sh
echo "Arguments received: $@"
optparse.define short=c long=container-name desc="Name of the Docker container to run tests in" variable=CONTAINER_NAME
optparse.define short=i long=docker-image desc="Docker image name to start container from" variable=DOCKER_IMAGE
optparse.define short=r long=result-path desc="Name suffix used in core dump file path" variable=RESULT
optparse.define short=s long=do-setup desc="Run setup-repo.sh inside the container" variable=DO_SETUP
optparse.define short=u long=packages-url desc="Packages url" variable=PACKAGES_URL
source $(optparse.build)
if [[ "$EUID" -ne 0 ]]; then
error "Please run script as root"
exit 1
fi
if [[ -z "${CONTAINER_NAME:-}" || -z "${DOCKER_IMAGE:-}" || -z "${RESULT:-}" || -z "${DO_SETUP:-}" || -z "${PACKAGES_URL:-}" ]]; then
echo "Please provide --container-name, --docker-image, --result-path, --packages-url and --do-setup parameters, e.g. ./prepare_test_stage.sh --container-name smoke11212 --docker-image detravi/ubuntu:24.04 --result-path ubuntu24.04 --packages-url https://cspkg.s3.amazonaws.com/stable-23.10/pull_request/91/10.6-enterprise --do-setup true"
exit 1
fi
start_container() {
if [[ "$RESULT" == *rocky* ]]; then
SYSTEMD_PATH="/usr/lib/systemd/systemd"
MTR_PATH="/usr/share/mysql-test"
else
SYSTEMD_PATH="systemd"
MTR_PATH="/usr/share/mysql/mysql-test"
fi
docker_run_args=(
--env OS="$RESULT"
--env PACKAGES_URL="$PACKAGES_URL"
--env DEBIAN_FRONTEND=noninteractive
--env MCS_USE_S3_STORAGE=0
--name "$CONTAINER_NAME"
--ulimit core=-1
--privileged
--detach
)
if [[ "$CONTAINER_NAME" == *smoke* ]]; then
docker_run_args+=(--memory 3g)
elif [[ "$CONTAINER_NAME" == *mtr* ]]; then
docker_run_args+=(--shm-size=500m --memory 8g --env MYSQL_TEST_DIR="$MTR_PATH")
elif [[ "$CONTAINER_NAME" == *cmapi* ]]; then
docker_run_args+=(--env PYTHONPATH="${PYTHONPATH}")
elif [[ "$CONTAINER_NAME" == *upgrade* ]]; then
docker_run_args+=(--env UCF_FORCE_CONFNEW=1 --volume /sys/fs/cgroup:/sys/fs/cgroup:ro)
elif [[ "$CONTAINER_NAME" == *regression* ]]; then
docker_run_args+=(--shm-size=500m --memory 12g)
else
echo "Unknown container type: $CONTAINER_NAME"
exit 1
fi
docker_run_args+=("$DOCKER_IMAGE" "$SYSTEMD_PATH" --unit=basic.target)
docker run "${docker_run_args[@]}"
sleep 5
bash "$COLUMNSTORE_SOURCE_PATH"/core_dumps/docker-awaiter.sh "$CONTAINER_NAME"
if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then
error "Container '$CONTAINER_NAME' has not started!"
exit 1
fi
}
start_container
if [[ "$RESULT" != *rocky* ]]; then
execInnerDocker "$CONTAINER_NAME" 'sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d'
fi
#list_cgroups
echo "Docker CGroups opts here"
ls -al /sys/fs/cgroup/cgroup.controllers || true
ls -al /sys/fs/cgroup/ || true
ls -al /sys/fs/cgroup/memory || true
execInnerDocker "$CONTAINER_NAME" 'echo Inner Docker CGroups opts here'
execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/cgroup.controllers || true'
execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/ || true'
execInnerDocker "$CONTAINER_NAME" 'ls -al /sys/fs/cgroup/memory || true'
# Prepare core dump directory inside container
execInnerDocker "$CONTAINER_NAME" 'mkdir -p core && chmod 777 core'
docker cp "$COLUMNSTORE_SOURCE_PATH"/core_dumps/. "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/build/utils.sh "$CONTAINER_NAME":/
docker cp "$COLUMNSTORE_SOURCE_PATH"/setup-repo.sh "$CONTAINER_NAME":/
if [[ "$DO_SETUP" == "true" ]]; then
execInnerDocker "$CONTAINER_NAME" '/setup-repo.sh'
fi
# install deps
if [[ "$RESULT" == *rocky* ]]; then
execInnerDockerWithRetry "$CONTAINER_NAME" 'yum update -y && yum install -y cracklib-dicts diffutils elfutils epel-release findutils iproute gawk gcc-c++ gdb hostname lz4 patch perl procps-ng rsyslog sudo tar wget which'
else
change_ubuntu_mirror_in_docker "$CONTAINER_NAME" "us"
execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y elfutils findutils iproute2 g++ gawk gdb hostname liblz4-tool patch procps rsyslog sudo tar wget'
fi
# Configure core dump naming pattern
execInnerDocker "$CONTAINER_NAME" 'sysctl -w kernel.core_pattern="/core/%E_${RESULT}_core_dump.%p"'
#Install columnstore in container
echo "Installing columnstore..."
if [[ "$RESULT" == *rocky* ]]; then
execInnerDockerWithRetry "$CONTAINER_NAME" 'yum install -y MariaDB-columnstore-engine MariaDB-test'
else
execInnerDockerWithRetry "$CONTAINER_NAME" 'apt update -y && apt install -y mariadb-plugin-columnstore mariadb-test'
fi
sleep 5
echo "PrepareTestStage completed in $CONTAINER_NAME"

96
build/report_test_stage.sh Executable file
View File

@@ -0,0 +1,96 @@
#!/bin/bash
set -eo pipefail
CONTAINER_NAME=$1
RESULT=$2
STAGE=$3
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
echo "Arguments received: $@"
if [[ "$EUID" -ne 0 ]]; then
error "Please run script as root"
exit 1
fi
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
error "Container '${CONTAINER_NAME}' is not running."
exit 1
fi
if [[ "$RESULT" == *rocky* ]]; then
SYSTEMD_PATH="/usr/lib/systemd/systemd"
MTR_PATH="/usr/share/mysql-test"
else
SYSTEMD_PATH="systemd"
MTR_PATH="/usr/share/mysql/mysql-test"
fi
echo "Reporting test stage: ${STAGE} executed in ${CONTAINER_NAME} container"
if [[ "${CONTAINER_NAME}" == *smoke* ]] || [[ "${CONTAINER_NAME}" == *mtr* ]] || [[ "${CONTAINER_NAME}" == *cmapi* ]]; then
# common logs for smoke, mtr, cmapi
echo "---------- start mariadb service logs ----------"
execInnerDocker "$CONTAINER_NAME" 'journalctl -u mariadb --no-pager || echo "mariadb service failure"'
echo "---------- end mariadb service logs ----------"
echo
echo "---------- start columnstore debug log ----------"
execInnerDocker "$CONTAINER_NAME" 'cat /var/log/mariadb/columnstore/debug.log || echo "missing columnstore debug.log"'
echo "---------- end columnstore debug log ----------"
if [[ "${CONTAINER_NAME}" == *mtr* ]]; then
echo
docker cp "${CONTAINER_NAME}:${MTR_PATH}/var/log" "/drone/src/${RESULT}/mtr-logs" || echo "missing ${MTR_PATH}/var/log"
fi
if [[ "${CONTAINER_NAME}" == *cmapi* ]]; then
echo
echo "---------- start cmapi log ----------"
execInnerDocker "$CONTAINER_NAME" 'cat /var/log/mariadb/columnstore/cmapi_server.log || echo "missing cmapi_server.log"'
echo "---------- end cmapi log ----------"
fi
elif [[ "${CONTAINER_NAME}" == *upgrade* ]]; then
# nothing to report here for upgrade
:
elif [[ "${CONTAINER_NAME}" == *regression* ]]; then
echo "---------- start columnstore regression short report ----------"
execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; cat go.log || echo "missing go.log"'
echo "---------- end columnstore regression short report ----------"
echo
docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/reg-logs/" "/drone/src/${RESULT}/" || echo "missing regression logs"
docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs.tgz"
execInnerDocker "$CONTAINER_NAME" 'tar czf regressionQueries.tgz /mariadb-columnstore-regression-test/mysql/queries/'
execInnerDocker "$CONTAINER_NAME" 'cd /mariadb-columnstore-regression-test/mysql/queries/nightly/alltest; tar czf testErrorLogs2.tgz *.log /var/log/mariadb/columnstore || echo "failed to grab regression results"'
docker cp "${CONTAINER_NAME}:/mariadb-columnstore-regression-test/mysql/queries/nightly/alltest/testErrorLogs2.tgz" "/drone/src/${RESULT}/" || echo "missing testErrorLogs2.tgz"
docker cp "${CONTAINER_NAME}:regressionQueries.tgz" "/drone/src/${RESULT}/" || echo "missing regressionQueries.tgz"
else
echo "Unknown stage's container provided: ${CONTAINER_NAME}"
exit 1
fi
execInnerDocker "$CONTAINER_NAME" "/logs.sh ${STAGE}"
execInnerDocker "$CONTAINER_NAME" "/core_dump_check.sh core /core/ ${STAGE}"
docker cp "${CONTAINER_NAME}:/core/" "/drone/src/${RESULT}/"
docker cp "${CONTAINER_NAME}:/unit_logs/" "/drone/src/${RESULT}/"
execInnerDocker "$CONTAINER_NAME" "/core_dump_drop.sh core"
echo "Saved artifacts:"
ls -R "/drone/src/${RESULT}/"
echo "Done reporting ${STAGE}"
cleanup() {
if [[ -n $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
echo "Cleaning up container ${CONTAINER_NAME}..."
docker rm -f "${CONTAINER_NAME}" || echo "Can't remove container ${CONTAINER_NAME}!"
fi
}
#Remove the container on exit
trap cleanup EXIT

40
build/run_smoke.sh Executable file
View File

@@ -0,0 +1,40 @@
#!/bin/bash
set -eo pipefail
SCRIPT_LOCATION=$(dirname "$0")
source "$SCRIPT_LOCATION"/utils.sh
CONTAINER_NAME="$1"
echo "Arguments received: $@"
if [[ "$EUID" -ne 0 ]]; then
error "Please run script as root"
exit 1
fi
if [[ -z $(docker ps -q --filter "name=${CONTAINER_NAME}") ]]; then
error "Container '${CONTAINER_NAME}' is not running."
exit 1
fi
message "Running smoke checks..."
# start mariadb and mariadb-columnstore services and run simple query
execInnerDocker "$CONTAINER_NAME" 'systemctl start mariadb'
execInnerDocker "$CONTAINER_NAME" '/usr/bin/mcsSetConfig SystemConfig CGroup just_no_group_use_local'
execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore'
execInnerDocker "$CONTAINER_NAME" 'mariadb -e "CREATE DATABASE IF NOT EXISTS test; \
CREATE TABLE test.t1 (a INT) ENGINE=Columnstore; \
INSERT INTO test.t1 VALUES (1); \
SELECT * FROM test.t1;"'
# restart both services, wait a bit, then insert and select again
execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb'
execInnerDocker "$CONTAINER_NAME" 'systemctl restart mariadb-columnstore'
sleep 10
execInnerDocker "$CONTAINER_NAME" 'mariadb -e "INSERT INTO test.t1 VALUES (2); SELECT * FROM test.t1;"'

85
build/update-clang-version.sh Normal file
View File

@@ -0,0 +1,85 @@
#!/bin/bash
function register_clang_version() {
local version=$1
local priority=$2
update-alternatives \
--verbose \
--install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${version} ${priority} \
--slave /usr/bin/llvm-ar llvm-ar /usr/bin/llvm-ar-${version} \
--slave /usr/bin/llvm-as llvm-as /usr/bin/llvm-as-${version} \
--slave /usr/bin/llvm-bcanalyzer llvm-bcanalyzer /usr/bin/llvm-bcanalyzer-${version} \
--slave /usr/bin/llvm-c-test llvm-c-test /usr/bin/llvm-c-test-${version} \
--slave /usr/bin/llvm-cat llvm-cat /usr/bin/llvm-cat-${version} \
--slave /usr/bin/llvm-cfi-verify llvm-cfi-verify /usr/bin/llvm-cfi-verify-${version} \
--slave /usr/bin/llvm-cov llvm-cov /usr/bin/llvm-cov-${version} \
--slave /usr/bin/llvm-cvtres llvm-cvtres /usr/bin/llvm-cvtres-${version} \
--slave /usr/bin/llvm-cxxdump llvm-cxxdump /usr/bin/llvm-cxxdump-${version} \
--slave /usr/bin/llvm-cxxfilt llvm-cxxfilt /usr/bin/llvm-cxxfilt-${version} \
--slave /usr/bin/llvm-diff llvm-diff /usr/bin/llvm-diff-${version} \
--slave /usr/bin/llvm-dis llvm-dis /usr/bin/llvm-dis-${version} \
--slave /usr/bin/llvm-dlltool llvm-dlltool /usr/bin/llvm-dlltool-${version} \
--slave /usr/bin/llvm-dwarfdump llvm-dwarfdump /usr/bin/llvm-dwarfdump-${version} \
--slave /usr/bin/llvm-dwp llvm-dwp /usr/bin/llvm-dwp-${version} \
--slave /usr/bin/llvm-exegesis llvm-exegesis /usr/bin/llvm-exegesis-${version} \
--slave /usr/bin/llvm-extract llvm-extract /usr/bin/llvm-extract-${version} \
--slave /usr/bin/llvm-lib llvm-lib /usr/bin/llvm-lib-${version} \
--slave /usr/bin/llvm-link llvm-link /usr/bin/llvm-link-${version} \
--slave /usr/bin/llvm-lto llvm-lto /usr/bin/llvm-lto-${version} \
--slave /usr/bin/llvm-lto2 llvm-lto2 /usr/bin/llvm-lto2-${version} \
--slave /usr/bin/llvm-mc llvm-mc /usr/bin/llvm-mc-${version} \
--slave /usr/bin/llvm-mca llvm-mca /usr/bin/llvm-mca-${version} \
--slave /usr/bin/llvm-modextract llvm-modextract /usr/bin/llvm-modextract-${version} \
--slave /usr/bin/llvm-mt llvm-mt /usr/bin/llvm-mt-${version} \
--slave /usr/bin/llvm-nm llvm-nm /usr/bin/llvm-nm-${version} \
--slave /usr/bin/llvm-objcopy llvm-objcopy /usr/bin/llvm-objcopy-${version} \
--slave /usr/bin/llvm-objdump llvm-objdump /usr/bin/llvm-objdump-${version} \
--slave /usr/bin/llvm-opt-report llvm-opt-report /usr/bin/llvm-opt-report-${version} \
--slave /usr/bin/llvm-pdbutil llvm-pdbutil /usr/bin/llvm-pdbutil-${version} \
--slave /usr/bin/llvm-PerfectShuffle llvm-PerfectShuffle /usr/bin/llvm-PerfectShuffle-${version} \
--slave /usr/bin/llvm-profdata llvm-profdata /usr/bin/llvm-profdata-${version} \
--slave /usr/bin/llvm-ranlib llvm-ranlib /usr/bin/llvm-ranlib-${version} \
--slave /usr/bin/llvm-rc llvm-rc /usr/bin/llvm-rc-${version} \
--slave /usr/bin/llvm-readelf llvm-readelf /usr/bin/llvm-readelf-${version} \
--slave /usr/bin/llvm-readobj llvm-readobj /usr/bin/llvm-readobj-${version} \
--slave /usr/bin/llvm-rtdyld llvm-rtdyld /usr/bin/llvm-rtdyld-${version} \
--slave /usr/bin/llvm-size llvm-size /usr/bin/llvm-size-${version} \
--slave /usr/bin/llvm-split llvm-split /usr/bin/llvm-split-${version} \
--slave /usr/bin/llvm-stress llvm-stress /usr/bin/llvm-stress-${version} \
--slave /usr/bin/llvm-strings llvm-strings /usr/bin/llvm-strings-${version} \
--slave /usr/bin/llvm-strip llvm-strip /usr/bin/llvm-strip-${version} \
--slave /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${version} \
--slave /usr/bin/llvm-tblgen llvm-tblgen /usr/bin/llvm-tblgen-${version} \
--slave /usr/bin/llvm-undname llvm-undname /usr/bin/llvm-undname-${version} \
--slave /usr/bin/llvm-xray llvm-xray /usr/bin/llvm-xray-${version}
update-alternatives \
--verbose \
--install /usr/bin/clang clang /usr/bin/clang-${version} ${priority} \
--slave /usr/bin/clang++ clang++ /usr/bin/clang++-${version} \
--slave /usr/bin/clang-format clang-format /usr/bin/clang-format-${version} \
--slave /usr/bin/clang-cpp clang-cpp /usr/bin/clang-cpp-${version} \
--slave /usr/bin/clang-cl clang-cl /usr/bin/clang-cl-${version} \
--slave /usr/bin/clangd clangd /usr/bin/clangd-${version} \
--slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${version} \
--slave /usr/bin/clang-check clang-check /usr/bin/clang-check-${version} \
--slave /usr/bin/clang-query clang-query /usr/bin/clang-query-${version} \
--slave /usr/bin/asan_symbolize asan_symbolize /usr/bin/asan_symbolize-${version} \
--slave /usr/bin/bugpoint bugpoint /usr/bin/bugpoint-${version} \
--slave /usr/bin/dsymutil dsymutil /usr/bin/dsymutil-${version} \
--slave /usr/bin/lld lld /usr/bin/lld-${version} \
--slave /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${version} \
--slave /usr/bin/lld-link lld-link /usr/bin/lld-link-${version} \
--slave /usr/bin/llc llc /usr/bin/llc-${version} \
--slave /usr/bin/lli lli /usr/bin/lli-${version} \
--slave /usr/bin/obj2yaml obj2yaml /usr/bin/obj2yaml-${version} \
--slave /usr/bin/opt opt /usr/bin/opt-${version} \
--slave /usr/bin/sanstats sanstats /usr/bin/sanstats-${version} \
--slave /usr/bin/verify-uselistorder verify-uselistorder /usr/bin/verify-uselistorder-${version} \
--slave /usr/bin/wasm-ld wasm-ld /usr/bin/wasm-ld-${version} \
--slave /usr/bin/yaml2obj yaml2obj /usr/bin/yaml2obj-${version}
}
register_clang_version $1 $2

233
build/utils.sh
View File

@@ -1,13 +1,24 @@
#!/bin/bash
if [[ -n "$TERM" && "$TERM" != "dumb" && $(command -v tput) ]]; then
TPUT_AVAILABLE=true
else
TPUT_AVAILABLE=false
fi
if [[ $TPUT_AVAILABLE == true ]]; then
color_normal=$(tput sgr0)
color_bold=$(tput bold)
color_red="$color_bold$(tput setaf 1)"
color_green="$color_bold$(tput setaf 2)"
color_fawn=$(tput setaf 3); color_beige="$color_fawn"
color_fawn=$(tput setaf 3)
color_beige="$color_fawn"
color_yellow="$color_bold$color_fawn"
color_darkblue=$(tput setaf 4)
color_blue="$color_bold$color_darkblue"
color_purple=$(tput setaf 5); color_magenta="$color_purple"
color_purple=$(tput setaf 5)
color_magenta="$color_purple"
color_pink="$color_bold$color_purple"
color_darkcyan=$(tput setaf 6)
color_cyan="$color_bold$color_darkcyan"
@@ -15,7 +26,6 @@ color_gray=$(tput setaf 7)
color_darkgray="$color_bold"$(tput setaf 0)
color_white="$color_bold$color_gray"
if [[ $(tput colors) == '256' ]]; then
color_red=$(tput setaf 196)
color_yellow=$(tput setaf 228)
@@ -24,63 +34,89 @@ if [[ $(tput colors) == '256' ]]; then
color_darkgray=$(tput setaf 59)
fi
message()
{
echo $color_cyan ・ $@$color_normal
else
# Basic attributes
color_normal="\e[0m" # sgr0
color_bold="\e[1m" # bold
# Standard 8-color palette
color_red="\e[1;31m" # bold + setaf 1
color_green="\e[1;32m" # bold + setaf 2
color_fawn="\e[33m" # setaf 3
color_beige="$color_fawn" # alias
color_yellow="\e[1;33m" # bold + setaf 3
color_darkblue="\e[34m" # setaf 4
color_blue="\e[1;34m" # bold + setaf 4
color_purple="\e[35m" # setaf 5
color_magenta="$color_purple" # alias
color_pink="\e[1;35m" # bold + setaf 5
color_darkcyan="\e[36m" # setaf 6
color_cyan="\e[1;36m" # bold + setaf 6
color_gray="\e[37m" # setaf 7
color_darkgray="\e[1;30m" # bold + setaf 0
color_white="\e[1;37m" # bold + setaf 7
if [ "$TERM" = "xterm-256color" ] || [ "$COLORTERM" = "truecolor" ] || [ "$COLORTERM" = "24bit" ]; then
# Only set 256-color codes if actually in a 256-color terminal
color_red="\e[91m" # bright red
color_yellow="\e[93m" # light yellow
color_cyan="\e[96m" # bright cyan
color_green="\e[92m" # light green
color_darkgray="\e[90m" # dark gray
fi
fi
message() {
echo -e $color_cyan ・ $@$color_normal
}
warn()
{
echo $color_yellow ・ $@$color_normal
warn() {
echo -e $color_yellow ・ $@$color_normal
}
error()
{
echo $color_red ・ $@$color_normal
error() {
echo -e $color_red ・ $@$color_normal
}
message_split()
{
echo $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
message_split() {
echo -e $color_darkgray ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ $color_normal
}
message_splitted()
{
message_splitted() {
message_split
if [[ $TPUT_AVAILABLE == true ]]; then
echo $color_green ・ $@$color_normal
else
echo "$@"
fi
message_split
}
colorify_array()
{
colorify_array() {
PROMT=""
for a in "$@"
do
for a in "$@"; do
if [[ $TPUT_AVAILABLE == true ]]; then
i=$((((i + 1) % (123 - 106)) + 106))
if [[ $(tput colors) == '256' ]]; then
PROMT="$PROMT $(tput setaf $i)$a$color_normal"
else
PROMT="$PROMT $a"
fi
else
PROMT="$PROMT $a"
fi
done
echo $PROMT
}
newline_array()
{
newline_array() {
PROMT=""
for a in "$@"
do
for a in "$@"; do
PROMT="$PROMT$a\n"
done
echo -e $PROMT
}
function spinner
{
function spinner() {
freq=${1:-10}
points=(⣾ ⣽ ⣻ ⢿ ⡿ ⣟ ⣯ ⣷)
colored_points=($(colorify_array ${points[@]}))
@@ -91,15 +127,24 @@ function spinner
line_num=$((line_num + 1))
if [[ $((line_num % freq)) = 0 ]]; then
point_num=$(((point_num + 1) % len))
if [[ $TPUT_AVAILABLE == true ]]; then
echo -ne "\r${colored_points[point_num]}"
else
echo -ne "\r${points[point_num]}"
fi
done;
fi
done
echo
}
function onelinearizator() {
while read data; do
echo -ne "\r\e[K$data"
done
echo
}
detect_distro()
{
detect_distro() {
if [ -f /etc/os-release ]; then
. /etc/os-release
export OS=$NAME
@@ -122,27 +167,27 @@ detect_distro()
OS=$(uname -s)
OS_VERSION=$(uname -r)
fi
OS=$(echo $OS | cut -f 1 -d " ")
OS=$(echo $OS | cut -f 1 -d " " | tr '[:upper:]' '[:lower:]')":"$OS_VERSION
message "Detected $color_yellow$OS $OS_VERSION$color_normal"
}
menuStr=""
function hideCursor() {
printf "\033[?25l"
printf "\e[?25l"
# capture CTRL+C so cursor can be reset
trap "showCursor && exit 0" 2
}
function showCursor() {
printf "\033[?25h"
printf "\e[?25h"
}
function clearLastMenu() {
local msgLineCount=$(printf "$menuStr" | wc -l)
# moves the curser up N lines so the output overwrites it
echo -en "\033[${msgLineCount}A"
echo -en "\e[${msgLineCount}A"
# clear to end of screen to ensure there's no text left behind from previous input
[ $1 ] && tput ed
@@ -202,8 +247,8 @@ function renderMenu(){
}
function getChoice() {
local KEY__ARROW_UP=$(echo -e "\033[A")
local KEY__ARROW_DOWN=$(echo -e "\033[B")
local KEY__ARROW_UP=$(echo -e "\e[A")
local KEY__ARROW_DOWN=$(echo -e "\e[B")
local KEY__ENTER=$(echo -e "\n")
local captureInput=true
local displayHelp=false
@@ -246,25 +291,25 @@ function getChoice(){
# just display help
if $displayHelp; then
echo;
echo
echo "Usage: getChoice [OPTION]..."
echo "Renders a keyboard navigable menu with a visual indicator of what's selected."
echo;
echo
echo " -h, --help Displays this message"
echo " -i, --index The initially selected index for the options"
echo " -m, --max Limit how many options are displayed"
echo " -o, --options An Array of options for a User to choose from"
echo " -q, --query Question or statement presented to the User"
echo;
echo
echo "Example:"
echo " foodOptions=(\"pizza\" \"burgers\" \"chinese\" \"sushi\" \"thai\" \"italian\" \"shit\")"
echo;
echo
echo " getChoice -q \"What do you feel like eating?\" -o foodOptions -i \$((\${#foodOptions[@]}-1)) -m 4"
echo " printf \"\\n First choice is '\${selectedChoice}'\\n\""
echo;
echo
echo " getChoice -q \"Select another option in case the first isn't available\" -o foodOptions"
echo " printf \"\\n Second choice is '\${selectedChoice}'\\n\""
echo;
echo
return 0
fi
@@ -282,7 +327,7 @@ function getChoice(){
hideCursor
while $captureInput; do
read -rsn3 key # `3` captures the escape (\033'), bracket ([), & type (A) characters.
read -rsn3 key # `3` captures the escape (\e'), bracket ([), & type (A) characters.
case "$key" in
"$KEY__ARROW_UP")
@@ -308,7 +353,6 @@ function getChoice(){
done
}
function optparse.throw_error() {
local message="$1"
error "OPTPARSE: ERROR: $message"
@@ -322,8 +366,8 @@ function optparse.define(){
fi
for option_id in $(seq 1 $#); do
local option="$(eval "echo \$$option_id")"
local key="$( echo $option | awk -F "=" '{print $1}' )";
local value="$( echo $option | awk -F "=" '{print $2}' )";
local key="$(echo $option | awk -F "=" '{print $1}')"
local value="$(echo $option | awk -F "=" '{print $2}')"
#essentials: shortname, longname, description
if [ "$key" = "short" ]; then
@@ -375,7 +419,7 @@ function optparse.define(){
# -----------------------------------------------------------------------------------------------------------------------------
function optparse.build() {
local build_file="$(mktemp -t "optparse-XXXXXX.tmp")"
local build_file="$(mktemp "${TMPDIR:-/tmp}/optparse-XXXXXX")"
# Building getopts header here
@@ -444,3 +488,92 @@ EOF
# Return file name to parent
echo "$build_file"
}
function retry_eval() {
if [ "$#" -lt 2 ]; then
error "Usage: retry_eval <max_retries> <command...>"
return 1
fi
local max_retries=$1
shift # Remove max_retries from arguments
local attempt=1
local initial_delay=1
while [ "$attempt" -le "$max_retries" ]; do
message_split
message "Attempt $attempt of $max_retries: $*"
if eval "$@"; then
message "Command '$@' done"
message_split
return 0
fi
if [ "$attempt" -lt "$max_retries" ]; then
delay=$((initial_delay * 2 ** (attempt - 1)))
warn "Retrying command "$@" in $delay seconds..."
message_split
sleep "$delay"
fi
((attempt++))
done
error "Max retries reached for command: $*"
message_split
exit 13
}
function execInnerDocker() {
local container_name=$1
shift 1 # Remove first arg (container_name)
docker exec -t "$container_name" bash -c "$@"
local dockerCommandExitCode=$?
if [[ $dockerCommandExitCode -ne 0 ]]; then
error "Command \"$@\" failed in container \"$container_name\""
exit $dockerCommandExitCode
fi
}
function change_ubuntu_mirror() {
local region="$1"
message "Changing Ubuntu mirror to $region"
sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list 2>/dev/null || true
sed -i "s|//\(${region}\.\)\?archive\.ubuntu\.com|//${region}.archive.ubuntu.com|g" /etc/apt/sources.list.d/ubuntu.sources 2>/dev/null || true
cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list 2>/dev/null | grep archive || true
message_split
}
function execInnerDockerWithRetry() {
local max_retries=5
local container_name=$1
shift 1
local cmd=("$@")
local attempt=1
local dockerCommandExitCode=0
local docker_funcs=$(declare -f retry_eval color_normal color_cyan color_yellow color_red error warn message message_split)
# Build the full command to execute in docker
local full_command="$docker_funcs; retry_eval $max_retries \"${cmd[*]}\""
# Execute the command in docker
docker exec -t "$container_name" bash -c "$full_command"
dockerCommandExitCode=$?
if [[ $dockerCommandExitCode -ne 0 ]]; then
error "Command \"${cmd[*]}\" failed in container \"$container_name\" after $max_retries attempts"
return $dockerCommandExitCode
fi
return 0
}
change_ubuntu_mirror_in_docker() {
local container_name=$1
local region=$2
local docker_funcs=$(declare -f color_normal color_cyan color_yellow color_red error warn message message_split change_ubuntu_mirror)
execInnerDocker "$container_name" "$docker_funcs; change_ubuntu_mirror ${region}"
}

241
cmake-format.py Normal file
View File

@@ -0,0 +1,241 @@
# ----------------------------------
# Options affecting listfile parsing
# ----------------------------------
with section("parse"):
# Specify structure for custom cmake functions
additional_commands = { 'foo': { 'flags': ['BAR', 'BAZ'],
'kwargs': {'DEPENDS': '*', 'HEADERS': '*', 'SOURCES': '*'}}}
# Override configurations per-command where available
override_spec = {}
# Specify variable tags.
vartags = []
# Specify property tags.
proptags = []
# -----------------------------
# Options affecting formatting.
# -----------------------------
with section("format"):
# Disable formatting entirely, making cmake-format a no-op
disable = False
# How wide to allow formatted cmake files
line_width = 120
# How many spaces to tab for indent
tab_size = 4
# If true, lines are indented using tab characters (utf-8 0x09) instead of
# <tab_size> space characters (utf-8 0x20). In cases where the layout would
# require a fractional tab character, the behavior of the fractional
# indentation is governed by <fractional_tab_policy>
use_tabchars = False
# If <use_tabchars> is True, then the value of this variable indicates how
# fractional indentions are handled during whitespace replacement. If set to
# 'use-space', fractional indentation is left as spaces (utf-8 0x20). If set
# to `round-up` fractional indentation is replaced with a single tab character
# (utf-8 0x09) effectively shifting the column to the next tabstop
fractional_tab_policy = 'use-space'
# If an argument group contains more than this many sub-groups (parg or kwarg
# groups) then force it to a vertical layout.
max_subgroups_hwrap = 2
# If a positional argument group contains more than this many arguments, then
# force it to a vertical layout.
max_pargs_hwrap = 6
# If a cmdline positional group consumes more than this many lines without
# nesting, then invalidate the layout (and nest)
max_rows_cmdline = 2
# If true, separate flow control names from their parentheses with a space
separate_ctrl_name_with_space = False
# If true, separate function names from parentheses with a space
separate_fn_name_with_space = False
# If a statement is wrapped to more than one line, than dangle the closing
# parenthesis on its own line.
dangle_parens = True
# If the trailing parenthesis must be 'dangled' on its on line, then align it
# to this reference: `prefix`: the start of the statement, `prefix-indent`:
# the start of the statement, plus one indentation level, `child`: align to
# the column of the arguments
dangle_align = 'prefix'
# If the statement spelling length (including space and parenthesis) is
# smaller than this amount, then force reject nested layouts.
min_prefix_chars = 4
# If the statement spelling length (including space and parenthesis) is larger
# than the tab width by more than this amount, then force reject un-nested
# layouts.
max_prefix_chars = 10
# If a candidate layout is wrapped horizontally but it exceeds this many
# lines, then reject the layout.
max_lines_hwrap = 2
# What style line endings to use in the output.
line_ending = 'unix'
# Format command names consistently as 'lower' or 'upper' case
command_case = 'canonical'
# Format keywords consistently as 'lower' or 'upper' case
keyword_case = 'unchanged'
# A list of command names which should always be wrapped
always_wrap = []
# If true, the argument lists which are known to be sortable will be sorted
# lexicographicall
enable_sort = True
# If true, the parsers may infer whether or not an argument list is sortable
# (without annotation).
autosort = False
# By default, if cmake-format cannot successfully fit everything into the
# desired linewidth it will apply the last, most aggressive attempt that it
# made. If this flag is True, however, cmake-format will print error, exit
# with non-zero status code, and write-out nothing
require_valid_layout = False
# A dictionary mapping layout nodes to a list of wrap decisions. See the
# documentation for more information.
layout_passes = {}
# ------------------------------------------------
# Options affecting comment reflow and formatting.
# ------------------------------------------------
with section("markup"):
# What character to use for bulleted lists
bullet_char = '*'
# What character to use as punctuation after numerals in an enumerated list
enum_char = '.'
# If comment markup is enabled, don't reflow the first comment block in each
# listfile. Use this to preserve formatting of your copyright/license
# statements.
first_comment_is_literal = False
# If comment markup is enabled, don't reflow any comment block which matches
# this (regex) pattern. Default is `None` (disabled).
literal_comment_pattern = None
# Regular expression to match preformat fences in comments default=
# ``r'^\s*([`~]{3}[`~]*)(.*)$'``
fence_pattern = '^\\s*([`~]{3}[`~]*)(.*)$'
# Regular expression to match rulers in comments default=
# ``r'^\s*[^\w\s]{3}.*[^\w\s]{3}$'``
ruler_pattern = '^\\s*[^\\w\\s]{3}.*[^\\w\\s]{3}$'
# If a comment line matches starts with this pattern then it is explicitly a
# trailing comment for the preceding argument. Default is '#<'
explicit_trailing_pattern = '#<'
# If a comment line starts with at least this many consecutive hash
# characters, then don't lstrip() them off. This allows for lazy hash rulers
# where the first hash char is not separated by space
hashruler_min_length = 10
# If true, then insert a space between the first hash char and remaining hash
# chars in a hash ruler, and normalize its length to fill the column
canonicalize_hashrulers = True
# enable comment markup parsing and reflow
enable_markup = True
# ----------------------------
# Options affecting the linter
# ----------------------------
with section("lint"):
# a list of lint codes to disable
disabled_codes = []
# regular expression pattern describing valid function names
function_pattern = '[0-9a-z_]+'
# regular expression pattern describing valid macro names
macro_pattern = '[0-9A-Z_]+'
# regular expression pattern describing valid names for variables with global
# (cache) scope
global_var_pattern = '[A-Z][0-9A-Z_]+'
# regular expression pattern describing valid names for variables with global
# scope (but internal semantic)
internal_var_pattern = '_[A-Z][0-9A-Z_]+'
# regular expression pattern describing valid names for variables with local
# scope
local_var_pattern = '[a-z][a-z0-9_]+'
# regular expression pattern describing valid names for privatedirectory
# variables
private_var_pattern = '_[0-9a-z_]+'
# regular expression pattern describing valid names for public directory
# variables
public_var_pattern = '[A-Z][0-9A-Z_]+'
# regular expression pattern describing valid names for function/macro
# arguments and loop variables.
argument_var_pattern = '[a-z][a-z0-9_]+'
# regular expression pattern describing valid names for keywords used in
# functions or macros
keyword_pattern = '[A-Z][0-9A-Z_]+'
# In the heuristic for C0201, how many conditionals to match within a loop in
# before considering the loop a parser.
max_conditionals_custom_parser = 2
# Require at least this many newlines between statements
min_statement_spacing = 1
# Require no more than this many newlines between statements
max_statement_spacing = 2
max_returns = 6
max_branches = 12
max_arguments = 5
max_localvars = 15
max_statements = 50
# -------------------------------
# Options affecting file encoding
# -------------------------------
with section("encode"):
# If true, emit the unicode byte-order mark (BOM) at the start of the file
emit_byteorder_mark = False
# Specify the encoding of the input file. Defaults to utf-8
input_encoding = 'utf-8'
# Specify the encoding of the output file. Defaults to utf-8. Note that cmake
# only claims to support utf-8 so be careful when using anything else
output_encoding = 'utf-8'
# -------------------------------------
# Miscellaneous configurations options.
# -------------------------------------
with section("misc"):
# A dictionary containing any per-command configuration overrides. Currently
# only `command_case` is supported.
per_command = {}

105
cmake/ColumnstoreLibrary.cmake Normal file
View File

@@ -0,0 +1,105 @@
set(DEBIAN_INSTALL_FILE "${CMAKE_BINARY_DIR}/mariadb-plugin-columnstore.install.generated")
file(WRITE ${DEBIAN_INSTALL_FILE} "#File is generated by ColumnstoreLibrary.cmake, do not edit\n")
macro(add_to_debian_install_file file_path)
string(SUBSTRING "${file_path}" 1 -1 BINARY_ENTRY)
file(RELATIVE_PATH CMAKEFILE "${CMAKE_SOURCE_DIR}/storage/columnstore/columnstore" ${CMAKE_CURRENT_LIST_FILE})
string(STRIP "${BINARY_ENTRY}" BINARY_ENTRY)
file(APPEND ${DEBIAN_INSTALL_FILE} "${BINARY_ENTRY} # added in ${CMAKEFILE}\n")
endmacro()
function(get_target_output_filename TARGET_NAME OUTPUT_VAR)
# 1. Get the target's OUTPUT_NAME (falls back to TARGET_NAME)
get_target_property(OUTPUT_NAME ${TARGET_NAME} OUTPUT_NAME)
if(NOT OUTPUT_NAME)
set(OUTPUT_NAME "${TARGET_NAME}")
endif()
# 1. Get the correct suffix based on target type
get_target_property(TARGET_TYPE ${TARGET_NAME} TYPE)
if(TARGET_TYPE STREQUAL "EXECUTABLE")
set(SUFFIX "${CMAKE_EXECUTABLE_SUFFIX}")
set(PREFIX "") # No prefix for executables
elseif(TARGET_TYPE STREQUAL "SHARED_LIBRARY")
set(SUFFIX "${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(PREFIX "lib")
elseif(TARGET_TYPE STREQUAL "STATIC_LIBRARY")
set(SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(PREFIX "lib")
else()
message(WARNING "Unknown target type for ${TARGET_NAME}")
set(SUFFIX "")
endif()
# 1. Combine into the final filename
set(${OUTPUT_VAR}
"${PREFIX}${OUTPUT_NAME}${SUFFIX}"
PARENT_SCOPE
)
endfunction()
macro(columnstore_install_target target destination)
install(
TARGETS ${target}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_target_output_filename(${target} OUTPUT_FILENAME)
add_to_debian_install_file("${destination}/${OUTPUT_FILENAME}")
endmacro()
macro(columnstore_install_file file destination)
install(
FILES ${file}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_filename_component(FILENAME ${file} NAME)
add_to_debian_install_file("${destination}/${FILENAME}")
endmacro()
macro(columnstore_install_program file destination)
install(
PROGRAMS ${file}
DESTINATION ${destination}
COMPONENT columnstore-engine
)
get_filename_component(FILENAME ${file} NAME)
add_to_debian_install_file(
"${destination}/${FILENAME}
"
)
endmacro()
macro(columnstore_static_library libname)
add_definitions(-fPIC -DPIC)
add_library(${libname} STATIC ${ARGN})
endmacro()
macro(columnstore_shared_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${ENGINE_LIBDIR})
endmacro()
macro(columnstore_library libname)
if(COLUMNSTORE_STATIC_LIBRARIES)
columnstore_static_library(${libname} ${ARGN})
else()
columnstore_shared_library(${libname} ${ARGN})
endif()
endmacro()
macro(columnstore_mysql_plugin_library libname)
add_library(${libname} SHARED ${ARGN})
columnstore_install_target(${libname} ${MARIADB_PLUGINDIR})
endmacro()
macro(columnstore_link libname)
target_link_libraries(${libname} ${ARGN})
endmacro()
macro(columnstore_executable executable_name)
add_executable(${executable_name} ${ARGN})
columnstore_install_target(${executable_name} ${ENGINE_BINDIR})
endmacro()

10
cmake/boost.1.88.named_proxy.hpp.patch Normal file
View File

@@ -0,0 +1,10 @@
+++ a/boost/interprocess/detail/named_proxy.hpp 2025-04-14 16:24:12.018395298 +0000
+++ b/boost/interprocess/detail/named_proxy.hpp 2025-04-14 16:24:12.018395298 +0000
@@ -89,6 +89,7 @@
} BOOST_INTERPROCESS_CATCH_END
}
+ virtual ~CtorArgN() {}
private:
template<std::size_t ...IdxPack>
void construct(void *mem, true_, const index_tuple<IdxPack...>&)

6
cmake/columnstore_version.cmake
View File

@@ -29,6 +29,7 @@ MACRO(GET_COLUMNSTORE_VERSION)
SET(CMAPI_VERSION_MAJOR ${CS_MAJOR_VERSION})
SET(CMAPI_VERSION_MINOR ${CS_MINOR_VERSION})
SET(CMAPI_VERSION_PATCH ${CS_PATCH_VERSION}${CS_EXTRA_VERSION})
SET(CMAPI_VERSION_RELEASE ${CS_RELEASE_VERSION})
ENDIF()
IF(NOT "${CS_MAJOR_VERSION}" MATCHES "[0-9]+" OR
@@ -39,11 +40,6 @@ IF(NOT "${CS_MAJOR_VERSION}" MATCHES "[0-9]+" OR
SET(VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}")
MESSAGE("== MariaDB-Columnstore ${VERSION}")
IF (NOT INSTALL_LAYOUT)
SET(CPACK_PACKAGE_VERSION_MAJOR ${CS_MAJOR_VERSION})
SET(CPACK_PACKAGE_VERSION_MINOR ${CS_MINOR_VERSION})
SET(CPACK_PACKAGE_VERSION_PATCH ${CS_PATCH_VERSION}${CS_EXTRA_VERSION})
ENDIF ()
SET(PACKAGE_VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}")
SET(PACKAGE_RELEASE "${CS_RELEASE_VERSION}")
MATH(EXPR MCSVERSIONHEX "${CS_MAJOR_VERSION} * 256 + ${CS_MINOR_VERSION}" OUTPUT_FORMAT HEXADECIMAL)

120
cmake/compiler_flags.cmake Normal file
View File

@@ -0,0 +1,120 @@
macro(SET_FLAGS)
foreach(F ${ARGV})
my_check_and_set_compiler_flag(${F} DEBUG RELWITHDEBINFO MINSIZEREL)
endforeach()
endmacro()
macro(SET_FLAGS_DEBUG)
foreach(F ${ARGV})
my_check_and_set_compiler_flag(${F} DEBUG)
endforeach()
endmacro()
macro(SET_FLAGS_RELEASE)
foreach(F ${ARGV})
my_check_and_set_compiler_flag(${F} RELWITHDEBINFO)
endforeach()
endmacro()
# C++ standard {
if(have_CXX__std_c__20)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20")
else()
my_check_cxx_compiler_flag("-std=c++2a")
if(have_CXX__std_c__2a)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
else()
message_once(CS_NO_CXX20 "C++ Compiler does not understand -std=c++20")
return()
endif()
endif()
unset(CMAKE_CXX_STANDARD)
# } end C++ standard
# Hacks to keep alive with MariaDB server {
string(REPLACE -D_GLIBCXX_DEBUG "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
string(REPLACE -D_GLIBCXX_ASSERTIONS "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
# } end hacks
if(WITH_COLUMNSTORE_ASAN)
set(WERROR_FLAG)
else()
set(WERROR_FLAG -Werror)
endif()
# Maintainer flags, works when build is done via bootstrap_mcs.sh {
set(COLUMNSTORE_MAINTAINER_FLAGS ${WERROR_FLAG})
# } end Maintainer flags
# Release, Debug and common flags {
set(FLAGS_ALL
-Wall
-Wextra
-ggdb3
-fno-omit-frame-pointer
-fno-strict-aliasing
-fsigned-char
-msse4.2
-DHAVE_CONFIG_H
-DBOOST_BIND_GLOBAL_PLACEHOLDERS
)
set(FLAGS_RELEASE -O3 -DDBUG_OFF)
set(FLAGS_DEBUG -O0 -D_DEBUG)
# } end Release, Debug and common flags
# linker flags {
set(ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed")
# } end linker flags
# compiler specific flags {
set(CLANG_FLAGS
# suppressed warnings
-Wno-unused-parameter
-Wno-cast-function-type-strict
-Wno-deprecated-copy
-Wno-deprecated-declarations
-Wno-deprecated-enum-enum-conversion
-Wno-format-truncation
-Wno-register
-Wno-typedef-redefinition
-Wno-missing-template-arg-list-after-template-kw
-Wno-unused-parameter
)
set(GNU_FLAGS # suppressed warnings
-Wno-deprecated-copy -Wno-deprecated-declarations -Wno-format-truncation -Wno-register -Wno-unused-parameter
)
# } end compiler specific flags
# Sanitizers {
set(ASAN_FLAGS -U_FORTIFY_SOURCE -fsanitize=address -fsanitize-address-use-after-scope -fPIC)
# } end Sanitizers
# configured by cmake/configureEngine.cmake {
if(MASK_LONGDOUBLE)
my_check_and_set_compiler_flag("-DMASK_LONGDOUBLE")
endif()
# } end configured by cmake/configureEngine.cmake
# apply them all
set_flags(${FLAGS_ALL})
set_flags_debug(${FLAGS_DEBUG})
set_flags_release(${FLAGS_RELEASE})
if(COLUMNSTORE_MAINTAINER)
set_flags(${COLUMNSTORE_MAINTAINER_FLAGS})
endif()
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set_flags(${CLANG_FLAGS})
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set_flags(${GNU_FLAGS})
endif()
if(WITH_COLUMNSTORE_ASAN)
set_flags(${ASAN_FLAGS})
endif()

663
cmake/configureEngine.cmake
View File

@@ -1,146 +1,142 @@
#
# Tests for header files
#
INCLUDE (CheckIncludeFiles)
INCLUDE (CheckIncludeFileCXX)
INCLUDE (CheckCSourceCompiles)
INCLUDE (CheckCXXSourceRuns)
INCLUDE (CheckCXXSourceCompiles)
INCLUDE (CheckStructHasMember)
INCLUDE (CheckLibraryExists)
INCLUDE (CheckFunctionExists)
INCLUDE (CheckCCompilerFlag)
INCLUDE (CheckCXXCompilerFlag)
INCLUDE (CheckCXXSourceRuns)
INCLUDE (CheckSymbolExists)
INCLUDE (CheckCXXSymbolExists)
INCLUDE (CheckTypeSize)
include(CheckIncludeFiles)
include(CheckIncludeFileCXX)
include(CheckCSourceCompiles)
include(CheckCXXSourceRuns)
include(CheckCXXSourceCompiles)
include(CheckStructHasMember)
include(CheckLibraryExists)
include(CheckFunctionExists)
include(CheckCCompilerFlag)
include(CheckCXXCompilerFlag)
include(CheckCXXSourceRuns)
include(CheckSymbolExists)
include(CheckCXXSymbolExists)
include(CheckTypeSize)
CHECK_INCLUDE_FILE_CXX (alloca.h HAVE_ALLOCA_H)
CHECK_INCLUDE_FILE_CXX (arpa/inet.h HAVE_ARPA_INET_H)
CHECK_INCLUDE_FILE_CXX (dlfcn.h HAVE_DLFCN_H)
CHECK_INCLUDE_FILE_CXX (fcntl.h HAVE_FCNTL_H)
CHECK_INCLUDE_FILE_CXX (inttypes.h HAVE_INTTYPES_H)
CHECK_INCLUDE_FILE_CXX (limits.h HAVE_LIMITS_H)
CHECK_INCLUDE_FILE_CXX (malloc.h HAVE_MALLOC_H)
CHECK_INCLUDE_FILE_CXX (memory.h HAVE_MEMORY_H)
CHECK_INCLUDE_FILE_CXX (ncurses.h HAVE_NCURSES_H)
CHECK_INCLUDE_FILE_CXX (netdb.h HAVE_NETDB_H)
CHECK_INCLUDE_FILE_CXX (netinet/in.h HAVE_NETINET_IN_H)
CHECK_INCLUDE_FILE_CXX (stddef.h HAVE_STDDEF_H)
CHECK_INCLUDE_FILE_CXX (stdint.h HAVE_STDINT_H)
CHECK_INCLUDE_FILE_CXX (stdlib.h HAVE_STDLIB_H)
CHECK_INCLUDE_FILE_CXX (strings.h HAVE_STRINGS_H)
CHECK_INCLUDE_FILE_CXX (string.h HAVE_STRING_H)
CHECK_INCLUDE_FILE_CXX (syslog.h HAVE_SYSLOG_H)
CHECK_INCLUDE_FILE_CXX (sys/file.h HAVE_SYS_FILE_H)
CHECK_INCLUDE_FILE_CXX (sys/mount.h HAVE_SYS_MOUNT_H)
CHECK_INCLUDE_FILE_CXX (sys/select.h HAVE_SYS_SELECT_H)
CHECK_INCLUDE_FILE_CXX (sys/socket.h HAVE_SYS_SOCKET_H)
CHECK_INCLUDE_FILE_CXX (sys/statfs.h HAVE_SYS_STATFS_H)
CHECK_INCLUDE_FILE_CXX (sys/stat.h HAVE_SYS_STAT_H)
CHECK_INCLUDE_FILE_CXX (sys/timeb.h HAVE_SYS_TIMEB_H)
CHECK_INCLUDE_FILE_CXX (sys/time.h HAVE_SYS_TIME_H)
CHECK_INCLUDE_FILE_CXX (sys/types.h HAVE_SYS_TYPES_H)
CHECK_INCLUDE_FILE_CXX (sys/wait.h HAVE_SYS_WAIT_H)
CHECK_INCLUDE_FILE_CXX (unistd.h HAVE_UNISTD_H)
CHECK_INCLUDE_FILE_CXX (utime.h HAVE_UTIME_H)
CHECK_INCLUDE_FILE_CXX (values.h HAVE_VALUES_H)
CHECK_INCLUDE_FILE_CXX (vfork.h HAVE_VFORK_H)
CHECK_INCLUDE_FILE_CXX (wchar.h HAVE_WCHAR_H)
CHECK_INCLUDE_FILE_CXX (wctype.h HAVE_WCTYPE_H)
CHECK_INCLUDE_FILE_CXX (zlib.h HAVE_ZLIB_H)
check_include_file_cxx(alloca.h HAVE_ALLOCA_H)
check_include_file_cxx(arpa/inet.h HAVE_ARPA_INET_H)
check_include_file_cxx(dlfcn.h HAVE_DLFCN_H)
check_include_file_cxx(fcntl.h HAVE_FCNTL_H)
check_include_file_cxx(inttypes.h HAVE_INTTYPES_H)
check_include_file_cxx(limits.h HAVE_LIMITS_H)
check_include_file_cxx(malloc.h HAVE_MALLOC_H)
check_include_file_cxx(memory.h HAVE_MEMORY_H)
check_include_file_cxx(ncurses.h HAVE_NCURSES_H)
check_include_file_cxx(netdb.h HAVE_NETDB_H)
check_include_file_cxx(netinet/in.h HAVE_NETINET_IN_H)
check_include_file_cxx(stddef.h HAVE_STDDEF_H)
check_include_file_cxx(stdint.h HAVE_STDINT_H)
check_include_file_cxx(stdlib.h HAVE_STDLIB_H)
check_include_file_cxx(strings.h HAVE_STRINGS_H)
check_include_file_cxx(string.h HAVE_STRING_H)
check_include_file_cxx(syslog.h HAVE_SYSLOG_H)
check_include_file_cxx(sys/file.h HAVE_SYS_FILE_H)
check_include_file_cxx(sys/mount.h HAVE_SYS_MOUNT_H)
check_include_file_cxx(sys/select.h HAVE_SYS_SELECT_H)
check_include_file_cxx(sys/socket.h HAVE_SYS_SOCKET_H)
check_include_file_cxx(sys/statfs.h HAVE_SYS_STATFS_H)
check_include_file_cxx(sys/stat.h HAVE_SYS_STAT_H)
check_include_file_cxx(sys/timeb.h HAVE_SYS_TIMEB_H)
check_include_file_cxx(sys/time.h HAVE_SYS_TIME_H)
check_include_file_cxx(sys/types.h HAVE_SYS_TYPES_H)
check_include_file_cxx(sys/wait.h HAVE_SYS_WAIT_H)
check_include_file_cxx(unistd.h HAVE_UNISTD_H)
check_include_file_cxx(utime.h HAVE_UTIME_H)
check_include_file_cxx(values.h HAVE_VALUES_H)
check_include_file_cxx(vfork.h HAVE_VFORK_H)
check_include_file_cxx(wchar.h HAVE_WCHAR_H)
check_include_file_cxx(wctype.h HAVE_WCTYPE_H)
check_include_file_cxx(zlib.h HAVE_ZLIB_H)
CHECK_FUNCTION_EXISTS (_getb67 GETB1)
CHECK_FUNCTION_EXISTS (GETB67 GETB2)
CHECK_FUNCTION_EXISTS (getb67 GETB3)
check_function_exists(_getb67 GETB1)
check_function_exists(GETB67 GETB2)
check_function_exists(getb67 GETB3)
IF(GETB1)
SET (CRAY_STACKSEG_END 1)
ELSEIF(GETB2)
SET (CRAY_STACKSEG_END 1)
ELSEIF(GETB3)
SET (CRAY_STACKSEG_END 1)
ENDIF()
if(GETB1)
set(CRAY_STACKSEG_END 1)
elseif(GETB2)
set(CRAY_STACKSEG_END 1)
elseif(GETB3)
set(CRAY_STACKSEG_END 1)
endif()
CHECK_FUNCTION_EXISTS (alarm HAVE_ALARM)
CHECK_FUNCTION_EXISTS (btowc HAVE_BTOWC)
CHECK_FUNCTION_EXISTS (dup2 HAVE_DUP2)
CHECK_FUNCTION_EXISTS (error_at_line HAVE_ERROR_AT_LINE)
CHECK_FUNCTION_EXISTS (floor HAVE_FLOOR)
CHECK_FUNCTION_EXISTS (fork HAVE_FORK)
CHECK_FUNCTION_EXISTS (ftime HAVE_FTIME)
CHECK_FUNCTION_EXISTS (ftruncate HAVE_FTRUNCATE)
CHECK_FUNCTION_EXISTS (getenv HAVE_DECL_GETENV)
CHECK_FUNCTION_EXISTS (gethostbyname HAVE_GETHOSTBYNAME)
CHECK_FUNCTION_EXISTS (getpagesize HAVE_GETPAGESIZE)
CHECK_FUNCTION_EXISTS (gettimeofday HAVE_GETTIMEOFDAY)
CHECK_FUNCTION_EXISTS (inet_ntoa HAVE_INET_NTOA)
CHECK_FUNCTION_EXISTS (isascii HAVE_ISASCII)
CHECK_FUNCTION_EXISTS (localtime_r HAVE_LOCALTIME_R)
CHECK_FUNCTION_EXISTS (malloc HAVE_MALLOC)
CHECK_FUNCTION_EXISTS (mbsrtowcs HAVE_MBSRTOWCS)
CHECK_FUNCTION_EXISTS (memchr HAVE_MEMCHR)
CHECK_FUNCTION_EXISTS (memmove HAVE_MEMMOVE)
CHECK_FUNCTION_EXISTS (mempcpy HAVE_MEMPCPY)
CHECK_FUNCTION_EXISTS (memset HAVE_MEMSET)
CHECK_FUNCTION_EXISTS (mkdir HAVE_MKDIR)
CHECK_FUNCTION_EXISTS (mktime HAVE_MKTIME)
CHECK_FUNCTION_EXISTS (pow HAVE_POW)
CHECK_FUNCTION_EXISTS (regcomp HAVE_REGCOMP)
CHECK_FUNCTION_EXISTS (rmdir HAVE_RMDIR)
CHECK_FUNCTION_EXISTS (select HAVE_SELECT)
CHECK_FUNCTION_EXISTS (setenv HAVE_SETENV)
CHECK_FUNCTION_EXISTS (setlocale HAVE_SETLOCALE)
CHECK_FUNCTION_EXISTS (socket HAVE_SOCKET)
CHECK_FUNCTION_EXISTS (stat HAVE_STAT)
CHECK_FUNCTION_EXISTS (strcasecmp HAVE_STRCASECMP)
CHECK_FUNCTION_EXISTS (strchr HAVE_STRCHR)
CHECK_FUNCTION_EXISTS (strcspn HAVE_STRCSPN)
CHECK_FUNCTION_EXISTS (strdup HAVE_STRDUP)
CHECK_FUNCTION_EXISTS (strerror HAVE_STRERROR)
CHECK_FUNCTION_EXISTS (strerror_r HAVE_STRERROR_R)
CHECK_FUNCTION_EXISTS (strftime HAVE_STRFTIME)
CHECK_FUNCTION_EXISTS (strrchr HAVE_STRRCHR)
CHECK_FUNCTION_EXISTS (strspn HAVE_STRSPN)
CHECK_FUNCTION_EXISTS (strstr HAVE_STRSTR)
CHECK_FUNCTION_EXISTS (strtod HAVE_STRTOD)
CHECK_FUNCTION_EXISTS (strtol HAVE_STRTOL)
CHECK_FUNCTION_EXISTS (strtoul HAVE_STRTOUL)
CHECK_FUNCTION_EXISTS (strtoull HAVE_STRTOULL)
CHECK_FUNCTION_EXISTS (utime HAVE_UTIME)
CHECK_FUNCTION_EXISTS (vfork HAVE_VFORK)
CHECK_FUNCTION_EXISTS (wmempcpy HAVE_WMEMPCPY)
check_function_exists(alarm HAVE_ALARM)
check_function_exists(btowc HAVE_BTOWC)
check_function_exists(dup2 HAVE_DUP2)
check_function_exists(error_at_line HAVE_ERROR_AT_LINE)
check_function_exists(floor HAVE_FLOOR)
check_function_exists(fork HAVE_FORK)
check_function_exists(ftime HAVE_FTIME)
check_function_exists(ftruncate HAVE_FTRUNCATE)
check_function_exists(getenv HAVE_DECL_GETENV)
check_function_exists(gethostbyname HAVE_GETHOSTBYNAME)
check_function_exists(getpagesize HAVE_GETPAGESIZE)
check_function_exists(gettimeofday HAVE_GETTIMEOFDAY)
check_function_exists(inet_ntoa HAVE_INET_NTOA)
check_function_exists(isascii HAVE_ISASCII)
check_function_exists(localtime_r HAVE_LOCALTIME_R)
check_function_exists(malloc HAVE_MALLOC)
check_function_exists(mbsrtowcs HAVE_MBSRTOWCS)
check_function_exists(memchr HAVE_MEMCHR)
check_function_exists(memmove HAVE_MEMMOVE)
check_function_exists(mempcpy HAVE_MEMPCPY)
check_function_exists(memset HAVE_MEMSET)
check_function_exists(mkdir HAVE_MKDIR)
check_function_exists(mktime HAVE_MKTIME)
check_function_exists(pow HAVE_POW)
check_function_exists(regcomp HAVE_REGCOMP)
check_function_exists(rmdir HAVE_RMDIR)
check_function_exists(select HAVE_SELECT)
check_function_exists(setenv HAVE_SETENV)
check_function_exists(setlocale HAVE_SETLOCALE)
check_function_exists(socket HAVE_SOCKET)
check_function_exists(stat HAVE_STAT)
check_function_exists(strcasecmp HAVE_STRCASECMP)
check_function_exists(strchr HAVE_STRCHR)
check_function_exists(strcspn HAVE_STRCSPN)
check_function_exists(strdup HAVE_STRDUP)
check_function_exists(strerror HAVE_STRERROR)
check_function_exists(strerror_r HAVE_STRERROR_R)
check_function_exists(strftime HAVE_STRFTIME)
check_function_exists(strrchr HAVE_STRRCHR)
check_function_exists(strspn HAVE_STRSPN)
check_function_exists(strstr HAVE_STRSTR)
check_function_exists(strtod HAVE_STRTOD)
check_function_exists(strtol HAVE_STRTOL)
check_function_exists(strtoul HAVE_STRTOUL)
check_function_exists(strtoull HAVE_STRTOULL)
check_function_exists(utime HAVE_UTIME)
check_function_exists(vfork HAVE_VFORK)
check_function_exists(wmempcpy HAVE_WMEMPCPY)
CHECK_CXX_SYMBOL_EXISTS (alloca alloca.h HAVE_ALLOCA)
CHECK_CXX_SYMBOL_EXISTS (strerror_r string.h HAVE_DECL_STRERROR_R)
CHECK_CXX_SYMBOL_EXISTS (tm sys/time.h TM_IN_SYS_TIME)
check_cxx_symbol_exists(alloca alloca.h HAVE_ALLOCA)
check_cxx_symbol_exists(strerror_r string.h HAVE_DECL_STRERROR_R)
check_cxx_symbol_exists(tm sys/time.h TM_IN_SYS_TIME)
# AC_TYPE_SIGNAL
CHECK_TYPE_SIZE (ptrdiff_t PTRDIFF_T)
CHECK_TYPE_SIZE (_Bool __BOOL)
CHECK_TYPE_SIZE (mode_t mode_t_test)
IF(NOT HAVE_mode_t_test)
SET (mode_t int)
ENDIF()
CHECK_TYPE_SIZE(off_t off_t_test)
IF(NOT HAVE_off_t_test)
SET (off_t long int)
ENDIF()
CHECK_TYPE_SIZE(pid_t pid_t_test)
IF(NOT HAVE_pid_t_test)
SET (pid_t int)
ENDIF()
CHECK_TYPE_SIZE(size_t size_t_test)
IF(NOT HAVE_size_t_test)
SET (size_t unsigned int)
ENDIF()
check_type_size(ptrdiff_t PTRDIFF_T)
check_type_size(_Bool __BOOL)
check_type_size(mode_t mode_t_test)
if(NOT HAVE_mode_t_test)
set(mode_t int)
endif()
check_type_size(off_t off_t_test)
if(NOT HAVE_off_t_test)
set(off_t long int)
endif()
check_type_size(pid_t pid_t_test)
if(NOT HAVE_pid_t_test)
set(pid_t int)
endif()
check_type_size(size_t size_t_test)
if(NOT HAVE_size_t_test)
set(size_t unsigned int)
endif()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
@@ -152,70 +148,71 @@ main ()
;
return 0;
}" STDC_HEADERS)
}"
STDC_HEADERS
)
set(TEST_INCLUDES "
#include <stdio.h>"
)
SET (TEST_INCLUDES
"
#include <stdio.h>")
IF (HAVE_SYS_TYPES_H)
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <sys/types.h>")
ENDIF()
IF (HAVE_SYS_STAT_H)
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <sys/stat.h>")
ENDIF()
IF (STDC_HEADERS)
SET ( TEST_INCLUDES
if(HAVE_SYS_TYPES_H)
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <sys/types.h>"
)
endif()
if(HAVE_SYS_STAT_H)
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <sys/stat.h>"
)
endif()
if(STDC_HEADERS)
set(TEST_INCLUDES
"${TEST_INCLUDES}
# include <stdlib.h>
# include <stddef.h>")
ELSE()
IF()
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <stdlib.h>")
ENDIF()
ENDIF()
IF (HAVE_STRING_H)
IF(NOT STDC_HEADERS)
IF (HAVE_MEMORY_H)
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <memory.h>")
ENDIF()
ENDIF()
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <string.h>")
ENDIF()
IF (HAVE_STRINGS_H)
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <strings.h>")
ENDIF()
IF (HAVE_INTTYPES_H)
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <inttypes.h>")
ENDIF()
IF (HAVE_STDINT_H)
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <stdint.h>")
ENDIF()
IF (HAVE_UNISTD_H)
SET ( TEST_INCLUDES
"${TEST_INCLUDES}
# include <unistd.h>")
ENDIF()
# include <stddef.h>"
)
else()
if()
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <stdlib.h>"
)
endif()
endif()
if(HAVE_STRING_H)
if(NOT STDC_HEADERS)
if(HAVE_MEMORY_H)
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <memory.h>"
)
endif()
endif()
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <string.h>"
)
endif()
if(HAVE_STRINGS_H)
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <strings.h>"
)
endif()
if(HAVE_INTTYPES_H)
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <inttypes.h>"
)
endif()
if(HAVE_STDINT_H)
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <stdint.h>"
)
endif()
if(HAVE_UNISTD_H)
set(TEST_INCLUDES "${TEST_INCLUDES}
# include <unistd.h>"
)
endif()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
${TEST_INCLUDES}
# include <wchar.h>
@@ -225,10 +222,11 @@ main ()
mbstate_t x; return sizeof x;
;
return 0;
}" HAVE_MBSTATE_T)
}"
HAVE_MBSTATE_T
)
CHECK_CXX_SOURCE_RUNS(
check_cxx_source_runs(
"
${TEST_INCLUDES}
@@ -238,13 +236,14 @@ struct stat sbuf;
return stat (\"\", &sbuf) == 0;
;
return 0;
}" STAT_EMPTY_STRING_BUG)
IF (NOT STAT_EMPTY_STRING_BUG)
SET (HAVE_STAT_EMPTY_STRING_BUG 1)
ENDIF()
}"
STAT_EMPTY_STRING_BUG
)
if(NOT STAT_EMPTY_STRING_BUG)
set(HAVE_STAT_EMPTY_STRING_BUG 1)
endif()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
${TEST_INCLUDES}
#include <stdbool.h>
@@ -304,27 +303,31 @@ main ()
;
return 0;
}" HAVE_STDBOOL_H)
}"
HAVE_STDBOOL_H
)
IF (HAVE_UTIME_H)
CHECK_CXX_SOURCE_COMPILES(
if(HAVE_UTIME_H)
check_cxx_source_compiles(
"${TEST_INCLUDES}
# include <utime.h>
int
main ()
{
struct stat s, t;
return ! (stat (\"conftest.data\", &s) == 0
&& utime (\"conftest.data\", 0) == 0
&& stat (\"conftest.data\", &t) == 0
return ! (stat (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.data\", &s) == 0
&& utime (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.data\", 0) == 0
&& stat (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.data\", &t) == 0
&& t.st_mtime >= s.st_mtime
&& t.st_mtime - s.st_mtime < 120);
;
return 0;
}" HAVE_UTIME_NULL)
ENDIF()
}"
HAVE_UTIME_NULL
)
endif()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
${TEST_INCLUDES}
int
@@ -334,9 +337,11 @@ main ()
return fork () < 0;
;
return 0;
}" HAVE_WORKING_FORK)
}"
HAVE_WORKING_FORK
)
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"${TEST_INCLUDES}
#include <sys/wait.h>
#ifdef HAVE_VFORK_H
@@ -424,13 +429,15 @@ main ()
|| fstat(fileno(stdout), &st) != 0
);
}
}" HAVE_WORKING_VFORK)
}"
HAVE_WORKING_VFORK
)
IF (NOT HAVE_WORKING_VFORK)
SET (VFORK fork)
ENDIF()
if(NOT HAVE_WORKING_VFORK)
set(VFORK fork)
endif()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
#include <sys/types.h>
#include <signal.h>
@@ -441,21 +448,23 @@ main ()
return *(signal (0, 0)) (0) == 1;
;
return 0;
}" RET_SIGNAL_TYPES)
IF (RET_SIGNAL_TYPES)
SET (RETSIGTYPE int)
ELSE()
SET (RETSIGTYPE void)
ENDIF()
}"
RET_SIGNAL_TYPES
)
if(RET_SIGNAL_TYPES)
set(RETSIGTYPE int)
else()
set(RETSIGTYPE void)
endif()
# IF(NOT LSTAT_FOLLOWS_SLASHED_SYMLINK)
EXECUTE_PROCESS(
execute_process(
COMMAND rm -f conftest.sym conftest.file
COMMAND touch conftest.file
COMMAND ln -s conftest.file conftest.sym
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
)
CHECK_CXX_SOURCE_RUNS(
check_cxx_source_runs(
"
${TEST_INCLUDES}
int
@@ -465,30 +474,29 @@ struct stat sbuf;
/* Linux will dereference the symlink and fail, as required by POSIX.
That is better in the sense that it means we will not
have to compile and use the lstat wrapper. */
return lstat (\"conftest.sym/\", &sbuf) == 0;
return lstat (\"${CMAKE_CURRENT_BINARY_DIR}/conftest.sym/\", &sbuf) == 0;
;
return 0;
}" LSTAT_FOLLOWS_SLASHED_SYMLINK)
}"
LSTAT_FOLLOWS_SLASHED_SYMLINK
)
set(SELECT_INCLUDES ${TEST_INCLUDES})
if(HAVE_SYS_SELECT_H)
set(SELECT_INCULDES "${SELECT_INCLUDES}
# include <sys/select.h>"
)
endif()
if(HAVE_SYS_SOCKET_H)
set(SELECT_INCULDES "${SELECT_INCLUDES}
# include <sys/select.h>"
)
endif()
SET (SELECT_INCLUDES ${TEST_INCLUDES})
IF (HAVE_SYS_SELECT_H)
SET (SELECT_INCULDES
"${SELECT_INCLUDES}
# include <sys/select.h>")
ENDIF()
IF (HAVE_SYS_SOCKET_H)
SET (SELECT_INCULDES
"${SELECT_INCLUDES}
# include <sys/select.h>")
ENDIF()
FOREACH( ARG234 "fd_set *" "int *" "void *")
FOREACH( ARG1 "int" "size_t" "unsigned long int" "unsigned int")
FOREACH( ARG5 "struct timeval *" "const struct timeval *")
CHECK_CXX_SOURCE_COMPILES(
foreach(ARG234 "fd_set *" "int *" "void *")
foreach(ARG1 "int" "size_t" "unsigned long int" "unsigned int")
foreach(ARG5 "struct timeval *" "const struct timeval *")
check_cxx_source_compiles(
"
${SELECT_INCLUDES}
#ifdef HAVE_SYS_SELECT_H
@@ -507,25 +515,26 @@ extern int select (${ARG1},
;
return 0;
}
" SELECT_ARGS)
IF(SELECT_ARGS)
SET (SELECT_TYPE_ARG1 ${ARG1})
SET (SELECT_TYPE_ARG234 ${ARG234})
SET (SELECT_TYPE_ARG5 ${ARG5})
BREAK()
ENDIF()
ENDFOREACH()
IF(SELECT_ARGS)
BREAK()
ENDIF()
ENDFOREACH()
IF(SELECT_ARGS)
BREAK()
ENDIF()
ENDFOREACH()
"
SELECT_ARGS
)
if(SELECT_ARGS)
set(SELECT_TYPE_ARG1 ${ARG1})
set(SELECT_TYPE_ARG234 ${ARG234})
set(SELECT_TYPE_ARG5 ${ARG5})
break()
endif()
endforeach()
if(SELECT_ARGS)
break()
endif()
endforeach()
if(SELECT_ARGS)
break()
endif()
endforeach()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
#include <sys/types.h>
#include <sys/stat.h>
@@ -549,12 +558,14 @@ int main()
{
return 0;
}
" STATS_MACROS_CHECK)
IF (NOT STATS_MACROS_CHECK)
SET (STAT_MACROS_BROKEN 1)
ENDIF()
"
STATS_MACROS_CHECK
)
if(NOT STATS_MACROS_CHECK)
set(STAT_MACROS_BROKEN 1)
endif()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
${TEST_INCLUDES}
int
@@ -569,9 +580,11 @@ main ()
;
return 0;
}
" STRERROR_R_CHAR_P)
"
STRERROR_R_CHAR_P
)
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
#include <sys/types.h>
#include <sys/time.h>
@@ -585,10 +598,11 @@ return 0;
;
return 0;
}
" TIME_WITH_SYS_TIME)
"
TIME_WITH_SYS_TIME
)
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
int
main ()
@@ -647,12 +661,14 @@ main ()
;
return 0;
}
" CONST_CONFORM_CHECK)
IF (NOT CONST_CONFORM_CHECK)
SET (const "")
ENDIF()
"
CONST_CONFORM_CHECK
)
if(NOT CONST_CONFORM_CHECK)
set(const "")
endif()
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
int
main ()
@@ -664,14 +680,16 @@ return !x && !y;
;
return 0;
}
" WORKING_VOLATILE)
IF (NOT WORKING_VOLATILE)
SET (volatile "")
ENDIF()
"
WORKING_VOLATILE
)
if(NOT WORKING_VOLATILE)
set(volatile "")
endif()
FOREACH (RESTRICT_KW __restrict __restrict__ _Restrict restrict)
foreach(RESTRICT_KW __restrict __restrict__ _Restrict restrict)
CHECK_CXX_SOURCE_COMPILES(
check_cxx_source_compiles(
"
typedef int * int_ptr;
int foo (int_ptr ${RESTRICT_KW} ip) {
@@ -687,16 +705,17 @@ int s[1];
;
return 0;
}
" RESTRICT_CHECK)
IF (RESTRICT_CHECK)
SET (restrict ${RESTRICT_KW})
BREAK()
ENDIF()
ENDFOREACH()
"
RESTRICT_CHECK
)
if(RESTRICT_CHECK)
set(restrict ${RESTRICT_KW})
break()
endif()
endforeach()
FOREACH(INLINE_KW inline __inline__ __inline)
CHECK_CXX_SOURCE_COMPILES(
foreach(INLINE_KW inline __inline__ __inline)
check_cxx_source_compiles(
"
#ifndef __cplusplus
typedef int foo_t;
@@ -704,24 +723,24 @@ static ${INLINE_KW} foo_t static_foo () {return 0; }
${INLINE_KW} foo_t foo () {return 0; }
int main (){return 0;}
#endif
" INLINE)
IF (INLINE)
SET (inline ${INLINE_KW})
BREAK()
ENDIF()
ENDFOREACH()
IF (NOT INLINE)
SET (inline "")
ENDIF()
EXECUTE_PROCESS(
COMMAND rm -f conftest.data conftest.file conftest.sym
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
"
INLINE
)
CHECK_CXX_SOURCE_RUNS("
if(INLINE)
set(inline ${INLINE_KW})
break()
endif()
endforeach()
if(NOT INLINE)
set(inline "")
endif()
execute_process(COMMAND rm -f conftest.data conftest.file conftest.sym WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
check_cxx_source_runs(
"
#include <limits>
int main()
{
@@ -736,4 +755,28 @@ int main()
return 0;
return 1;
}"
MASK_LONGDOUBLE)
MASK_LONGDOUBLE
)
find_package(Git QUIET)
if(GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git)
execute_process(
COMMAND git describe --match=NeVeRmAtCh --always --dirty
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
OUTPUT_VARIABLE GIT_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE
)
else()
set(GIT_VERSION "source")
endif()
# releasenum is used by external scripts for various tasks. Leave it alone.
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE)
columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/build/releasenum ${ENGINE_SUPPORTDIR})
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE)
columnstore_install_file(${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine ${ENGINE_SUPPORTDIR})

20
cmake/dirs.cmake Normal file
View File

@@ -0,0 +1,20 @@
set(ENGINE_SYSCONFDIR "/etc")
set(ENGINE_DATADIR "/var/lib/columnstore")
set(ENGINE_LOGDIR "/var/log/mariadb/columnstore")
set(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR}")
set(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR}")
set(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR}")
set(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include")
set(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR}")
set(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR}")
set(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore")
set(ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
if(INSTALL_SYSCONF2DIR)
set(MARIADB_MYCNFDIR "${INSTALL_SYSCONF2DIR}")
else()
set(MARIADB_MYCNFDIR "/etc/mysql")
endif()
set(SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR})
set(SERVER_BUILD_DIR ${CMAKE_BINARY_DIR})

108
cmake/includes.cmake Normal file
View File

@@ -0,0 +1,108 @@
message_once(SERVER_BUILD_INCLUDE_DIR "SERVER_BUILD_INCLUDE_DIR = ${SERVER_BUILD_INCLUDE_DIR}")
message_once(SERVER_SOURCE_ROOT_DIR "SERVER_SOURCE_ROOT_DIR = ${SERVER_SOURCE_ROOT_DIR}")
set(ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp")
set(ENGINE_WE_SHARED_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/shared")
set(ENGINE_UTILS_IDBDATAFILE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/idbdatafile")
set(ENGINE_UTILS_LOGGINGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/loggingcpp"
"${CMAKE_CURRENT_BINARY_DIR}/utils/loggingcpp"
)
set(ENGINE_UTILS_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/configcpp")
set(ENGINE_UTILS_COMPRESS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/compress")
set(ENGINE_VERSIONING_BRM_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/versioning/BRM")
set(ENGINE_UTILS_ROWGROUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rowgroup")
set(ENGINE_UTILS_COMMON_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/common")
set(ENGINE_UTILS_DATACONVERT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/dataconvert")
set(ENGINE_UTILS_RWLOCK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/rwlock")
set(ENGINE_UTILS_FUNCEXP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/funcexp")
set(ENGINE_UTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils")
set(ENGINE_OAM_OAMCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/oam/oamcpp")
set(ENGINE_DBCON_DDLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackageproc")
set(ENGINE_DBCON_DDLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/ddlpackage")
set(ENGINE_DBCON_EXECPLAN_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/execplan")
set(ENGINE_UTILS_STARTUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/startup")
set(ENGINE_DBCON_JOBLIST_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/joblist")
set(ENGINE_WE_WRAPPER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/wrapper")
set(ENGINE_WE_SERVER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/server")
set(ENGINE_DBCON_DMLPKG_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackage")
set(ENGINE_WE_CLIENT_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/client")
set(ENGINE_DBCON_DMLPKGPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/dbcon/dmlpackageproc")
set(ENGINE_UTILS_CACHEUTILS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/cacheutils")
set(ENGINE_UTILS_MYSQLCL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/mysqlcl_idb")
set(ENGINE_UTILS_QUERYTELE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querytele")
set(ENGINE_UTILS_JOINER_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/joiner")
set(ENGINE_UTILS_THREADPOOL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/threadpool")
set(ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchloader")
set(ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup")
set(ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats")
set(ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client")
set(ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml")
set(ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes")
set(ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache")
set(ENGINE_PRIMPROC_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/primproc")
set(ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql")
set(ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include")
if(PCRE_INCLUDE_DIRS)
set(ENGINE_SERVER_PCRE_INCLUDE "${PCRE_INCLUDE_DIRS}")
else()
set(ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre")
endif()
set(ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include")
set(ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/")
set(ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk")
set(ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR})
set(ENGINE_COMMON_INCLUDES
${ENGINE_DEFAULT_INCLUDES}
${Boost_INCLUDE_DIRS}
${LIBXML2_INCLUDE_DIR}
${ENGINE_UTILS_MESSAGEQCPP_INCLUDE}
${ENGINE_WE_SHARED_INCLUDE}
${ENGINE_UTILS_IDBDATAFILE_INCLUDE}
${ENGINE_UTILS_LOGGINGCPP_INCLUDE}
${ENGINE_UTILS_CONFIGCPP_INCLUDE}
${ENGINE_UTILS_COMPRESS_INCLUDE}
${ENGINE_VERSIONING_BRM_INCLUDE}
${ENGINE_UTILS_ROWGROUP_INCLUDE}
${ENGINE_UTILS_COMMON_INCLUDE}
${ENGINE_UTILS_DATACONVERT_INCLUDE}
${ENGINE_UTILS_RWLOCK_INCLUDE}
${ENGINE_UTILS_FUNCEXP_INCLUDE}
${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE}
${ENGINE_UTILS_INCLUDE}
${ENGINE_OAM_OAMCPP_INCLUDE}
${ENGINE_DBCON_DDLPKGPROC_INCLUDE}
${ENGINE_DBCON_DDLPKG_INCLUDE}
${ENGINE_DBCON_EXECPLAN_INCLUDE}
${ENGINE_UTILS_STARTUP_INCLUDE}
${ENGINE_DBCON_JOBLIST_INCLUDE}
${ENGINE_WE_WRAPPER_INCLUDE}
${ENGINE_WE_SERVER_INCLUDE}
${ENGINE_DBCON_DMLPKG_INCLUDE}
${ENGINE_WE_CLIENT_INCLUDE}
${ENGINE_DBCON_DMLPKGPROC_INCLUDE}
${ENGINE_UTILS_CACHEUTILS_INCLUDE}
${ENGINE_UTILS_MYSQLCL_INCLUDE}
${ENGINE_UTILS_QUERYTELE_INCLUDE}
${ENGINE_UTILS_THRIFT_INCLUDE}
${ENGINE_UTILS_JOINER_INCLUDE}
${ENGINE_UTILS_THREADPOOL_INCLUDE}
${ENGINE_UTILS_BATCHLDR_INCLUDE}
${ENGINE_UTILS_DDLCLEANUP_INCLUDE}
${ENGINE_UTILS_QUERYSTATS_INCLUDE}
${ENGINE_WE_CONFIGCPP_INCLUDE}
${ENGINE_SERVER_SQL_INCLUDE}
${ENGINE_SERVER_INCLUDE_INCLUDE}
${ENGINE_SERVER_PCRE_INCLUDE}
${ENGINE_SERVER_WSREP_API_INCLUDE}
${ENGINE_SERVER_WSREP_INCLUDE}
${ENGINE_UTILS_UDFSDK_INCLUDE}
${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE}
${ENGINE_DATATYPES_INCLUDE}
)
set(LIBMARIADB_BININC_DIR ${CMAKE_BINARY_DIR}/libmariadb/include)
set(LIBMARIADB_SRCINC_DIR ${CMAKE_SOURCE_DIR}/libmariadb/include)
set(SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include)
set(SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR})

44
cmake/libs.cmake Normal file
View File

@@ -0,0 +1,44 @@
set(ENGINE_DT_LIB datatypes)
set(ENGINE_COMMON_LIBS
boost_thread
configcpp
idbboot
loggingcpp
messageqcpp
pthread
rt
xml2
${ENGINE_DT_LIB}
)
set(ENGINE_OAM_LIBS oamcpp)
set(ENGINE_BRM_LIBS brm cacheutils idbdatafile rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS})
set(PLUGIN_EXEC_LIBS
common
compress
dataconvert
execplan
funcexp
joiner
querytele
regr
rowgroup
threadpool
udfsdk
windowfunction
${ENGINE_BRM_LIBS}
)
set(ENGINE_EXEC_LIBS joblist querystats libmysql_client ${PLUGIN_EXEC_LIBS})
set(PLUGIN_WRITE_LIBS
cacheutils
ddlpackage
ddlpackageproc
dmlpackage
dmlpackageproc
idbdatafile
writeengine
writeengineclient
)
set(ENGINE_WRITE_LIBS ${PLUGIN_WRITE_LIBS} ${ENGINE_EXEC_LIBS})
set(MARIADB_CLIENT_LIBS libmariadb)
set(MARIADB_STRING_LIBS dbug strings mysys)

64
cmake/packages.cmake Normal file
View File

@@ -0,0 +1,64 @@
find_package(BISON)
if(NOT BISON_FOUND)
message_once(CS_NO_BISON "bison not found!")
return()
endif()
find_program(LEX_EXECUTABLE flex DOC "path to the flex executable")
if(NOT LEX_EXECUTABLE)
find_program(LEX_EXECUTABLE lex DOC "path to the lex executable")
if(NOT LEX_EXECUTABLE)
message_once(CS_NO_LEX "flex/lex not found!")
return()
endif()
endif()
find_package(LibXml2)
if(NOT LIBXML2_FOUND)
message_once(CS_NO_LIBXML "Could not find a usable libxml2 development environment!")
return()
endif()
find_package(Snappy)
if(NOT SNAPPY_FOUND)
message_once(
CS_NO_SNAPPY
"Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian"
)
return()
endif()
find_package(CURL)
if(NOT CURL_FOUND)
message_once(CS_NO_CURL "libcurl development headers not found")
return()
endif()
find_program(AWK_EXECUTABLE awk DOC "path to the awk executable")
if(NOT AWK_EXECUTABLE)
message_once(CS_NO_AWK "awk not found!")
return()
endif()
set(HAVE_LZ4
0
CACHE INTERNAL ""
)
if(WITH_COLUMNSTORE_LZ4 STREQUAL "ON" OR WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO")
find_package(LZ4)
if(NOT LZ4_FOUND)
if(WITH_COLUMNSTORE_LZ4 STREQUAL "AUTO")
message_once(CS_LZ4 "LZ4 not found, building without LZ4")
else()
message(FATAL_ERROR "LZ4 not found.")
endif()
else()
message_once(CS_LZ4 "Building with LZ4")
set(HAVE_LZ4
1
CACHE INTERNAL ""
)
endif()
else()
message_once(CS_LZ4 "Building without LZ4")
endif()

29
cmake/thrift.cmake
View File

@@ -1,12 +1,30 @@
set(WITH_THRIFT
"bundled"
CACHE STRING "Which Thrift to use (possible values are 'bundled', 'system', or 'auto')"
)
if(WITH_THRIFT STREQUAL "system" OR WITH_THRIFT STREQUAL "auto")
find_package(Thrift)
if(Thrift_FOUND)
add_custom_target(external_thrift)
set(THRIFT_INCLUDE_DIR "${THRIFT_INCLUDE_DIR}")
set(THRIFT_LIBRARY "${THRIFT_LIBRARIES}")
return()
elseif(WITH_THRIFT STREQUAL "system")
message(FATAL_ERROR "System Thrift requested but not found!")
endif()
endif()
include(ExternalProject)
set(INSTALL_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/external/thrift)
SET(THRIFT_INCLUDE_DIRS "${INSTALL_LOCATION}/include")
SET(THRIFT_LIBRARY_DIRS "${INSTALL_LOCATION}/lib")
set(THRIFT_INCLUDE_DIRS "${INSTALL_LOCATION}/include")
set(THRIFT_LIBRARY_DIRS "${INSTALL_LOCATION}/lib")
set(THRIFT_LIBRARY ${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${CMAKE_STATIC_LIBRARY_SUFFIX})
ExternalProject_Add(external_thrift
ExternalProject_Add(
external_thrift
URL https://github.com/apache/thrift/archive/refs/tags/v0.17.0.tar.gz
URL_HASH SHA256=f5888bcd3b8de40c2c2ab86896867ad9b18510deb412cba3e5da76fb4c604c29
PREFIX ${INSTALL_LOCATION}
@@ -22,8 +40,7 @@ ExternalProject_Add(external_thrift
-DBUILD_TESTING=NO
-DBUILD_SHARED_LIBS=NO
-DCMAKE_CXX_FLAGS:STRING="-fPIC"
-DBOOST_INCLUDEDIR=${Boost_INCLUDE_DIRS}
-DBOOST_LIBRARYDIR=${Boost_LIBRARY_DIRS}
-DBOOST_ROOT=${BOOST_ROOT}
BUILD_BYPRODUCTS "${THRIFT_LIBRARY_DIRS}/${CMAKE_STATIC_LIBRARY_PREFIX}thrift${CMAKE_STATIC_LIBRARY_SUFFIX}"
EXCLUDE_FROM_ALL TRUE
)

55
core_dumps/columnstore_schema.sql Normal file
View File

@@ -0,0 +1,55 @@
DROP DATABASE IF EXISTS `columnstore_bts`;
CREATE DATABASE `columnstore_bts`;
USE `columnstore_bts`;
CREATE TABLE `airlines` (
`iata_code` varchar(2) DEFAULT NULL,
`airline` varchar(30) DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci;
CREATE TABLE `airports` (
`iata_code` varchar(3) DEFAULT NULL,
`airport` varchar(80) DEFAULT NULL,
`city` varchar(30) DEFAULT NULL,
`state` varchar(2) DEFAULT NULL,
`country` varchar(30) DEFAULT NULL,
`latitude` decimal(11,4) DEFAULT NULL,
`longitude` decimal(11,4) DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci;
CREATE TABLE `flights` (
`year` smallint(6) DEFAULT NULL,
`month` tinyint(4) DEFAULT NULL,
`day` tinyint(4) DEFAULT NULL,
`day_of_week` tinyint(4) DEFAULT NULL,
`fl_date` date DEFAULT NULL,
`carrier` varchar(2) DEFAULT NULL,
`tail_num` varchar(6) DEFAULT NULL,
`fl_num` smallint(6) DEFAULT NULL,
`origin` varchar(5) DEFAULT NULL,
`dest` varchar(5) DEFAULT NULL,
`crs_dep_time` varchar(4) DEFAULT NULL,
`dep_time` varchar(4) DEFAULT NULL,
`dep_delay` decimal(13,2) DEFAULT NULL,
`taxi_out` decimal(13,2) DEFAULT NULL,
`wheels_off` varchar(4) DEFAULT NULL,
`wheels_on` varchar(4) DEFAULT NULL,
`taxi_in` decimal(13,2) DEFAULT NULL,
`crs_arr_time` varchar(4) DEFAULT NULL,
`arr_time` varchar(4) DEFAULT NULL,
`arr_delay` decimal(13,2) DEFAULT NULL,
`cancelled` decimal(13,2) DEFAULT NULL,
`cancellation_code` varchar(20) DEFAULT NULL,
`diverted` decimal(13,2) DEFAULT NULL,
`crs_elapsed_time` decimal(13,2) DEFAULT NULL,
`actual_elapsed_time` decimal(13,2) DEFAULT NULL,
`air_time` decimal(13,2) DEFAULT NULL,
`distance` decimal(13,2) DEFAULT NULL,
`carrier_delay` decimal(13,2) DEFAULT NULL,
`weather_delay` decimal(13,2) DEFAULT NULL,
`nas_delay` decimal(13,2) DEFAULT NULL,
`security_delay` decimal(13,2) DEFAULT NULL,
`late_aircraft_delay` decimal(13,2) DEFAULT NULL
) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci;

38
core_dumps/docker-awaiter.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/bin/bash
DOCKER_IMAGE=$1
zhdun()
{
command=$1
expected_result=$2
waiting_message=$3
retries=$4
sleep_delay=$5
result=$($command)
result="${result%%[[:cntrl:]]}"
retries_counter=1
while true;
do
if [ "$result" != "$expected_result" ]; then
echo $waiting_message " Status: " $result ", attempt: " $retries_counter
sleep $sleep_delay
else
echo Finished waiting for \'"$command"\' to return \'"$expected_result"\'
exit
fi
if [ $retries_counter -ge $retries ]; then
echo "Tired to wait for retry, $retries_counter attemps were made"
exit
fi
retries_counter=$(($retries_counter + 1))
done
}
check_result="running"
check_command="docker exec -t $DOCKER_IMAGE systemctl is-system-running"
waiting_message="Waiting for docker container to start systemd."
zhdun "$check_command" "$check_result" "$waiting_message" 60 2

24
core_dumps/save_stack.sh Normal file
View File

@@ -0,0 +1,24 @@
#!/usr/bin/env sh
set -x
set -o pipefail
SCRIPT_LOCATION=$(dirname "$0")
LOG_PATH=$1
dump_stack ()
{
name=$1
echo "\nStack trace of $1"
eu-stack -p `pidof $name` -n 0 | tee ${LOG_PATH}/${name}_callstacks.txt
}
dump_stack "mariadbd"
dump_stack "workernode"
dump_stack "controllernode"
dump_stack "WriteEngineServer"
dump_stack "DDLProc"
dump_stack "DMLProc"
dump_stack "PrimProc"

45
core_dumps/upgrade_data.sh Executable file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -xeuo pipefail
MARIADB=$(which mysql)
CPIMPORT=$(which cpimport)
SCHEMA_DIR=$(dirname "$0")
NAME1='airports'
NAME2='airlines'
get_data ()
{
NAME=$1
if curl -o "${SCHEMA_DIR}/${NAME}.csv" -# "https://sample-columnstore-data.s3.us-west-2.amazonaws.com/${NAME}.csv"; then
echo -e "Downloaded '${NAME}.csv' ... done\n"
else
echo -e "Downloading '${NAME}.csv' ... failed"
exit 1
fi
}
import_data ()
{
NAME=$1
echo -e "\nLoading '${NAME}.csv' with cpimport ..."
if ! $CPIMPORT -m 1 -s ',' -E '"' columnstore_bts "${NAME}" -l "${SCHEMA_DIR}/${NAME}.csv"; then
echo -e "loading '${NAME}.csv' ... fail"
exit 1
fi
}
if $MARIADB <"${SCHEMA_DIR}"/columnstore_schema.sql &>/dev/null; then
echo -e "Creating 'columnstore_bts' schema..." "done"
else
echo -e "Creating 'columnstore_bts' schema..." "fail"
exit 1
fi
get_data "$NAME1"
get_data "$NAME2"
import_data "$NAME1"
import_data "$NAME2"

62
core_dumps/upgrade_setup_deb.sh Executable file
View File

@@ -0,0 +1,62 @@
#!/usr/bin/env bash
source ./utils.sh
set -xeuo pipefail
VERSION="$1"
RESULT="$2"
ARCH="$3"
LINK="$4"
UPGRADE_TOKEN="$5"
DEBIAN_FRONTEND=noninteractive
UCF_FORCE_CONFNEW=1
apt install --yes rsyslog
sed -i "s/exit 101/exit 0/g" /usr/sbin/policy-rc.d
bash -c "apt update --yes && apt install -y procps wget curl"
wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup
chmod +x mariadb_es_repo_setup
bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools"
apt update --yes
apt install --yes -oDebug::RunScripts=1 mariadb-server mariadb-client mariadb-plugin-columnstore
systemctl start mariadb
systemctl start mariadb-columnstore
INITIAL_VERSION=$(mariadb -e "select @@version;")
bash -c "./upgrade_data.sh"
bash -c "./upgrade_verify.sh"
touch /etc/apt/auth.conf
cat << EOF > /etc/apt/auth.conf
machine ${LINK}${RESULT}/
EOF
bash -c "./setup-repo.sh"
# Configuration file '/etc/columnstore/Columnstore.xml'
# ==> Modified (by you or by a script) since installation.
# ==> Package distributor has shipped an updated version.
# What would you like to do about it ? Your options are:
# Y or I : install the package maintainer's version
# N or O : keep your currently-installed version
# D : show the differences between the versions
# Z : start a shell to examine the situation
# The default action is to keep your current version.
# the -o options are used to make choise of keep your currently-installed version without interactive prompt
apt-get --yes --with-new-pkgs -oDebug::RunScripts=1 -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade
UPGRADED_VERSION=$(mariadb -e "select @@version;")
if [[ "$INITIAL_VERSION" == "$UPGRADED_VERSION" ]]; then
error "The upgrade didn't happen!"
exit 1
else
message_splitted "The upgrade from "$INITIAL_VERSION" to "$UPGRADED_VERSION" succeded!"
bash -c "./upgrade_verify.sh"
fi

42
core_dumps/upgrade_setup_rpm.sh Executable file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env bash
source ./utils.sh
set -xeuo pipefail
VERSION="$1"
RESULT="$2"
ARCH="$3"
LINK="$4"
UPGRADE_TOKEN="$5"
yum clean all
yum install -y wget which procps-ng diffutils rsyslog
wget https://dlm.mariadb.com/enterprise-release-helpers/mariadb_es_repo_setup -O mariadb_es_repo_setup
chmod +x mariadb_es_repo_setup
bash -c "./mariadb_es_repo_setup --token=${UPGRADE_TOKEN} --apply --mariadb-server-version=${VERSION} --skip-maxscale --skip-tools --skip-check-installed"
yum repo-pkgs mariadb-es-main list
yum -y install MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
systemctl start mariadb
systemctl start mariadb-columnstore
INITIAL_VERSION=$(mariadb -e "select @@version;")
bash -c "./upgrade_data.sh"
bash -c "./upgrade_verify.sh"
bash -c "./setup-repo.sh"
yum repo-pkgs repo list
yum -y update MariaDB-server MariaDB-client MariaDB-columnstore-engine MariaDB-columnstore-engine-debuginfo
UPGRADED_VERSION=$(mariadb -e "select @@version;")
if [[ "$INITIAL_VERSION" == "$UPGRADED_VERSION" ]]; then
error "The upgrade didn't happen!"
exit 1
else
message_splitted "The upgrade from "$INITIAL_VERSION" to "$UPGRADED_VERSION" succeded!"
bash -c "./upgrade_verify.sh"
fi

20
core_dumps/upgrade_verify.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -xeuo pipefail
SCHEMA_DIR=$(dirname "$0")
NAME1='airports'
NAME2='airlines'
test_data ()
{
NAME=$1
mariadb --init-command="SET sql_mode=''" -vvv -e "select * into outfile '/tmp/${NAME}.test.csv' FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' FROM columnstore_bts.${NAME};"
diff <(tail -n +2 "${NAME}.csv") <(tail -n +2 "/tmp/${NAME}.test.csv")
rm "/tmp/${NAME}.test.csv"
}
test_data "$NAME1"
test_data "$NAME2"

36
tests/CMakeLists.txt
View File

@@ -1,23 +1,25 @@
include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE})
MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO)
MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC")
if(COLUMNSTORE_ASAN_FOR_UNITTESTS)
my_check_and_set_compiler_flag("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO)
my_check_and_set_compiler_flag("-fsanitize=address -fsanitize-address-use-after-scope -fPIC")
endif()
if(WITH_UNITTESTS)
cmake_policy(SET CMP0054 NEW)
set(EXTERNAL_INSTALL_LOCATION ${CMAKE_BINARY_DIR}/external)
ExternalProject_Add(googletest
ExternalProject_Add(
googletest
GIT_REPOSITORY https://github.com/google/googletest
GIT_TAG release-1.12.0
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} -DBUILD_SHARED_LIBS=ON
)
# this is a workaround for gtest's bug still not closed
# https://github.com/google/googletest/issues/3659
# this is a workaround for gtest's bug still not closed https://github.com/google/googletest/issues/3659
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
MY_CHECK_AND_SET_COMPILER_FLAG("-U__cpp_char8_t")
my_check_and_set_compiler_flag("-U__cpp_char8_t")
endif()
include_directories(${EXTERNAL_INSTALL_LOCATION}/include)
# lib64 for RPM-based distros
link_directories(${EXTERNAL_INSTALL_LOCATION}/lib ${EXTERNAL_INSTALL_LOCATION}/lib64)
@@ -30,7 +32,6 @@ if (WITH_UNITTESTS)
target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS})
gtest_add_tests(TARGET rowgroup_tests TEST_PREFIX columnstore:)
add_executable(rewritetest rewritetest.cpp)
add_dependencies(rewritetest googletest)
target_link_libraries(rewritetest ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} messageqcpp execplan)
@@ -59,7 +60,9 @@ if (WITH_UNITTESTS)
add_executable(column_scan_filter_tests primitives_column_scan_and_filter.cpp)
target_compile_options(column_scan_filter_tests PRIVATE -Wno-error -Wno-sign-compare)
add_dependencies(column_scan_filter_tests googletest)
target_link_libraries(column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc)
target_link_libraries(
column_scan_filter_tests ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc
)
gtest_add_tests(TARGET column_scan_filter_tests TEST_PREFIX columnstore:)
add_executable(simd_processors simd_processors.cpp)
@@ -99,8 +102,17 @@ endif()
if(WITH_MICROBENCHMARKS AND (NOT CMAKE_BUILD_TYPE STREQUAL "Debug"))
find_package(benchmark REQUIRED)
add_executable(primitives_scan_bench primitives_scan_bench.cpp)
target_include_directories(primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} )
target_link_libraries(primitives_scan_bench ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${GTEST_LIBRARIES} processor dbbc benchmark::benchmark)
target_include_directories(
primitives_scan_bench PUBLIC ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE}
)
target_link_libraries(
primitives_scan_bench
${ENGINE_LDFLAGS}
${ENGINE_WRITE_LIBS}
${GTEST_LIBRARIES}
processor
dbbc
benchmark::benchmark
)
add_test(NAME columnstore_microbenchmarks:primitives_scan_bench, COMMAND primitives_scan_bench)
endif()

2
tests/counting_allocator.cpp
View File

@@ -32,7 +32,7 @@ struct TestClass
{
int value[1024];
TestClass(int val) : value(val)
TestClass(int val) : value{val}
{
}
};

14
utils/funcexp/CMakeLists.txt
View File

@@ -1,7 +1,5 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
# func_decode_oracle.cpp
@@ -144,13 +142,17 @@ set(funcexp_LIB_SRCS
func_year.cpp
func_yearweek.cpp
jsonhelpers.cpp
sql_crypt.cpp)
sql_crypt.cpp
)
add_library(funcexp SHARED ${funcexp_LIB_SRCS})
add_dependencies(funcexp loggingcpp)
target_link_libraries(funcexp ${NETSNMP_LIBRARIES})
install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
target_link_libraries(funcexp ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS})
install(
TARGETS funcexp
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)