From 3ce19abdae59e9ae64f9fec35b7504b5b88dcaf4 Mon Sep 17 00:00:00 2001 From: Leonid Fedorov <79837786+mariadb-LeonidFedorov@users.noreply.github.com> Date: Fri, 21 Apr 2023 21:24:48 +0300 Subject: [PATCH] Options to build with TSAN, UBSAN and skipping smoke (#2826) --- build/bootstrap_mcs.sh | 69 ++++++++++++++++++++++-------- dbcon/ddlpackage/ddlpkg.h | 4 +- dbcon/mysql/ha_mcs.cpp | 4 +- oam/install_scripts/CMakeLists.txt | 51 ++++++++++++++++++---- tests/CMakeLists.txt | 7 ++- writeengine/shared/we_define.h | 1 + writeengine/shared/we_fileop.cpp | 11 +++++ 7 files changed, 115 insertions(+), 32 deletions(-) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 7378d732a..8911fe8fe 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -34,11 +34,14 @@ optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true -optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true +optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true +optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true +optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true optparse.define short=T long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true +optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true source $( optparse.build ) @@ -207,18 +210,26 @@ build() MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja" fi - if [[ $ASAN = true ]] ; then - warn "Building with ASAN" + warn "Building with Address Sanitizer " MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" fi + if [[ $TSAN = true ]] ; then + warn "Building with Thread Sanitizer" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + fi + + if [[ $UBSAN = true ]] ; then + warn "Building with UB Sanitizer" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + fi + if [[ $WITHOUT_COREDUMPS = true ]] ; then warn "Cores are not dumped" else MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON" - - warn Builds with boreDumps CoreDump pattern changed to ${REPORT_PATH}/core_%e.%p + warn Building with CoreDumps: /proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern fi @@ -350,8 +361,10 @@ fix_config_files() THREAD_STACK_SIZE="20M" SYSTEMD_SERVICE_DIR="/usr/lib/systemd/system" + MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service + COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf + if [[ $ASAN = true ]] ; then - COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf if grep -q thread_stack $COLUMNSTORE_CONFIG; then warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN" else @@ -359,14 +372,33 @@ fix_config_files() message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG" fi - MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service if grep -q ASAN $MDB_SERVICE_FILE; then warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility" else - echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}'" >> $MDB_SERVICE_FILE + echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${REPORT_PATH}/asan.mariadb'" >> $MDB_SERVICE_FILE message "ASAN options were added to $MDB_SERVICE_FILE" fi fi + + if [[ $TSAN = true ]] ; then + if grep -q TSAN $MDB_SERVICE_FILE; then + warn "MDB Server has TSAN options in $MDB_SERVICE_FILE, check it's compatibility" + else + echo Environment="'TSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/tsan.mariadb'" >> $MDB_SERVICE_FILE + message "TSAN options were added to $MDB_SERVICE_FILE" + fi + fi + + if [[ $UBSAN = true ]] ; then + if grep -q UBSAN $MDB_SERVICE_FILE; then + warn "MDB Server has UBSAN options in $MDB_SERVICE_FILE, check it's compatibility" + else + echo Environment="'UBSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/ubsan.mariadb'" >> $MDB_SERVICE_FILE + message "UBSAN options were added to $MDB_SERVICE_FILE" + fi + fi + + message Reloading systemd systemctl daemon-reload } @@ -452,14 +484,17 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' smoke() { - message "Creating test database" - mariadb -e "create database if not exists test;" - message "Selecting magic numbers" - MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql` - if [[ $MAGIC == '42' ]] ; then - message "Great answer correct" - else - warn "Smoke failed, answer is '$MAGIC'" + if [[ $SKIP_SMOKE = false ]] ; then + message_split + message "Creating test database" + mariadb -e "create database if not exists test;" + message "Selecting magic numbers" + MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql` + if [[ $MAGIC == '42' ]] ; then + message "Great answer correct" + else + warn "Smoke failed, answer is '$MAGIC'" + fi fi } @@ -491,6 +526,4 @@ start_service smoke generate_svgs - - message_splitted "FINISHED" \ No newline at end of file diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index c775a6dbb..bada82db3 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -928,10 +928,10 @@ struct AtaRenameColumn : public AlterTableAction struct ColumnType { /** @brief Deserialize from ByteStream */ - EXPORT virtual int unserialize(messageqcpp::ByteStream& bs); + EXPORT int unserialize(messageqcpp::ByteStream& bs); /** @brief Serialize to ByteStream */ - EXPORT virtual int serialize(messageqcpp::ByteStream& bs); + EXPORT int serialize(messageqcpp::ByteStream& bs); /** @brief For deserialization. */ ColumnType() : fCharset(NULL), fExplicitLength(false) diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index a9c319f6d..c735dbc45 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -1809,10 +1809,10 @@ static int columnstore_init_func(void* p) fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(), columnstore_release.c_str()); - strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version)); + strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version) - 1); cs_version[sizeof(cs_version) - 1] = 0; - strncpy(cs_commit_hash, columnstore_commit_hash.c_str(), sizeof(cs_commit_hash)); + strncpy(cs_commit_hash, columnstore_commit_hash.c_str(), sizeof(cs_commit_hash) - 1); cs_commit_hash[sizeof(cs_commit_hash) - 1] = 0; mcs_hton = (handlerton*)p; diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index e61184af4..964d1a55f 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -27,16 +27,51 @@ if (WITH_COLUMNSTORE_ASAN) SET(LD_PRELOAD_STRING "") - SET(ALLOC_CONFIG "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}") - SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) - SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) - SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) - SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG}) - SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) - SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) - SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG}) + SET(ALLOC_CONFIG "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false") + SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.primproc) + SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.dmlproc) + SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.ddlproc) + SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.writeengine) + SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.controllernode) + SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.workernode) + SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.storagemanager) endif() +if (WITH_UBSAN) + SET(UBSAN_PATH "/tmp/asan") + if (WITH_COLUMNSTORE_REPORT_PATH) + SET(UBSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/ubsan") + endif (WITH_COLUMNSTORE_REPORT_PATH) + + SET(LD_PRELOAD_STRING "") + SET(ALLOC_CONFIG "UBSAN_OPTIONS=abort_on_error=0") + SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.primproc) + SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.dmlproc) + SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.ddlproc) + SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.writeengine) + SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.controllernode) + SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.workernode) + SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.storagemanager) +endif() + +if (WITH_TSAN) + SET(TSAN_PATH "/tmp/tsan") + if (WITH_COLUMNSTORE_REPORT_PATH) + SET(TSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/tsan") + endif (WITH_COLUMNSTORE_REPORT_PATH) + + SET(LD_PRELOAD_STRING "") + SET(ALLOC_CONFIG "TSAN_OPTIONS=abort_on_error=0:log_path=${TSAN_PATH}") + SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.primproc) + SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.dmlproc) + SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.ddlproc) + SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.writeengine) + SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.controllernode) + SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.workernode) + SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.storagemanager) +endif() + + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall" @ONLY) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 385a53cc1..2de67c70b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,6 +1,9 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} ) -MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) -MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") + +IF(NOT WITH_UBSAN AND NOT WITH_TSAN) + MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) + MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") +ENDIF() if (WITH_UNITTESTS) cmake_policy(SET CMP0054 NEW) diff --git a/writeengine/shared/we_define.h b/writeengine/shared/we_define.h index 1716c6856..c433bd722 100644 --- a/writeengine/shared/we_define.h +++ b/writeengine/shared/we_define.h @@ -151,6 +151,7 @@ const int ERR_FILE_FLUSH = ERR_FILEBASE + 18; // Error flushing file const int ERR_FILE_GLOBBING = ERR_FILEBASE + 19; // Error globbing a file name const int ERR_FILE_EOF = ERR_FILEBASE + 20; // EOF const int ERR_FILE_CHOWN = ERR_FILEBASE + 21; // EOF +const int ERR_INTERNAL = ERR_FILEBASE + 22; // EOF //-------------------------------------------------------------------------- // XML level error diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 0797b5d63..da725ff03 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -2105,6 +2105,11 @@ int FileOp::oid2FileName(FID fid, char* fullFileName, bool bCreateDir, uint16_t RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, partition, segment))); // see if file exists in specified DBRoot; return if found + if (fullFileName == nullptr) + { + return ERR_INTERNAL; + } + if (dbRoot > 0) { sprintf(fullFileName, "%s/%s", Config::getDBRootByNum(dbRoot).c_str(), tempFileName); @@ -2220,6 +2225,12 @@ int FileOp::oid2DirName(FID fid, char* oidDirName) const return NO_ERROR; } + + if (oidDirName == nullptr) + { + return ERR_INTERNAL; + } + RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, 0, 0))); // Now try to find the directory in each of the DBRoots.