Options to build with TSAN, UBSAN and skipping smoke (#2826)

build/bootstrap_mcs.sh

@@ -34,11 +34,14 @@ optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP
 optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
 optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH
 optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
-optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
 optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
+optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
+optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=false value=true
+optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
 optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core"
 optparse.define short=N long=ninja desc="Build with ninja" variable=USE_NINJA default=false value=true
 optparse.define short=T long=draw-deps desc="Draw dependencies graph" variable=DRAW_DEPS default=false value=true
+optparse.define short=M long=skip-smoke desc="Skip final smoke test" variable=SKIP_SMOKE default=false value=true
 
 source $( optparse.build )
 
@@ -207,18 +210,26 @@ build()
         MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -GNinja"
     fi
 
-
     if [[ $ASAN = true ]] ; then
-        warn "Building with ASAN"
+        warn "Building with Address Sanitizer "
         MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
     fi
 
+    if [[ $TSAN = true ]] ; then
+        warn "Building with Thread Sanitizer"
+        MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_TSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
+    fi
+
+    if [[ $UBSAN = true ]] ; then
+        warn "Building with UB Sanitizer"
+        MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_UBSAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}"
+    fi
+
     if [[ $WITHOUT_COREDUMPS = true ]] ; then
         warn "Cores are not dumped"
     else
         MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON"
-
+        warn Building with CoreDumps: /proc/sys/kernel/core_pattern changed to ${REPORT_PATH}/core_%e.%p
-        warn Builds with boreDumps CoreDump pattern changed to ${REPORT_PATH}/core_%e.%p
         echo "${REPORT_PATH}/core_%e.%p" > /proc/sys/kernel/core_pattern
     fi
 
@@ -350,8 +361,10 @@ fix_config_files()
     THREAD_STACK_SIZE="20M"
 
     SYSTEMD_SERVICE_DIR="/usr/lib/systemd/system"
+    MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service
+    COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf
+
     if [[ $ASAN = true ]] ; then
-        COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf
         if grep -q thread_stack $COLUMNSTORE_CONFIG; then
             warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN"
         else
@@ -359,14 +372,33 @@ fix_config_files()
             message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG"
         fi
 
-        MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service
         if grep -q ASAN $MDB_SERVICE_FILE; then
             warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility"
         else
-            echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}'" >> $MDB_SERVICE_FILE
+            echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${REPORT_PATH}/asan.mariadb'" >> $MDB_SERVICE_FILE
             message "ASAN options were added to $MDB_SERVICE_FILE"
         fi
     fi
+
+    if [[ $TSAN = true ]] ; then
+        if grep -q TSAN $MDB_SERVICE_FILE; then
+            warn "MDB Server has TSAN options in $MDB_SERVICE_FILE, check it's compatibility"
+        else
+            echo Environment="'TSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/tsan.mariadb'" >> $MDB_SERVICE_FILE
+            message "TSAN options were added to $MDB_SERVICE_FILE"
+        fi
+    fi
+
+    if [[ $UBSAN = true ]] ; then
+        if grep -q UBSAN $MDB_SERVICE_FILE; then
+            warn "MDB Server has UBSAN options in $MDB_SERVICE_FILE, check it's compatibility"
+        else
+            echo Environment="'UBSAN_OPTIONS=abort_on_error=0,log_path=${REPORT_PATH}/ubsan.mariadb'" >> $MDB_SERVICE_FILE
+            message "UBSAN options were added to $MDB_SERVICE_FILE"
+        fi
+    fi
+
+    message Reloading systemd
     systemctl daemon-reload
 }
 
@@ -452,14 +484,17 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf'
 
 smoke()
 {
-    message "Creating test database"
+    if [[ $SKIP_SMOKE = false ]] ; then
-    mariadb -e "create database if not exists test;"
+        message_split
-    message "Selecting magic numbers"
+        message "Creating test database"
-    MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql`
+        mariadb -e "create database if not exists test;"
-    if [[ $MAGIC == '42' ]] ; then
+        message "Selecting magic numbers"
-        message "Great answer correct"
+        MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql`
-    else
+        if [[ $MAGIC == '42' ]] ; then
-        warn "Smoke failed, answer is '$MAGIC'"
+            message "Great answer correct"
+        else
+            warn "Smoke failed, answer is '$MAGIC'"
+        fi
     fi
 }
 
@@ -491,6 +526,4 @@ start_service
 smoke
 generate_svgs
 
-
-
 message_splitted "FINISHED"

dbcon/ddlpackage/ddlpkg.h

@@ -928,10 +928,10 @@ struct AtaRenameColumn : public AlterTableAction
 struct ColumnType
 {
   /** @brief Deserialize from ByteStream */
-  EXPORT virtual int unserialize(messageqcpp::ByteStream& bs);
+  EXPORT int unserialize(messageqcpp::ByteStream& bs);
 
   /** @brief Serialize to ByteStream */
-  EXPORT virtual int serialize(messageqcpp::ByteStream& bs);
+  EXPORT int serialize(messageqcpp::ByteStream& bs);
 
   /** @brief For deserialization. */
   ColumnType() : fCharset(NULL), fExplicitLength(false)

dbcon/mysql/ha_mcs.cpp

@@ -1809,10 +1809,10 @@ static int columnstore_init_func(void* p)
   fprintf(stderr, "Columnstore: Started; Version: %s-%s\n", columnstore_version.c_str(),
           columnstore_release.c_str());
 
-  strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version));
+  strncpy(cs_version, columnstore_version.c_str(), sizeof(cs_version) - 1);
   cs_version[sizeof(cs_version) - 1] = 0;
 
-  strncpy(cs_commit_hash, columnstore_commit_hash.c_str(), sizeof(cs_commit_hash));
+  strncpy(cs_commit_hash, columnstore_commit_hash.c_str(), sizeof(cs_commit_hash) - 1);
   cs_commit_hash[sizeof(cs_commit_hash) - 1] = 0;
 
   mcs_hton = (handlerton*)p;

oam/install_scripts/CMakeLists.txt

@@ -27,16 +27,51 @@ if (WITH_COLUMNSTORE_ASAN)
 
 
   SET(LD_PRELOAD_STRING "")
-  SET(ALLOC_CONFIG "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}")
+  SET(ALLOC_CONFIG "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false")
-  SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
+  SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.primproc)
-  SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
+  SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.dmlproc)
-  SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG})
+  SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.ddlproc)
-  SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG})
+  SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.writeengine)
-  SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG})
+  SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.controllernode)
-  SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG})
+  SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.workernode)
-  SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG})
+  SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${ASAN_PATH}.storagemanager)
 endif()
 
+if (WITH_UBSAN)
+  SET(UBSAN_PATH "/tmp/asan")
+  if (WITH_COLUMNSTORE_REPORT_PATH)
+    SET(UBSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/ubsan")
+  endif (WITH_COLUMNSTORE_REPORT_PATH)
+
+  SET(LD_PRELOAD_STRING "")
+  SET(ALLOC_CONFIG "UBSAN_OPTIONS=abort_on_error=0")
+  SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.primproc)
+  SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.dmlproc)
+  SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.ddlproc)
+  SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.writeengine)
+  SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.controllernode)
+  SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.workernode)
+  SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${UBSAN_PATH}.storagemanager)
+endif()
+
+if (WITH_TSAN)
+  SET(TSAN_PATH "/tmp/tsan")
+  if (WITH_COLUMNSTORE_REPORT_PATH)
+    SET(TSAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/tsan")
+  endif (WITH_COLUMNSTORE_REPORT_PATH)
+
+  SET(LD_PRELOAD_STRING "")
+  SET(ALLOC_CONFIG "TSAN_OPTIONS=abort_on_error=0:log_path=${TSAN_PATH}")
+  SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.primproc)
+  SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.dmlproc)
+  SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.ddlproc)
+  SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.writeengine)
+  SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.controllernode)
+  SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.workernode)
+  SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG},log_path=${TSAN_PATH}.storagemanager)
+endif()
+
+
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall" @ONLY)

tests/CMakeLists.txt

@@ -1,6 +1,9 @@
 include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_BLOCKCACHE_INCLUDE} ${ENGINE_PRIMPROC_INCLUDE} )
-MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO)
+
-MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC")
+IF(NOT WITH_UBSAN AND NOT WITH_TSAN)
+    MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO)
+    MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC")
+ENDIF()
 
 if (WITH_UNITTESTS)
     cmake_policy(SET CMP0054 NEW)

writeengine/shared/we_define.h

@@ -151,6 +151,7 @@ const int ERR_FILE_FLUSH = ERR_FILEBASE + 18;           // Error flushing file
 const int ERR_FILE_GLOBBING = ERR_FILEBASE + 19;        // Error globbing a file name
 const int ERR_FILE_EOF = ERR_FILEBASE + 20;             // EOF
 const int ERR_FILE_CHOWN = ERR_FILEBASE + 21;           // EOF
+const int ERR_INTERNAL = ERR_FILEBASE + 22;             // EOF
 
 //--------------------------------------------------------------------------
 // XML level error

writeengine/shared/we_fileop.cpp

@@ -2105,6 +2105,11 @@ int FileOp::oid2FileName(FID fid, char* fullFileName, bool bCreateDir, uint16_t
   RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, partition, segment)));
 
   // see if file exists in specified DBRoot; return if found
+  if (fullFileName == nullptr)
+  {
+    return ERR_INTERNAL;
+  }
+
   if (dbRoot > 0)
   {
     sprintf(fullFileName, "%s/%s", Config::getDBRootByNum(dbRoot).c_str(), tempFileName);
@@ -2220,6 +2225,12 @@ int FileOp::oid2DirName(FID fid, char* oidDirName) const
     return NO_ERROR;
   }
 
+
+  if (oidDirName == nullptr)
+  {
+    return ERR_INTERNAL;
+  }
+
   RETURN_ON_ERROR((Convertor::oid2FileName(fid, tempFileName, dbDir, 0, 0)));
 
   // Now try to find the directory in each of the DBRoots.