diff --git a/CMakeLists.txt b/CMakeLists.txt index 48e0ed4db..e9da392e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,7 +205,7 @@ IF (NOT INSTALL_LAYOUT) ENDIF() OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default}) OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF) - IF(SECURITY_HARDENED) + IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN AND NOT WITH_GPROF) # security-enhancing flags MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC") MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now") @@ -237,10 +237,15 @@ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-register") endif() +IF (WITH_COLUMNSTORE_ASAN) + MY_CHECK_AND_SET_COMPILER_FLAG("-U_FORTIFY_SOURCE" DEBUG RELWITHDEBINFO) + MY_CHECK_AND_SET_COMPILER_FLAG("-fsanitize=address -fsanitize-address-use-after-scope -fPIC") +ENDIF() + MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-copy" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-deprecated-declarations" DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall") +MY_CHECK_AND_SET_COMPILER_FLAG("-Werror -Wall -Wextra") SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") SET (ENGINE_DT_LIB datatypes) SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot boost_thread xml2 pthread rt ${ENGINE_DT_LIB}) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index fcdb3d4b9..117d1ec36 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -18,17 +18,25 @@ message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo") DISTRO_OPTIONS=("Ubuntu" "CentOS" "Debian" "Rocky") + +cd $SCRIPT_LOCATION +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) BRANCHES=($(git branch --list --no-color| grep "[^* ]+" -Eo)) +cd - + optparse.define short=t long=build-type desc="Build Type: ${BUILD_TYPE_OPTIONS[*]}" variable=MCS_BUILD_TYPE optparse.define short=d long=distro desc="Choouse your OS: ${DISTRO_OPTIONS[*]}" variable=OS -optparse.define short=s long=skip-deps desc="Skip install dependences" variable=SKIP_DEPS default=false value=true +optparse.define short=D long=install-deps desc="Install dependences" variable=INSTALL_DEPS default=false value=true optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true optparse.define short=S long=skip-columnstore-submodules desc="Skip columnstore submodules initialization" variable=SKIP_SUBMODULES default=false value=true optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true optparse.define short=B long=run-microbench="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true -optparse.define short=b long=branch desc="Choouse git branch ('none' for menu)" variable=BRANCH +optparse.define short=b long=branch desc="Choose git branch. For menu use -b \"\"" variable=BRANCH default=$CURRENT_BRANCH optparse.define short=D long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true +optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true +optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true +optparse.define short=P long=report-path desc="Path for storing reports and profiles" variable=REPORT_PATH default="/core" source $( optparse.build ) @@ -45,18 +53,27 @@ INSTALL_PREFIX="/usr/" DATA_DIR="/var/lib/mysql/data" CMAKE_BIN_NAME=cmake CTEST_BIN_NAME=ctest +CONFIG_DIR="/etc/my.cnf.d" + +if [[ $OS = 'Ubuntu' || $OS = 'Debian' ]]; then + CONFIG_DIR="/etc/mysql/mariadb.conf.d" +fi + select_branch() { + cd $SCRIPT_LOCATION + CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + if [[ ! " ${BRANCHES[*]} " =~ " ${BRANCH} " ]]; then - if [[ $BRANCH = 'none' ]]; then + if [[ $BRANCH = "" ]]; then getChoice -q "Select your branch" -o BRANCHES BRANCH=$selectedChoice fi - cd $SCRIPT_LOCATION - message "Selecting $BRANCH branch for Columnstore" - git checkout $BRANCH - cd - + if [[ $BRANCH != $CURRENT_BRANCH ]]; then + message "Selecting $BRANCH branch for Columnstore" + git checkout $BRANCH + fi message "Turning off Columnstore submodule auto update via gitconfig" cd $MDB_SOURCE_PATH @@ -64,8 +81,6 @@ select_branch() cd - fi - cd $SCRIPT_LOCATION - CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) cd - message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_normal branch" } @@ -136,7 +151,9 @@ clean_old_installation() rm -rf /var/lib/columnstore/local/ rm -f /var/lib/columnstore/storagemanager/storagemanager-lock rm -f /var/lib/columnstore/storagemanager/cs-initialized + rm -rf /var/log/mariadb/columnstore/* rm -rf /tmp/* + rm -rf $REPORT_PATH rm -rf /var/lib/mysql rm -rf /var/run/mysqld rm -rf $DATA_DIR @@ -161,7 +178,8 @@ build() -DWITH_WSREP=OFF -DWITH_SSL=system -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL_PREFIX - -DCMAKE_EXPORT_COMPILE_COMMANDS=1" + -DCMAKE_EXPORT_COMPILE_COMMANDS=1 + " if [[ $SKIP_UNIT_TESTS = true ]] ; then @@ -172,11 +190,21 @@ build() message "Buiding with unittests" fi + if [[ $ASAN = true ]] ; then + warn "Building with ASAN" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_ASAN=ON -DWITH_COLUMNSTORE_ASAN=ON -DWITH_COLUMNSTORE_REPORT_PATH=${REPORT_PATH}" + fi + if [[ $WITHOUT_COREDUMPS = true ]] ; then warn "Cores are not dumped" - else MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DWITH_COREDUMPS=ON" + echo "${REPORT_PATH}/core_%e.%p" | sudo tee /proc/sys/kernel/core_pattern + fi + + if [[ $MAKEFILE_VERBOSE = true ]] ; then + warn "Verbosing Makefile Commands" + MDB_CMAKE_FLAGS="${MDB_CMAKE_FLAGS} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" fi if [[ $RUN_BENCHMARKS = true ]] ; then @@ -197,7 +225,6 @@ build() message "Buiding without microbenchmarks" fi - cd $MDB_SOURCE_PATH if [[ $SKIP_SUBMODULES = true ]] ; then @@ -228,11 +255,13 @@ build() message "building with flags $MDB_CMAKE_FLAGS" local CPUS=$(getconf _NPROCESSORS_ONLN) - ${CMAKE_BIN_NAME} . -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ - make -j $CPUS install + ${CMAKE_BIN_NAME} -DCMAKE_BUILD_TYPE=$MCS_BUILD_TYPE $MDB_CMAKE_FLAGS && \ | + make -j $CPUS + message "Installing silently" + make -j $CPUS install > /dev/null if [ $? -ne 0 ]; then - error "!!!! BUILD FAILED" + error "!!!! BUILD FAILED !!!!" exit 1 fi cd - @@ -259,7 +288,7 @@ run_unit_tests() else message "Running unittests" cd $MDB_SOURCE_PATH - ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) + ${CTEST_BIN_NAME} . -R columnstore: -j $(nproc) --progress cd - fi } @@ -271,14 +300,56 @@ run_microbenchmarks_tests() else message "Runnning microbenchmarks" cd $MDB_SOURCE_PATH - ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) + ${CTEST_BIN_NAME} . -V -R columnstore_microbenchmarks: -j $(nproc) --progress cd - fi } +disable_plugins_for_bootstrap() +{ + find /etc -type f -exec sed -i 's/plugin-load-add=auth_gssapi.so//g' {} + + find /etc -type f -exec sed -i 's/plugin-load-add=ha_columnstore.so//g' {} + +} + +enable_columnstore_back() +{ + echo plugin-load-add=ha_columnstore.so >> $CONFIG_DIR/columnstore.cnf +} + +fix_config_files() +{ + message Fixing config files + + THREAD_STACK_SIZE="20M" + + SYSTEMD_SERVICE_DIR="/usr/lib/systemd/system" + if [[ $ASAN = true ]] ; then + COLUMNSTORE_CONFIG=$CONFIG_DIR/columnstore.cnf + if grep -q thread_stack $COLUMNSTORE_CONFIG; then + warn "MDB Server has thread_stack settings on $COLUMNSTORE_CONFIG check it's compatibility with ASAN" + else + echo "thread_stack = ${THREAD_STACK_SIZE}" >> $COLUMNSTORE_CONFIG + message "thread_stack was set to ${THREAD_STACK_SIZE} in $COLUMNSTORE_CONFIG" + fi + + MDB_SERVICE_FILE=$SYSTEMD_SERVICE_DIR/mariadb.service + if grep -q ASAN $MDB_SERVICE_FILE; then + warn "MDB Server has ASAN options in $MDB_SERVICE_FILE, check it's compatibility" + else + echo Environment="'ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}'" >> $MDB_SERVICE_FILE + message "ASAN options were added to $MDB_SERVICE_FILE" + fi + fi + systemctl daemon-reload +} + install() { message "Installing MariaDB" + disable_plugins_for_bootstrap + + mkdir -p $REPORT_PATH + chmod 777 $REPORT_PATH check_user_and_group @@ -288,10 +359,15 @@ install() socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' mv $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_1.so || mv $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so /tmp/ha_columnstore_2.so + mkdir -p /var/lib/mysql + chown mysql:mysql /var/lib/mysql + message "Running mysql_install_db" - mysql_install_db --rpm --user=mysql + sudo -u mysql mysql_install_db --rpm --user=mysql > /dev/null mv /tmp/ha_columnstore_1.so $INSTALL_PREFIX/lib/mysql/plugin/ha_columnstore.so || mv /tmp/ha_columnstore_2.so $INSTALL_PREFIX/lib64/mysql/plugin/ha_columnstore.so + enable_columnstore_back + mkdir -p /etc/columnstore cp $MDB_SOURCE_PATH/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml @@ -308,7 +384,7 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' > /etc/mysql/debian.cnf fi - systemctl daemon-reload + fix_config_files if [ -d "/etc/mysql/mariadb.conf.d/" ]; then message "Copying configs from /etc/mysql/mariadb.conf.d/ to /etc/my.cnf.d" @@ -345,9 +421,23 @@ socket=/run/mysqld/mysqld.sock" > /etc/my.cnf.d/socket.cnf' chmod 777 /var/log/mariadb/columnstore } + +smoke() +{ + message "Creating test database" + mariadb -e "create database if not exists test;" + message "Selecting magic numbers" + MAGIC=`mysql -N test < $MDB_SOURCE_PATH/storage/columnstore/columnstore/tests/scripts/smoke.sql` + if [[ $MAGIC == '42' ]] ; then + message "Great answer correct" + else + warn "Smoke failed, answer is '$MAGIC'" + fi +} + select_branch -if [[ $SKIP_DEPS = false ]] ; then +if [[ $INSTALL_DEPS = true ]] ; then install_deps fi @@ -358,4 +448,6 @@ run_unit_tests run_microbenchmarks_tests install start_service +smoke + message "$color_green FINISHED $color_normal" diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 6acd70605..f46a696f4 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -164,6 +164,27 @@ const int128_t mcs_pow_10_128[20] = { 100000000000000000000000000000000000000_xxl, }; +const long long columnstore_precision[19] = {0, + 9, + 99, + 999, + 9999, + 99999, + 999999, + 9999999, + 99999999, + 999999999, + 9999999999LL, + 99999999999LL, + 999999999999LL, + 9999999999999LL, + 99999999999999LL, + 999999999999999LL, + 9999999999999999LL, + 99999999999999999LL, + 999999999999999999LL}; + + const int128_t ConversionRangeMaxValue[20] = {9999999999999999999_xxl, 99999999999999999999_xxl, 999999999999999999999_xxl, diff --git a/dbcon/ddlpackageproc/libddlpackageproc.rc b/dbcon/ddlpackageproc/libddlpackageproc.rc deleted file mode 100644 index abf70abc2..000000000 --- a/dbcon/ddlpackageproc/libddlpackageproc.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB DDL API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "libddlpackageproc" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "libddlpackageproc.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/dbcon/dmlpackageproc/libdmlpackageproc.rc b/dbcon/dmlpackageproc/libdmlpackageproc.rc deleted file mode 100644 index c8bbca741..000000000 --- a/dbcon/dmlpackageproc/libdmlpackageproc.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB DML API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "libdmlpackageproc" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "libdmlpackageproc.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/dbcon/joblist/libjoblist.rc b/dbcon/joblist/libjoblist.rc deleted file mode 100644 index 7b0e36a47..000000000 --- a/dbcon/joblist/libjoblist.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB Job List API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "libjoblist.dll" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "libjoblist.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index 1aeac576e..168e9a711 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -29,6 +29,7 @@ #include "blocksize.h" #include "calpontsystemcatalog.h" #include "joblisttypes.h" +#include #include @@ -701,7 +702,7 @@ const uint8_t ROUND_POS = 0x01; // actual value larger/longer than the stored v const uint8_t ROUND_NEG = 0x80; // actual value less than the stored value // Tied to ColByScanRequestHeader and ColByScanRangeRequestHeader. Check other headers if modifying. -struct NewColRequestHeader +struct /*alignas(utils::MAXCOLUMNWIDTH)*/ NewColRequestHeader { ISMPacketHeader ism; PrimitiveHeader hdr; diff --git a/dbcon/joblist/tupleannexstep.cpp b/dbcon/joblist/tupleannexstep.cpp index b95d90f16..c4aeff34e 100644 --- a/dbcon/joblist/tupleannexstep.cpp +++ b/dbcon/joblist/tupleannexstep.cpp @@ -298,7 +298,6 @@ void TupleAnnexStep::join() uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs) { - RGData rgDataOut; bool more = false; uint32_t rowCount = 0; @@ -306,18 +305,18 @@ uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs) { bs.restart(); - more = fOutputDL->next(fOutputIterator, &rgDataOut); + more = fOutputDL->next(fOutputIterator, &fRgDataOut); if (more && !cancelled()) { - fRowGroupDeliver.setData(&rgDataOut); + fRowGroupDeliver.setData(&fRgDataOut); fRowGroupDeliver.serializeRGData(bs); rowCount = fRowGroupDeliver.getRowCount(); } else { while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); + more = fOutputDL->next(fOutputIterator, &fRgDataOut); fEndOfResult = true; } @@ -327,7 +326,7 @@ uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs) handleException(std::current_exception(), logging::ERR_IN_DELIVERY, logging::ERR_ALWAYS_CRITICAL, "TupleAnnexStep::nextBand()"); while (more) - more = fOutputDL->next(fOutputIterator, &rgDataOut); + more = fOutputDL->next(fOutputIterator, &fRgDataOut); fEndOfResult = true; } @@ -335,8 +334,8 @@ uint32_t TupleAnnexStep::nextBand(messageqcpp::ByteStream& bs) if (fEndOfResult) { // send an empty / error band - rgDataOut.reinit(fRowGroupDeliver, 0); - fRowGroupDeliver.setData(&rgDataOut); + fRgDataOut.reinit(fRowGroupDeliver, 0); + fRowGroupDeliver.setData(&fRgDataOut); fRowGroupDeliver.resetRowGroup(0); fRowGroupDeliver.setStatus(status()); fRowGroupDeliver.serializeRGData(bs); diff --git a/dbcon/joblist/tupleannexstep.h b/dbcon/joblist/tupleannexstep.h index dcdc064c3..4443d3641 100644 --- a/dbcon/joblist/tupleannexstep.h +++ b/dbcon/joblist/tupleannexstep.h @@ -118,6 +118,7 @@ class TupleAnnexStep : public JobStep, public TupleDeliveryStep rowgroup::RowGroup fRowGroupIn; rowgroup::RowGroup fRowGroupOut; rowgroup::RowGroup fRowGroupDeliver; + rowgroup::RGData fRgDataOut; rowgroup::Row fRowIn; rowgroup::Row fRowOut; diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index ab7b8ca89..0e6c57991 100644 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1054,7 +1054,7 @@ void WindowFunctionStep::doPostProcessForSelect() for (int64_t i = begin; i < end; i++) { - if (rgData.rowData.get() == NULL) + if (!rgData.hasRowData()) { rgCapacity = ((rowsLeft > 8192) ? 8192 : rowsLeft); rowsLeft -= rgCapacity; diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index 2ce38f2d3..a9c319f6d 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -1941,7 +1941,7 @@ int ha_mcs_cache::flush_insert_cache() int error, error2; ha_maria* from = cache_handler; uchar* record = table->record[0]; - ulonglong copied_rows = 0; + [[maybe_unused]] ulonglong copied_rows = 0; DBUG_ENTER("flush_insert_cache"); DBUG_ASSERT(from->file->state->records == share->cached_rows); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index d50c47549..6d0b9a8bb 100644 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -2963,7 +2963,7 @@ SimpleColumn* getSmallestColumn(boost::shared_ptr csc, CalpontSelectExecutionPlan::ReturnedColumnList::const_iterator iter; - ReturnedColumn* rc; + ReturnedColumn* rc = nullptr; for (iter = cols.begin(); iter != cols.end(); iter++) { diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 2c1efadc4..4e5a153e0 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -569,7 +569,7 @@ extern "C" void calshowpartitions_deinit(UDF_INIT* initid) { - delete initid->ptr; + delete[] initid->ptr; } const char* calshowpartitions(UDF_INIT* initid, UDF_ARGS* args, char* result, unsigned long* length, @@ -1164,7 +1164,7 @@ extern "C" void calshowpartitionsbyvalue_deinit(UDF_INIT* initid) { - delete initid->ptr; + delete[] initid->ptr; } const char* calshowpartitionsbyvalue(UDF_INIT* initid, UDF_ARGS* args, char* result, diff --git a/ddlproc/DDLProc.rc b/ddlproc/DDLProc.rc deleted file mode 100644 index cb8b93952..000000000 --- a/ddlproc/DDLProc.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB DDL Processor" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "DDLProc" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "DDLProc.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/dmlproc/DMLProc.rc b/dmlproc/DMLProc.rc deleted file mode 100644 index d240733d8..000000000 --- a/dmlproc/DMLProc.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB DML Processor" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "DMLProc" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "DMLProc.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index 1614cb36d..e61184af4 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -8,6 +8,34 @@ if (WITH_COREDUMPS) endif (WITH_COREDUMPS) +SET(LD_PRELOAD_STRING "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}')") +SET(ALLOC_CONFIG "MALLOC_CONF=''") +SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) +SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) +SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) +SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG}) +SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) +SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) +SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG}) + + +if (WITH_COLUMNSTORE_ASAN) + SET(ASAN_PATH "/tmp/asan") + if (WITH_COLUMNSTORE_REPORT_PATH) + SET(ASAN_PATH "${WITH_COLUMNSTORE_REPORT_PATH}/asan") + endif (WITH_COLUMNSTORE_REPORT_PATH) + + + SET(LD_PRELOAD_STRING "") + SET(ALLOC_CONFIG "ASAN_OPTIONS=abort_on_error=1:disable_coredump=0,print_stats=false,detect_odr_violation=0,check_initialization_order=1,detect_stack_use_after_return=1,atexit=false,log_path=${ASAN_PATH}") + SET(PRIMPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) + SET(DMLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) + SET(DDLPROC_ALLOC_CONFIG ${ALLOC_CONFIG}) + SET(WRITEENGINE_ALLOC_CONFIG ${ALLOC_CONFIG}) + SET(CONTROLLERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) + SET(WORKERNODE_ALLOC_CONFIG ${ALLOC_CONFIG}) + SET(STORAGEMANAGER_ALLOC_CONFIG ${ALLOC_CONFIG}) +endif() configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" @ONLY) diff --git a/oam/install_scripts/mcs-controllernode.service.in b/oam/install_scripts/mcs-controllernode.service.in index 6827ada95..62b2ae4cb 100644 --- a/oam/install_scripts/mcs-controllernode.service.in +++ b/oam/install_scripts/mcs-controllernode.service.in @@ -13,7 +13,7 @@ Group=@DEFAULT_GROUP@ LimitNOFILE=65536 LimitNPROC=65536 LimitCORE=@CORE_DUMPS@ - +Environment="@CONTROLLERNODE_ALLOC_CONFIG@" ExecStart=@ENGINE_BINDIR@/controllernode ExecStop=@ENGINE_BINDIR@/mcs-stop-controllernode.sh $MAINPID diff --git a/oam/install_scripts/mcs-ddlproc.service.in b/oam/install_scripts/mcs-ddlproc.service.in index 1ba9275c9..5cc3f1003 100644 --- a/oam/install_scripts/mcs-ddlproc.service.in +++ b/oam/install_scripts/mcs-ddlproc.service.in @@ -14,6 +14,7 @@ LimitNOFILE=65536 LimitNPROC=65536 LimitCORE=@CORE_DUMPS@ +Environment="@DDLPROC_ALLOC_CONFIG@" ExecStart=@ENGINE_BINDIR@/DDLProc Restart=on-failure diff --git a/oam/install_scripts/mcs-dmlproc.service.in b/oam/install_scripts/mcs-dmlproc.service.in index 95ebe2f0d..0eb33b765 100644 --- a/oam/install_scripts/mcs-dmlproc.service.in +++ b/oam/install_scripts/mcs-dmlproc.service.in @@ -14,6 +14,7 @@ LimitNOFILE=65536 LimitNPROC=65536 LimitCORE=@CORE_DUMPS@ +Environment="@DMLPROC_ALLOC_CONFIG@" ExecStart=@ENGINE_BINDIR@/DMLProc Restart=on-failure diff --git a/oam/install_scripts/mcs-primproc.service.in b/oam/install_scripts/mcs-primproc.service.in index 3a2ada382..9ceba1512 100644 --- a/oam/install_scripts/mcs-primproc.service.in +++ b/oam/install_scripts/mcs-primproc.service.in @@ -16,7 +16,7 @@ LimitNPROC=65536 LimitCORE=@CORE_DUMPS@ ExecStartPre=/usr/bin/env bash -c "ldconfig -p | grep -m1 libjemalloc > /dev/null || echo 'Please install jemalloc to avoid ColumnStore performance degradation and unexpected service interruptions.'" -ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/PrimProc" +ExecStart=/usr/bin/env bash -c "@PRIMPROC_ALLOC_CONFIG@ @LD_PRELOAD_STRING@ exec @ENGINE_BINDIR@/PrimProc" Restart=on-failure TimeoutStopSec=2 diff --git a/oam/install_scripts/mcs-storagemanager.service.in b/oam/install_scripts/mcs-storagemanager.service.in index 87eb28e17..7752fa285 100644 --- a/oam/install_scripts/mcs-storagemanager.service.in +++ b/oam/install_scripts/mcs-storagemanager.service.in @@ -10,5 +10,5 @@ LimitNOFILE=65536 LimitNPROC=65536 LimitCORE=@CORE_DUMPS@ -ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/StorageManager" +ExecStart=/usr/bin/env bash -c "@STORAGEMANAGER_ALLOC_CONFIG@ @LD_PRELOAD_STRING@ exec @ENGINE_BINDIR@/StorageManager" SuccessExitStatus=255 diff --git a/oam/install_scripts/mcs-workernode.service.in b/oam/install_scripts/mcs-workernode.service.in index 9827ba872..b1cf7310f 100644 --- a/oam/install_scripts/mcs-workernode.service.in +++ b/oam/install_scripts/mcs-workernode.service.in @@ -11,6 +11,7 @@ LimitNOFILE=65536 LimitNPROC=65536 LimitCORE=@CORE_DUMPS@ +Environment="@WORKERNODE_ALLOC_CONFIG@" ExecStart=@ENGINE_BINDIR@/workernode DBRM_Worker%i ExecStopPost=@ENGINE_BINDIR@/mcs-savebrm.py ExecStopPost=/usr/bin/env bash -c "clearShm > /dev/null 2>&1" diff --git a/oam/install_scripts/mcs-writeengineserver.service.in b/oam/install_scripts/mcs-writeengineserver.service.in index 7b92e4da2..de225af5e 100644 --- a/oam/install_scripts/mcs-writeengineserver.service.in +++ b/oam/install_scripts/mcs-writeengineserver.service.in @@ -14,7 +14,7 @@ LimitNOFILE=65536 LimitNPROC=65536 LimitCORE=@CORE_DUMPS@ -ExecStart=/usr/bin/env bash -c "LD_PRELOAD=$(ldconfig -p | grep -m1 libjemalloc | awk '{print $1}') exec @ENGINE_BINDIR@/WriteEngineServer" +ExecStart=/usr/bin/env bash -c "@WRITEENGINE_ALLOC_CONFIG@ @LD_PRELOAD_STRING@ exec @ENGINE_BINDIR@/WriteEngineServer" Restart=on-failure TimeoutStopSec=20 diff --git a/oam/oamcpp/liboamcpp.rc b/oam/oamcpp/liboamcpp.rc deleted file mode 100644 index 00c76ea9c..000000000 --- a/oam/oamcpp/liboamcpp.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB OAM API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "liboamcpp" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "liboamcpp.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/primitives/primproc/PrimProc.rc b/primitives/primproc/PrimProc.rc deleted file mode 100644 index 54535710f..000000000 --- a/primitives/primproc/PrimProc.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB Primitive Processor" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "PrimProc" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "PrimProc.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 659792c57..6d148f2ef 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -931,7 +931,7 @@ void BatchPrimitiveProcessor::initProcessor() strValues.reset(new string[LOGICAL_BLOCK_RIDS]); outMsgSize = defaultBufferSize; - outputMsg.reset(reinterpret_cast(aligned_alloc(utils::MAXCOLUMNWIDTH, outMsgSize))); + outputMsg.reset(new(std::align_val_t(MAXCOLUMNWIDTH)) uint8_t[outMsgSize]); if (ot == ROW_GROUP) { diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index d7c6f1230..88790d1b9 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -232,7 +232,7 @@ class BatchPrimitiveProcessor /* Common space for primitive data */ alignas(utils::MAXCOLUMNWIDTH) uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH]; uint8_t blockDataAux[BLOCK_SIZE * execplan::AUX_COL_WIDTH]; - boost::scoped_array outputMsg; + std::unique_ptr outputMsg; uint32_t outMsgSize; std::vector filterSteps; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index fa2a880a7..6f7480074 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -631,7 +631,8 @@ void ColumnCommand::fillInPrimitiveMessageHeader(const int8_t outputType, const size_t inputMsgBufSize = baseMsgLength + (LOGICAL_BLOCK_RIDS * sizeof(primitives::RIDType)); if (!inputMsg) - inputMsg.reset(reinterpret_cast(aligned_alloc(utils::MAXCOLUMNWIDTH, inputMsgBufSize))); + inputMsg.reset(new(std::align_val_t(utils::MAXCOLUMNWIDTH)) uint8_t[inputMsgBufSize]); + primMsg = (NewColRequestHeader*)inputMsg.get(); outMsg = (ColResultHeader*)bpp->outputMsg.get(); diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index 19a4da597..cc08f6edb 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -30,6 +30,8 @@ #pragma once +#include +#include "columnwidth.h" #include "command.h" #include "calpontsystemcatalog.h" @@ -164,7 +166,7 @@ class ColumnCommand : public Command bool _isScan; - boost::scoped_array inputMsg; + std::unique_ptr inputMsg; NewColRequestHeader* primMsg; ColResultHeader* outMsg; diff --git a/primitives/primproc/serviceexemgr.h b/primitives/primproc/serviceexemgr.h index 905f77b0a..d605d557d 100644 --- a/primitives/primproc/serviceexemgr.h +++ b/primitives/primproc/serviceexemgr.h @@ -370,9 +370,9 @@ namespace exemgr // shared by multiple threads per session. ThreadCntPerSessionMap_t threadCntPerSessionMap_; std::mutex threadCntPerSessionMapMutex_; - ActiveStatementCounter* statementsRunningCount_; - joblist::DistributedEngineComm* dec_; - joblist::ResourceManager* rm_; + ActiveStatementCounter* statementsRunningCount_ = nullptr; + joblist::DistributedEngineComm* dec_ = nullptr; + joblist::ResourceManager* rm_ = nullptr; // Its attributes are set in Child() querytele::QueryTeleServerParms teleServerParms_; std::vector localNetIfaceSins_; diff --git a/primitives/primproc/sqlfrontsessionthread.cpp b/primitives/primproc/sqlfrontsessionthread.cpp index 5b43746f0..69cead754 100644 --- a/primitives/primproc/sqlfrontsessionthread.cpp +++ b/primitives/primproc/sqlfrontsessionthread.cpp @@ -248,7 +248,7 @@ void SQLFrontSessionThread::analyzeTableExecute(messageqcpp::ByteStream& bs, job auto rowCount = jl->projectTable(dummyTableOid, bs); while (rowCount) { - auto outRG = (static_cast(jl.get()))->getOutputRowGroup(); + auto const& outRG = (static_cast(jl.get()))->getOutputRowGroup(); statisticsManager->collectSample(outRG); rowCount = jl->projectTable(dummyTableOid, bs); } @@ -894,7 +894,7 @@ void SQLFrontSessionThread::operator()() std::unique_lock scoped(jlMutex); destructing++; std::thread bgdtor( - [jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing, &msgLog] + [jl, &jlMutex, &jlCleanupDone, stmtID, li, &destructing, &msgLog] { std::unique_lock scoped(jlMutex); const_cast(jl).reset(); // this happens second; does real destruction diff --git a/storage-manager/src/Config.h b/storage-manager/src/Config.h index 30c485751..b7e9a75b1 100644 --- a/storage-manager/src/Config.h +++ b/storage-manager/src/Config.h @@ -17,7 +17,17 @@ #pragma once +#ifndef __clang__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + #include + +#ifndef __clang__ + #pragma GCC diagnostic pop +#endif + #include #include #include diff --git a/storage-manager/src/MetadataFile.cpp b/storage-manager/src/MetadataFile.cpp index 820e7fe06..b4fb327f8 100644 --- a/storage-manager/src/MetadataFile.cpp +++ b/storage-manager/src/MetadataFile.cpp @@ -22,7 +22,16 @@ #include #include #define BOOST_SPIRIT_THREADSAFE +#ifndef __clang__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + #include + +#ifndef __clang__ + #pragma GCC diagnostic pop +#endif #include #include #include diff --git a/storage-manager/src/S3Storage.cpp b/storage-manager/src/S3Storage.cpp index a07c79863..34ad1f9d6 100644 --- a/storage-manager/src/S3Storage.cpp +++ b/storage-manager/src/S3Storage.cpp @@ -27,7 +27,16 @@ #include #include #define BOOST_SPIRIT_THREADSAFE +#ifndef __clang__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + #include + +#ifndef __clang__ + #pragma GCC diagnostic pop +#endif #include #include "Utilities.h" diff --git a/tests/scripts/fullmtr.sh b/tests/scripts/fullmtr.sh index bf992b82d..d10bd8187 100644 --- a/tests/scripts/fullmtr.sh +++ b/tests/scripts/fullmtr.sh @@ -3,14 +3,14 @@ mysql -e "create database if not exists test;" SOCKET=`mysql -e "show variables like 'socket';" | grep socket | cut -f2` cd /usr/share/mysql/mysql-test -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/setup ./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/basic | tee $CURRENT_DIR/mtr.basic.log 2>&1 -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/bugfixes | tee $CURRENT_DIR/mtr.bugfixes.log 2>&1 -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/devregression | tee $CURRENT_DIR/mtr.devregression.log 2>&1 -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/autopilot | tee $CURRENT_DIR/mtr.autopilot.log 2>&1 -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/extended | tee $CURRENT_DIR/mtr.extended.log 2>&1 -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/multinode | tee $CURRENT_DIR/mtr.multinode.log 2>&1 -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/oracle | tee $CURRENT_DIR/mtr.multinode.log 2>&1 -./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/1pmonly | tee $CURRENT_DIR/mtr.1pmonly.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/setup | tee $CURRENT_DIR/mtr.setup.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/bugfixes | tee $CURRENT_DIR/mtr.bugfixes.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/devregression | tee $CURRENT_DIR/mtr.devregression.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/autopilot | tee $CURRENT_DIR/mtr.autopilot.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/extended | tee $CURRENT_DIR/mtr.extended.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/multinode | tee $CURRENT_DIR/mtr.multinode.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/oracle | tee $CURRENT_DIR/mtr.oracle.log 2>&1 +./mtr --force --max-test-fail=0 --testcase-timeout=60 --extern socket=$SOCKET --suite=columnstore/1pmonly | tee $CURRENT_DIR/mtr.1pmonly.log 2>&1 cd - diff --git a/tests/scripts/smoke.sql b/tests/scripts/smoke.sql new file mode 100644 index 000000000..3236cc0f9 --- /dev/null +++ b/tests/scripts/smoke.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS smoke_table; +CREATE TABLE smoke_table(a int) ENGINE COLUMNSTORE; +INSERT INTO smoke_table VALUES(42); +SELECT * FROM smoke_table; +DROP TABLE smoke_table; diff --git a/tools/dbloadxml/colxml.rc b/tools/dbloadxml/colxml.rc deleted file mode 100644 index 017a2d66a..000000000 --- a/tools/dbloadxml/colxml.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB Bulk Load Utility" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "colxml" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "colxml.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 60d161068..0c060ae89 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -21,8 +21,16 @@ namespace utils { -const uint8_t MAXLEGACYWIDTH = 8ULL; -const uint8_t MAXCOLUMNWIDTH = 16ULL; +constexpr uint8_t MAXLEGACYWIDTH = 8ULL; +constexpr uint8_t MAXCOLUMNWIDTH = 16ULL; + +struct AlignedDeleter +{ + void operator()(uint8_t* ptr) + { + operator delete[](ptr, std::align_val_t(utils::MAXCOLUMNWIDTH)); + }; +}; inline bool isWide(uint8_t width) { diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index df77f8600..33b08364e 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -69,15 +69,17 @@ const fs::path defaultConfigFilePath(configDefaultFileName); namespace config { -Config* globConfigInstancePtr = nullptr; -Config::configMap_t Config::fInstanceMap; boost::mutex Config::fInstanceMapMutex; +Config::configMap_t Config::fInstanceMap; // duplicate to that in the Config class boost::mutex Config::fXmlLock; // duplicate to that in the Config class boost::mutex Config::fWriteXmlLock; std::atomic_bool globHasConfig; +ConfigUniqPtr globConfigInstancePtr; + + void Config::checkAndReloadConfig() { struct stat statbuf; @@ -105,20 +107,20 @@ Config* Config::makeConfig(const string& cf) if (globConfigInstancePtr) { globConfigInstancePtr->checkAndReloadConfig(); - return globConfigInstancePtr; + return globConfigInstancePtr.get(); } // Make this configurable at least at compile-time. std::string configFilePath = std::string(MCSSYSCONFDIR) + std::string("/columnstore/") + configDefaultFileName; - globConfigInstancePtr = new Config(configFilePath); + globConfigInstancePtr.reset(new Config(configFilePath)); globHasConfig.store(true, std::memory_order_relaxed); - return globConfigInstancePtr; + return globConfigInstancePtr.get(); } boost::mutex::scoped_lock lk(fInstanceMapMutex); globConfigInstancePtr->checkAndReloadConfig(); - return globConfigInstancePtr; + return globConfigInstancePtr.get(); } boost::mutex::scoped_lock lk(fInstanceMapMutex); @@ -526,21 +528,6 @@ void Config::writeConfigFile(messageqcpp::ByteStream msg) const /* static */ void Config::deleteInstanceMap() { - boost::mutex::scoped_lock lk(fInstanceMapMutex); - - for (Config::configMap_t::iterator iter = fInstanceMap.begin(); iter != fInstanceMap.end(); ++iter) - { - Config* instance = iter->second; - delete instance; - } - - fInstanceMap.clear(); - - if (globConfigInstancePtr) - { - delete globConfigInstancePtr; - globConfigInstancePtr = nullptr; - } } /* static */ @@ -643,4 +630,18 @@ std::string Config::getTempFileDir(Config::TempDirPurpose what) return {}; } +void Config::ConfigDeleter::operator()(Config* config) +{ + boost::mutex::scoped_lock lk(fInstanceMapMutex); + + for (Config::configMap_t::iterator iter = fInstanceMap.begin(); iter != fInstanceMap.end(); ++iter) + { + Config* instance = iter->second; + delete instance; + } + + fInstanceMap.clear(); + delete config; +} + } // namespace config diff --git a/utils/configcpp/configcpp.h b/utils/configcpp/configcpp.h index 10ce3cfff..e170e26f3 100644 --- a/utils/configcpp/configcpp.h +++ b/utils/configcpp/configcpp.h @@ -56,6 +56,11 @@ namespace config class Config { public: + struct ConfigDeleter + { + void operator()(Config* config); + }; + /** @brief Config factory method * * Creates a singleton Config object @@ -249,8 +254,14 @@ class Config * */ void checkAndReloadConfig(); + }; + + +using ConfigUniqPtr = std::unique_ptr; + + } // namespace config #undef EXPORT diff --git a/utils/configcpp/libconfigcpp.rc b/utils/configcpp/libconfigcpp.rc deleted file mode 100644 index b3e02fb95..000000000 --- a/utils/configcpp/libconfigcpp.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x2L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB Config API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "libconfigcpp" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "libconfigcpp.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/utils/configcpp/xmlparser.cpp b/utils/configcpp/xmlparser.cpp index 37b6bddea..e36b7c094 100644 --- a/utils/configcpp/xmlparser.cpp +++ b/utils/configcpp/xmlparser.cpp @@ -150,6 +150,7 @@ void XMLParser::setConfig(xmlDocPtr doc, const string& section, const string& na { xmlAddChild(cur2, xmlNewText((const xmlChar*)"\t")); cur3 = cur2->xmlChildrenNode; + xmlFree(cur3->content); } else { diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index fefca375f..de4595308 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -48,26 +48,6 @@ using namespace logging; namespace { -const int64_t columnstore_precision[19] = {0, - 9, - 99, - 999, - 9999, - 99999, - 999999, - 9999999, - 99999999, - 999999999, - 9999999999LL, - 99999999999LL, - 999999999999LL, - 9999999999999LL, - 99999999999999LL, - 999999999999999LL, - 9999999999999999LL, - 99999999999999999LL, - 999999999999999999LL}; - template bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&)) { @@ -475,25 +455,16 @@ void number_int_value(const string& data, cscDataType typeCode, if ((typeCode == datatypes::SystemCatalog::DECIMAL) || (typeCode == datatypes::SystemCatalog::UDECIMAL) || (ct.scale > 0)) { - T rangeUp, rangeLow; - - if (ct.precision < 19) + auto precision = + ct.precision == rowgroup::MagicPrecisionForCountAgg ? datatypes::INT128MAXPRECISION : ct.precision; + if (precision > datatypes::INT128MAXPRECISION || precision < 0) { - rangeUp = (T)columnstore_precision[ct.precision]; - } - else - { - auto precision = - ct.precision == rowgroup::MagicPrecisionForCountAgg ? datatypes::INT128MAXPRECISION : ct.precision; - if (precision > datatypes::INT128MAXPRECISION || precision < 0) - { - throw QueryDataExcept("Unsupported precision " + std::to_string(precision) + " converting DECIMAL ", - dataTypeErr); - } - rangeUp = datatypes::ConversionRangeMaxValue[ct.precision - 19]; + throw QueryDataExcept("Unsupported precision " + std::to_string(precision) + " converting DECIMAL ", + dataTypeErr); } - rangeLow = -rangeUp; + T rangeUp = dataconvert::decimalRangeUp(precision); + T rangeLow = -rangeUp; if (intVal > rangeUp) { @@ -2802,7 +2773,8 @@ int64_t DataConvert::stringToTime(const string& data) { if (!hasDate) { - day = strtol(data.substr(0, pos).c_str(), &end, 10); + std::string tmpDataSegment = data.substr(0, pos); + day = strtol(tmpDataSegment.c_str(), &end, 10); if (*end != '\0') return -1; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index bddcb6d2a..ea5464657 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -102,6 +102,7 @@ const int64_t IDB_pow[19] = {1, 100000000000000000LL, 1000000000000000000LL}; + const int32_t SECS_PER_MIN = 60; const int32_t MINS_PER_HOUR = 60; const int32_t HOURS_PER_DAY = 24; @@ -1545,6 +1546,20 @@ inline int128_t strtoll128(const char* data, bool& saturate, char** ep) return res; } + +template +T decimalRangeUp(int32_t precision) +{ + if (precision < 19) + { + return (T)datatypes::columnstore_precision[precision]; + } + else + { + return datatypes::ConversionRangeMaxValue[precision - 19]; + } +} + template <> inline int128_t string_to_ll(const std::string& data, bool& bSaturate) { diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 98a2aec76..ab7fe7477 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -1072,7 +1072,6 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, FunctionParm& parm, bool& if (decimal.isTSInt128ByPrecision()) { int128_t max_number_decimal = datatypes::ConversionRangeMaxValue[max_length - 19]; - uint128_t uval = parm[0]->data()->getUintVal(row, isNull); if (uval > (uint128_t)datatypes::Decimal::maxInt128) diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 6cc47fdae..5f2c42d7c 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -591,6 +591,12 @@ uint64_t dateAdd(uint64_t time, const string& expr, IntervalColumn::interval_typ if (-day < month_length[monthSave]) { + if (monthSave == 0) + { + monthSave = 12; + tmpYear--; + } + month--; monthSave--; @@ -613,6 +619,12 @@ uint64_t dateAdd(uint64_t time, const string& expr, IntervalColumn::interval_typ // BUG 5448 - changed from '==' to '<=' if (day <= 0) { + if (monthSave == 0) + { + monthSave = 12; + tmpYear--; + } + month--; monthSave--; @@ -635,6 +647,17 @@ uint64_t dateAdd(uint64_t time, const string& expr, IntervalColumn::interval_typ break; } + if (monthSave == 0) + { + monthSave = 12; + tmpYear--; + + if (isLeapYear(tmpYear)) + month_length[2] = 29; + else + month_length[2] = 28; + } + month--; monthSave--; diff --git a/utils/funcexp/func_json_unquote.cpp b/utils/funcexp/func_json_unquote.cpp index 863a87a36..e4704ba44 100644 --- a/utils/funcexp/func_json_unquote.cpp +++ b/utils/funcexp/func_json_unquote.cpp @@ -37,7 +37,7 @@ std::string Func_json_unquote::getStrVal(rowgroup::Row& row, FunctionParm& fp, b if (unlikely(jsEg.s.error) || jsEg.value_type != JSON_VALUE_STRING) return js.data(); - char* buf = (char*)alloca(jsEg.value_len); + char* buf = (char*)alloca(jsEg.value_len + 1); if ((strLen = json_unescape(cs, jsEg.value, jsEg.value + jsEg.value_len, &my_charset_utf8mb3_general_ci, (uchar*)buf, (uchar*)(buf + jsEg.value_len))) >= 0) { diff --git a/utils/idbdatafile/IDBFactory.cpp b/utils/idbdatafile/IDBFactory.cpp index b3d2e0feb..ce32158e4 100644 --- a/utils/idbdatafile/IDBFactory.cpp +++ b/utils/idbdatafile/IDBFactory.cpp @@ -47,15 +47,10 @@ bool IDBFactory::installDefaultPlugins() // protect these methods since we are changing our static data structure boost::mutex::scoped_lock lock(fac_guard); - s_plugins[IDBDataFile::BUFFERED] = - FileFactoryEnt(IDBDataFile::BUFFERED, "buffered", new BufferedFileFactory(), new PosixFileSystem()); - s_plugins[IDBDataFile::UNBUFFERED] = FileFactoryEnt(IDBDataFile::UNBUFFERED, "unbuffered", - new UnbufferedFileFactory(), new PosixFileSystem()); - - // TODO: use the installPlugin fcn below instead of declaring this statically, then remove the dependency - // IDBDatafile -> cloudio - // s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new - // SMFileSystem()); + s_plugins.emplace(IDBDataFile::BUFFERED, FileFactoryEnt(IDBDataFile::BUFFERED, "buffered", new BufferedFileFactory(), + new PosixFileSystem())); + s_plugins.emplace(IDBDataFile::UNBUFFERED, FileFactoryEnt(IDBDataFile::UNBUFFERED, "unbuffered", + new UnbufferedFileFactory(), new PosixFileSystem())); return false; } @@ -86,7 +81,7 @@ bool IDBFactory::installPlugin(const std::string& plugin) } FileFactoryEnt ent = (*(FileFactoryEntryFunc)functor)(); - s_plugins[ent.type] = ent; + s_plugins.emplace(ent.type, std::move(ent)); std::ostringstream oss; oss << "IDBFactory::installPlugin: installed filesystem plugin " << plugin; @@ -112,7 +107,7 @@ IDBDataFile* IDBFactory::open(IDBDataFile::Types type, const char* fname, const throw std::runtime_error(oss.str()); } - return s_plugins[type].factory->open(fname, mode, opts, colWidth); + return s_plugins.at(type).factory->open(fname, mode, opts, colWidth); } IDBFileSystem& IDBFactory::getFs(IDBDataFile::Types type) @@ -124,7 +119,14 @@ IDBFileSystem& IDBFactory::getFs(IDBDataFile::Types type) throw std::runtime_error(oss.str()); } - return *(s_plugins[type].filesystem); + return *(s_plugins.at(type).filesystem); } +FileFactoryEnt::~FileFactoryEnt() +{ + delete filesystem; + delete factory; +} + + } // namespace idbdatafile diff --git a/utils/idbdatafile/IDBFactory.h b/utils/idbdatafile/IDBFactory.h index ab17111e7..da4809db5 100644 --- a/utils/idbdatafile/IDBFactory.h +++ b/utils/idbdatafile/IDBFactory.h @@ -31,21 +31,31 @@ class IDBFileSystem; struct FileFactoryEnt { - FileFactoryEnt() : type(IDBDataFile::UNKNOWN), name("unknown"), factory(0), filesystem(0) - { - ; - } - FileFactoryEnt(IDBDataFile::Types t, const std::string& n, FileFactoryBase* f, IDBFileSystem* fs) : type(t), name(n), factory(f), filesystem(fs) { ; } - IDBDataFile::Types type; - std::string name; - FileFactoryBase* factory; - IDBFileSystem* filesystem; + FileFactoryEnt(const FileFactoryEnt&) = delete; + FileFactoryEnt& operator=(const FileFactoryEnt&) = delete; + FileFactoryEnt& operator=(FileFactoryEnt&&) = delete; + FileFactoryEnt(FileFactoryEnt&& temporary) + : factory(temporary.factory) + , filesystem(temporary.filesystem) + { + temporary.factory = nullptr; + temporary.filesystem = nullptr; + } + + + + ~FileFactoryEnt(); + + IDBDataFile::Types type = IDBDataFile::UNKNOWN; + std::string name = "unknown"; + FileFactoryBase* factory = nullptr; + IDBFileSystem* filesystem = nullptr; }; typedef FileFactoryEnt (*FileFactoryEntryFunc)(); @@ -101,8 +111,8 @@ class IDBFactory static FactoryMap s_plugins; - IDBFactory(); - virtual ~IDBFactory(); + IDBFactory() = delete; + ~IDBFactory() = delete; }; inline const std::string& IDBFactory::name(IDBDataFile::Types type) @@ -112,7 +122,7 @@ inline const std::string& IDBFactory::name(IDBDataFile::Types type) throw std::runtime_error("unknown plugin type in IDBFactory::name"); } - return s_plugins[type].name; + return s_plugins.at(type).name; } } // namespace idbdatafile diff --git a/utils/messageqcpp/messagequeuepool.cpp b/utils/messageqcpp/messagequeuepool.cpp index 19667f7d6..bdd812d80 100644 --- a/utils/messageqcpp/messagequeuepool.cpp +++ b/utils/messageqcpp/messagequeuepool.cpp @@ -15,18 +15,25 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include #include #include -#include #include +#include #include "messagequeuepool.h" #include "messagequeue.h" namespace messageqcpp { -boost::mutex queueMutex; + +std::mutex& getQueueMutex() +{ + static std::mutex queueMutex; + return queueMutex; +} + // Make linker happy -std::multimap MessageQueueClientPool::clientMap; +MessageQueueClientPool::ClientMapType MessageQueueClientPool::clientMap; // 300 seconds idle until cleanup #define MAX_IDLE_TIME 300 @@ -38,7 +45,7 @@ static uint64_t TimeSpecToSeconds(struct timespec* ts) MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrIp, uint64_t port) { - boost::mutex::scoped_lock lock(queueMutex); + std::scoped_lock lock(getQueueMutex()); std::ostringstream oss; oss << dnOrIp << "_" << port; @@ -58,16 +65,16 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& dnOrI clock_gettime(CLOCK_MONOTONIC, &now); uint64_t nowSeconds = TimeSpecToSeconds(&now); - newClientObject->client = new MessageQueueClient(dnOrIp, port); + newClientObject->client.reset(new MessageQueueClient(dnOrIp, port)); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; - clientMap.insert(std::pair(searchString, newClientObject)); - return newClientObject->client; + clientMap.emplace(std::move(searchString), std::move(newClientObject)); + return newClientObject->client.get(); } MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& module) { - boost::mutex::scoped_lock lock(queueMutex); + std::scoped_lock lock(getQueueMutex()); MessageQueueClient* returnClient = MessageQueueClientPool::findInPool(module); @@ -78,16 +85,19 @@ MessageQueueClient* MessageQueueClientPool::getInstance(const std::string& modul } // We didn't find one, create new one - ClientObject* newClientObject = new ClientObject(); + auto newClientObject = std::make_unique(); struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); uint64_t nowSeconds = TimeSpecToSeconds(&now); - newClientObject->client = new MessageQueueClient(module); + + + newClientObject->client.reset(new MessageQueueClient(module)); newClientObject->inUse = true; newClientObject->lastUsed = nowSeconds; - clientMap.insert(std::pair(module, newClientObject)); - return newClientObject->client; + auto result = newClientObject->client.get(); + clientMap.emplace(std::move(module), std::move(newClientObject)); + return result; } MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search) @@ -97,22 +107,21 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search uint64_t nowSeconds = TimeSpecToSeconds(&now); MessageQueueClient* returnClient = NULL; - std::multimap::iterator it = clientMap.begin(); + auto it = clientMap.begin(); + // Scan pool while (it != clientMap.end()) { - ClientObject* clientObject = it->second; + ClientObject* clientObject = it->second.get(); uint64_t elapsedTime = nowSeconds - clientObject->lastUsed; // If connection hasn't been used for MAX_IDLE_TIME we probably don't need it so drop it // Don't drop in use connections that have been in use a long time if ((elapsedTime >= MAX_IDLE_TIME) && (!clientObject->inUse)) { - delete clientObject->client; - delete clientObject; // Do this so we don't invalidate current interator - std::multimap::iterator toDelete = it; + auto toDelete = it; it++; clientMap.erase(toDelete); continue; @@ -120,15 +129,13 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search if (!clientObject->inUse) { - MessageQueueClient* client = clientObject->client; + MessageQueueClient* client = clientObject->client.get(); // If the unused socket isn't connected or has data pending read, destroy it if (!client->isConnected() || client->hasData()) { - delete client; - delete clientObject; // Do this so we don't invalidate current interator - std::multimap::iterator toDelete = it; + auto toDelete = it; it++; clientMap.erase(toDelete); continue; @@ -140,7 +147,7 @@ MessageQueueClient* MessageQueueClientPool::findInPool(const std::string& search { if ((returnClient == NULL) && (!clientObject->inUse)) { - returnClient = clientObject->client; + returnClient = clientObject->client.get(); clientObject->inUse = true; return returnClient; } @@ -160,12 +167,13 @@ void MessageQueueClientPool::releaseInstance(MessageQueueClient* client) if (client == NULL) return; - boost::mutex::scoped_lock lock(queueMutex); - std::multimap::iterator it = clientMap.begin(); + + std::scoped_lock lock(getQueueMutex()); + auto it = clientMap.begin(); while (it != clientMap.end()) { - if (it->second->client == client) + if (it->second->client.get() == client) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); @@ -188,15 +196,14 @@ void MessageQueueClientPool::deleteInstance(MessageQueueClient* client) if (client == NULL) return; - boost::mutex::scoped_lock lock(queueMutex); - std::multimap::iterator it = clientMap.begin(); + + std::scoped_lock lock(getQueueMutex()); + auto it = clientMap.begin(); while (it != clientMap.end()) { - if (it->second->client == client) + if (it->second->client.get() == client) { - delete it->second->client; - delete it->second; clientMap.erase(it); return; } diff --git a/utils/messageqcpp/messagequeuepool.h b/utils/messageqcpp/messagequeuepool.h index 418f0a87b..bc5fb52da 100644 --- a/utils/messageqcpp/messagequeuepool.h +++ b/utils/messageqcpp/messagequeuepool.h @@ -19,22 +19,20 @@ #include #include "messagequeue.h" +#include namespace messageqcpp { struct ClientObject { - MessageQueueClient* client; - uint64_t lastUsed; - bool inUse; - - ClientObject() : client(NULL), lastUsed(0), inUse(false) - { - } + std::unique_ptr client; + uint64_t lastUsed = 0; + bool inUse = false; }; class MessageQueueClientPool { + using ClientMapType = std::multimap>; public: static MessageQueueClient* getInstance(const std::string& module); static MessageQueueClient* getInstance(const std::string& dnOrIp, uint64_t port); @@ -46,7 +44,7 @@ class MessageQueueClientPool MessageQueueClientPool(){}; ~MessageQueueClientPool(){}; - static std::multimap clientMap; + static ClientMapType clientMap; }; } // namespace messageqcpp diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index c9983ce1b..4f6377631 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -50,19 +50,7 @@ namespace rowgroup { using cscType = execplan::CalpontSystemCatalog::ColDataType; -StringStore::StringStore() : empty(true), fUseStoreStringMutex(false) -{ -} -StringStore::StringStore(const StringStore&) -{ - throw logic_error("Don't call StringStore copy ctor"); -} - -StringStore& StringStore::operator=(const StringStore&) -{ - throw logic_error("Don't call StringStore operator="); -} StringStore::~StringStore() { @@ -86,10 +74,10 @@ StringStore::~StringStore() uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) { - MemChunk* lastMC = NULL; + MemChunk* lastMC = nullptr; uint64_t ret = 0; - empty = false; // At least a NULL is being stored. + empty = false; // At least a nullptr is being stored. // Sometimes the caller actually wants "" to be returned....... argggghhhh...... // if (len == 0) @@ -121,7 +109,7 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) } else { - if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4))) + if ((lastMC == nullptr) || (lastMC->capacity - lastMC->currentSize < (len + 4))) { // mem usage debugging // if (lastMC) @@ -215,20 +203,12 @@ void StringStore::clear() empty = true; } -UserDataStore::UserDataStore() : fUseUserDataMutex(false) -{ -} - -UserDataStore::~UserDataStore() -{ -} - uint32_t UserDataStore::storeUserData(mcsv1sdk::mcsv1Context& context, boost::shared_ptr data, uint32_t len) { uint32_t ret = 0; - if (len == 0 || data == NULL) + if (len == 0 || data == nullptr) { return numeric_limits::max(); } @@ -305,7 +285,7 @@ void UserDataStore::deserialize(ByteStream& bs) } mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - mcsv1sdk::UserData* userData = NULL; + mcsv1sdk::UserData* userData = nullptr; rc = funcIter->second->createUserData(userData, vStoreData[i].length); if (rc != mcsv1sdk::mcsv1_UDAF::SUCCESS) @@ -323,10 +303,6 @@ void UserDataStore::deserialize(ByteStream& bs) return; } -RGData::RGData() -{ - // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; -} RGData::RGData(const RowGroup& rg, uint32_t rowCount) { @@ -336,6 +312,9 @@ RGData::RGData(const RowGroup& rg, uint32_t rowCount) if (rg.usesStringTable() && rowCount > 0) strings.reset(new StringStore()); + userDataStore.reset(); + + #ifdef VALGRIND /* In a PM-join, we can serialize entire tables; not every value has been * filled in yet. Need to look into that. Valgrind complains that @@ -354,6 +333,8 @@ RGData::RGData(const RowGroup& rg) if (rg.usesStringTable()) strings.reset(new StringStore()); + userDataStore.reset(); + #ifdef VALGRIND /* In a PM-join, we can serialize entire tables; not every value has been * filled in yet. Need to look into that. Valgrind complains that @@ -366,6 +347,7 @@ RGData::RGData(const RowGroup& rg) void RGData::reinit(const RowGroup& rg, uint32_t rowCount) { rowData.reset(new uint8_t[rg.getDataSize(rowCount)]); + userDataStore.reset(); if (rg.usesStringTable()) strings.reset(new StringStore()); @@ -386,16 +368,6 @@ void RGData::reinit(const RowGroup& rg) reinit(rg, 8192); } -RGData::RGData(const RGData& r) : rowData(r.rowData), strings(r.strings), userDataStore(r.userDataStore) -{ - // cout << "rgdata++ = " << __sync_add_and_fetch(&rgDataCount, 1) << endl; -} - -RGData::~RGData() -{ - // cout << "rgdata-- = " << __sync_sub_and_fetch(&rgDataCount, 1) << endl; -} - void RGData::serialize(ByteStream& bs, uint32_t amount) const { // cout << "serializing!\n"; @@ -465,6 +437,7 @@ void RGData::clear() { rowData.reset(); strings.reset(); + userDataStore.reset(); } // UserDataStore is only used for UDAF. @@ -479,10 +452,6 @@ UserDataStore* RGData::getUserDataStore() return userDataStore.get(); } -Row::Row() : data(NULL), strings(NULL), userDataStore(NULL) -{ -} - Row::Row(const Row& r) : columnCount(r.columnCount) , baseRid(r.baseRid) @@ -502,11 +471,7 @@ Row::Row(const Row& r) , hasLongStringField(r.hasLongStringField) , sTableThreshold(r.sTableThreshold) , forceInline(r.forceInline) - , userDataStore(NULL) -{ -} - -Row::~Row() + , userDataStore(nullptr) { } @@ -1037,7 +1002,7 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const const CHARSET_INFO* Row::getCharset(uint32_t col) const { - if (charsets[col] == NULL) + if (charsets[col] == nullptr) { const_cast(charsets)[col] = &datatypes::Charset(charsetNumbers[col]).getCharset(); } @@ -1045,14 +1010,6 @@ const CHARSET_INFO* Row::getCharset(uint32_t col) const } RowGroup::RowGroup() - : columnCount(0) - , data(NULL) - , rgData(NULL) - , strings(NULL) - , useStringTable(true) - , hasCollation(false) - , hasLongStringField(false) - , sTableThreshold(20) { // 1024 is too generous to waste. oldOffsets.reserve(10); @@ -1071,7 +1028,7 @@ RowGroup::RowGroup(uint32_t colCount, const vector& positions, const v const vector& cprecision, uint32_t stringTableThreshold, bool stringTable, const vector& forceInlineData) : columnCount(colCount) - , data(NULL) + , data(nullptr) , oldOffsets(positions) , oids(roids) , keys(tkeys) @@ -1079,8 +1036,8 @@ RowGroup::RowGroup(uint32_t colCount, const vector& positions, const v , charsetNumbers(csNumbers) , scale(cscale) , precision(cprecision) - , rgData(NULL) - , strings(NULL) + , rgData(nullptr) + , strings(nullptr) , sTableThreshold(stringTableThreshold) { uint32_t i; @@ -1121,8 +1078,8 @@ RowGroup::RowGroup(uint32_t colCount, const vector& positions, const v useStringTable = (stringTable && hasLongStringField); offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); - // Set all the charsets to NULL for jit initialization. - charsets.insert(charsets.begin(), charsetNumbers.size(), NULL); + // Set all the charsets to nullptr for jit initialization. + charsets.insert(charsets.begin(), charsetNumbers.size(), nullptr); } RowGroup::RowGroup(const RowGroup& r) @@ -1189,14 +1146,6 @@ RowGroup& RowGroup::operator=(const RowGroup& r) } RowGroup::RowGroup(ByteStream& bs) - : columnCount(0) - , data(nullptr) - , rgData(nullptr) - , strings(nullptr) - , useStringTable(true) - , hasCollation(false) - , hasLongStringField(false) - , sTableThreshold(20) { this->deserialize(bs); } @@ -1267,22 +1216,13 @@ void RowGroup::deserialize(ByteStream& bs) else if (!useStringTable && !oldOffsets.empty()) offsets = &oldOffsets[0]; - // Set all the charsets to NULL for jit initialization. - charsets.insert(charsets.begin(), charsetNumbers.size(), NULL); + // Set all the charsets to nullptr for jit initialization. + charsets.insert(charsets.begin(), charsetNumbers.size(), nullptr); } void RowGroup::serializeRGData(ByteStream& bs) const { - // cout << "****** serializing\n" << toString() << en - // if (useStringTable || !hasLongStringField) rgData->serialize(bs, getDataSize()); - // else { - // uint64_t size; - // RGData *compressed = convertToStringTable(&size); - // compressed->serialize(bs, size); - // if (compressed != rgData) - // delete compressed; - // } } uint32_t RowGroup::getDataSize() const @@ -1367,7 +1307,7 @@ string RowGroup::toString(const std::vector& used) const // os << "strings = " << hex << (int64_t) strings << "\n"; // os << "data = " << (int64_t) data << "\n" << dec; - if (data != NULL) + if (data != nullptr) { Row r; initRow(&r); @@ -1589,7 +1529,7 @@ void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& const CHARSET_INFO* RowGroup::getCharset(uint32_t col) { - if (charsets[col] == NULL) + if (charsets[col] == nullptr) { charsets[col] = &datatypes::Charset(charsetNumbers[col]).getCharset(); } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 6c724bc68..2b3fe5e25 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -129,7 +129,11 @@ inline T derefFromTwoVectorPtrs(const std::vector* outer, const std::vector> mem; // To store strings > 64KB (BLOB/TEXT) std::vector> longStrings; - bool empty; - bool fUseStoreStringMutex; //@bug6065, make StringStore::storeString() thread safe + bool empty = true; + bool fUseStoreStringMutex = false; //@bug6065, make StringStore::storeString() thread safe boost::mutex fMutex; }; @@ -208,8 +209,13 @@ class UserDataStore }; public: - UserDataStore(); - virtual ~UserDataStore(); + UserDataStore() = default; + virtual ~UserDataStore() = default; + UserDataStore(const UserDataStore&) = delete; + UserDataStore(UserDataStore&&) = delete; + UserDataStore& operator=(const UserDataStore&) = delete; + UserDataStore& operator=(UserDataStore&&) = delete; + void serialize(messageqcpp::ByteStream&) const; void deserialize(messageqcpp::ByteStream&); @@ -231,12 +237,10 @@ class UserDataStore boost::shared_ptr getUserData(uint32_t offset) const; private: - UserDataStore(const UserDataStore&); - UserDataStore& operator=(const UserDataStore&); std::vector vStoreData; - bool fUseUserDataMutex; + bool fUseUserDataMutex = false; boost::mutex fMutex; }; @@ -248,13 +252,16 @@ class Row; class RGData { public: - RGData(); // useless unless followed by an = or a deserialize operation + RGData() = default; // useless unless followed by an = or a deserialize operation RGData(const RowGroup& rg, uint32_t rowCount); // allocates memory for rowData explicit RGData(const RowGroup& rg); - RGData(const RGData&); - virtual ~RGData(); + RGData& operator=(const RGData&) = default; + RGData& operator=(RGData&&) = default; + RGData(const RGData&) = default; + RGData(RGData&&) = default; + virtual ~RGData() = default; + - inline RGData& operator=(const RGData&); // amount should be the # returned by RowGroup::getDataSize() void serialize(messageqcpp::ByteStream&, uint32_t amount) const; @@ -268,7 +275,7 @@ class RGData void clear(); void reinit(const RowGroup& rg); void reinit(const RowGroup& rg, uint32_t rowCount); - inline void setStringStore(boost::shared_ptr& ss) + inline void setStringStore(std::shared_ptr& ss) { strings = ss; } @@ -301,18 +308,21 @@ class RGData return (userDataStore ? (userDataStore->useUserDataMutex()) : false); } - boost::shared_array rowData; - boost::shared_ptr strings; - boost::shared_ptr userDataStore; + bool hasRowData() const + { + return !!rowData; + } private: - // boost::shared_array rowData; - // boost::shared_ptr strings; + std::shared_ptr rowData; + std::shared_ptr strings; + std::shared_ptr userDataStore; // Need sig to support backward compat. RGData can deserialize both forms. static const uint32_t RGDATA_SIG = 0xffffffff; // won't happen for 'old' Rowgroup data friend class RowGroup; + friend class RowGroupStorage; }; class Row @@ -320,28 +330,26 @@ class Row public: struct Pointer { - inline Pointer() : data(NULL), strings(NULL), userDataStore(NULL) - { - } + inline Pointer() = default; // Pointer(uint8_t*) implicitly makes old code compatible with the string table impl; - inline Pointer(uint8_t* d) : data(d), strings(NULL), userDataStore(NULL) + inline Pointer(uint8_t* d) : data(d) { } - inline Pointer(uint8_t* d, StringStore* s) : data(d), strings(s), userDataStore(NULL) + inline Pointer(uint8_t* d, StringStore* s) : data(d), strings(s) { } inline Pointer(uint8_t* d, StringStore* s, UserDataStore* u) : data(d), strings(s), userDataStore(u) { } - uint8_t* data; - StringStore* strings; - UserDataStore* userDataStore; + uint8_t* data = nullptr; + StringStore* strings = nullptr; + UserDataStore* userDataStore = nullptr; }; - Row(); + Row() = default; Row(const Row&); - ~Row(); + ~Row() = default; Row& operator=(const Row&); bool operator==(const Row&) const; @@ -491,7 +499,7 @@ class Row template inline void setBinaryField_offset(const T* value, uint32_t width, uint32_t colIndex); // support VARBINARY - // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are + // Add 2-byte length at the CHARSET_INFO*beginning of the field. nullptr and zero length field are // treated the same, could use one of the length bit to distinguish these two cases. inline std::string getVarBinaryStringField(uint32_t colIndex) const; inline void setVarBinaryField(const std::string& val, uint32_t colIndex); @@ -579,31 +587,32 @@ class Row const CHARSET_INFO* getCharset(uint32_t col) const; - private: - uint32_t columnCount; - uint64_t baseRid; +private: + inline bool inStringTable(uint32_t col) const; + +private: + uint32_t columnCount = 0; + uint64_t baseRid = 0; // Note, the mem behind these pointer fields is owned by RowGroup not Row - uint32_t* oldOffsets; - uint32_t* stOffsets; - uint32_t* offsets; - uint32_t* colWidths; - execplan::CalpontSystemCatalog::ColDataType* types; - uint32_t* charsetNumbers; - CHARSET_INFO** charsets; - uint8_t* data; - uint32_t* scale; - uint32_t* precision; + uint32_t* oldOffsets = nullptr; + uint32_t* stOffsets = nullptr; + uint32_t* offsets = nullptr; + uint32_t* colWidths = nullptr; + execplan::CalpontSystemCatalog::ColDataType* types = nullptr; + uint32_t* charsetNumbers = nullptr; + CHARSET_INFO** charsets = nullptr; + uint8_t* data = nullptr; + uint32_t* scale = nullptr; + uint32_t* precision = nullptr; - StringStore* strings; - bool useStringTable; - bool hasCollation; - bool hasLongStringField; - uint32_t sTableThreshold; + StringStore* strings = nullptr; + bool useStringTable = true; + bool hasCollation = false; + bool hasLongStringField = false; + uint32_t sTableThreshold = 20; boost::shared_array forceInline; - inline bool inStringTable(uint32_t col) const; - - UserDataStore* userDataStore; // For UDAF + UserDataStore* userDataStore = nullptr; // For UDAF friend class RowGroup; }; @@ -1478,9 +1487,6 @@ class RowGroup : public messageqcpp::Serializeable inline bool usesStringTable() const; inline void setUseStringTable(bool); - // RGData *convertToInlineData(uint64_t *size = NULL) const; // caller manages the memory returned by - // this void convertToInlineDataInPlace(); RGData *convertToStringTable(uint64_t *size = NULL) - // const; void convertToStringTableInPlace(); void serializeRGData(messageqcpp::ByteStream&) const; inline uint32_t getStringTableThreshold() const; @@ -1516,17 +1522,17 @@ class RowGroup : public messageqcpp::Serializeable const uint16_t& blockNum); inline void getLocation(uint32_t* partNum, uint16_t* segNum, uint8_t* extentNum, uint16_t* blockNum); - inline void setStringStore(boost::shared_ptr); + inline void setStringStore(std::shared_ptr); const CHARSET_INFO* getCharset(uint32_t col); private: - uint32_t columnCount; - uint8_t* data; + uint32_t columnCount = 0; + uint8_t* data = nullptr; std::vector oldOffsets; // inline data offsets std::vector stOffsets; // string table offsets - uint32_t* offsets; // offsets either points to oldOffsets or stOffsets + uint32_t* offsets = nullptr; // offsets either points to oldOffsets or stOffsets std::vector colWidths; // oids: the real oid of the column, may have duplicates with alias. // This oid is necessary for front-end to decide the real column width. @@ -1544,12 +1550,12 @@ class RowGroup : public messageqcpp::Serializeable std::vector precision; // string table impl - RGData* rgData; - StringStore* strings; // note, strings and data belong to rgData - bool useStringTable; - bool hasCollation; - bool hasLongStringField; - uint32_t sTableThreshold; + RGData* rgData = nullptr; + StringStore* strings = nullptr; // note, strings and data belong to rgData + bool useStringTable = true; + bool hasCollation = false; + bool hasLongStringField = false; + uint32_t sTableThreshold = 20; boost::shared_array forceInline; static const uint32_t headerSize = 18; @@ -1586,7 +1592,7 @@ every row, they're a measurable performance penalty */ inline uint32_t RowGroup::getRowCount() const { // idbassert(data); - // if (!data) throw std::logic_error("RowGroup::getRowCount(): data is NULL!"); + // if (!data) throw std::logic_error("RowGroup::getRowCount(): data is nullptr!"); return *((uint32_t*)&data[rowCountOffset]); } @@ -1617,8 +1623,8 @@ inline void RowGroup::getRow(uint32_t rowNum, Row* r) const inline void RowGroup::setData(uint8_t* d) { data = d; - strings = NULL; - rgData = NULL; + strings = nullptr; + rgData = nullptr; setUseStringTable(false); } @@ -1652,7 +1658,7 @@ inline void RowGroup::setUseStringTable(bool b) offsets = &oldOffsets[0]; if (!useStringTable) - strings = NULL; + strings = nullptr; } inline uint64_t RowGroup::getBaseRid() const @@ -1712,7 +1718,7 @@ inline uint32_t RowGroup::getRowSizeWithStrings() const inline uint64_t RowGroup::getSizeWithStrings(uint64_t n) const { - if (strings == NULL) + if (strings == nullptr) return getDataSize(n); else return getDataSize(n) + strings->getSize(); @@ -1836,7 +1842,7 @@ inline uint32_t RowGroup::getStringTableThreshold() const return sTableThreshold; } -inline void RowGroup::setStringStore(boost::shared_ptr ss) +inline void RowGroup::setStringStore(std::shared_ptr ss) { if (useStringTable) { @@ -2031,7 +2037,7 @@ inline bool StringStore::isNullValue(uint64_t off) const if (off == std::numeric_limits::max()) return true; - // Long strings won't be NULL + // Long strings won't be nullptr if (off & 0x8000000000000000) return false; @@ -2054,7 +2060,7 @@ inline bool StringStore::isNullValue(uint64_t off) const if ((offset + length) > mc->currentSize) return true; - if (mc->data[offset + 4] == 0) // "" = NULL string for some reason... + if (mc->data[offset + 4] == 0) // "" = nullptr string for some reason... return true; return (memcmp(&mc->data[offset + 4], joblist::CPNULLSTRMARK.c_str(), 8) == 0); } @@ -2121,14 +2127,6 @@ inline uint64_t StringStore::getSize() const return ret; } -inline RGData& RGData::operator=(const RGData& r) -{ - rowData = r.rowData; - strings = r.strings; - userDataStore = r.userDataStore; - return *this; -} - inline void RGData::getRow(uint32_t num, Row* row) { uint32_t size = row->getSize(); diff --git a/utils/udfsdk/libudf_mysql.rc b/utils/udfsdk/libudf_mysql.rc deleted file mode 100644 index f5420aa25..000000000 --- a/utils/udfsdk/libudf_mysql.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB UDF API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "libudfsdk" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "libudfsdk.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/utils/udfsdk/libudfsdk.rc b/utils/udfsdk/libudfsdk.rc deleted file mode 100644 index f5420aa25..000000000 --- a/utils/udfsdk/libudfsdk.rc +++ /dev/null @@ -1,102 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB, Inc." - VALUE "FileDescription", "InfiniDB UDF API" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "libudfsdk" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "libudfsdk.dll" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index 180b09f31..a9e88decc 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include "messagequeue.h" #include "bytestream.h" @@ -57,21 +58,18 @@ void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double namespace BRM { -SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) - : slave(s) - , currentSaveFile(NULL) - , journalh(NULL) +SlaveComm::SlaveComm(string hostname) { config::Config* config = config::Config::makeConfig(); string tmp; - + slave = std::make_unique(); bool tellUser = true; for (;;) { try { - server = new MessageQueueServer(hostname); + server = std::make_unique(hostname); break; } catch (runtime_error& re) @@ -133,8 +131,8 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) journalName = savefile + "_journal"; const char* filename = journalName.c_str(); - journalh = IDBDataFile::open(IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0); - if (journalh == NULL) + journalh.reset(IDBDataFile::open(IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0)); + if (journalh == nullptr) throw runtime_error("Could not open the BRM journal for writing!"); } else @@ -162,8 +160,6 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) } SlaveComm::SlaveComm() - : currentSaveFile(NULL) - , journalh(NULL) { config::Config* config = config::Config::makeConfig(); @@ -192,23 +188,9 @@ SlaveComm::SlaveComm() server = NULL; standalone = true; printOnly = false; - slave = new SlaveDBRMNode(); + slave = std::make_unique(); } -SlaveComm::~SlaveComm() -{ - delete server; - server = NULL; - - if (firstSlave) - { - delete currentSaveFile; - currentSaveFile = NULL; - } - - delete journalh; - journalh = NULL; -} void SlaveComm::stop() { @@ -1946,8 +1928,7 @@ void SlaveComm::do_confirm() { if (!currentSaveFile) { - currentSaveFile = - IDBDataFile::open(IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0); + currentSaveFile.reset(IDBDataFile::open(IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0)); } if (currentSaveFile == NULL) @@ -1970,7 +1951,7 @@ void SlaveComm::do_confirm() if (err < (int)relative.length()) { ostringstream os; - os << "WorkerComm: currentfile write() returned " << err << " file pointer is " << currentSaveFile; + os << "WorkerComm: currentfile write() returned " << err << " file pointer is " << currentSaveFile.get(); if (err < 0) os << " errno: " << strerror(errno); @@ -1979,13 +1960,13 @@ void SlaveComm::do_confirm() } currentSaveFile->flush(); - delete currentSaveFile; - currentSaveFile = NULL; + + currentSaveFile = nullptr; saveFileToggle = !saveFileToggle; - delete journalh; - journalh = IDBDataFile::open(IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG), - journalName.c_str(), "w+b", 0); + ; + journalh.reset(IDBDataFile::open(IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG), + journalName.c_str(), "w+b", 0)); if (!journalh) throw runtime_error("Could not open the BRM journal for writing!"); @@ -2245,7 +2226,6 @@ void SlaveComm::do_ownerCheck(ByteStream& msg) master.write(reply); } -// FIXME: needs to be refactored along with SessionManagerServer::lookupProcessStatus() bool SlaveComm::processExists(const uint32_t pid, const string& pname) { string stat; @@ -2278,7 +2258,6 @@ bool SlaveComm::processExists(const uint32_t pid, const string& pname) return true; } - void SlaveComm::do_dmlLockLBIDRanges(ByteStream& msg) { ByteStream reply; diff --git a/versioning/BRM/slavecomm.h b/versioning/BRM/slavecomm.h index 48a2f5b54..18ad2af15 100644 --- a/versioning/BRM/slavecomm.h +++ b/versioning/BRM/slavecomm.h @@ -54,8 +54,10 @@ class SlaveComm EXPORT SlaveComm(); /** Use this ctor to have it connected to the rest of the DBRM system */ - EXPORT SlaveComm(std::string hostname, SlaveDBRMNode* s); // hostname = 'DBRM_WorkerN' - EXPORT ~SlaveComm(); + EXPORT SlaveComm(std::string hostname); // hostname = 'DBRM_WorkerN' + EXPORT ~SlaveComm() {}; + + SlaveDBRMNode& getSlaveNode() { return *slave; } EXPORT void run(); EXPORT void stop(); @@ -112,15 +114,15 @@ class SlaveComm void saveDelta(); bool processExists(const uint32_t pid, const std::string& pname); - messageqcpp::MessageQueueServer* server; + std::unique_ptr server; messageqcpp::IOSocket master; - SlaveDBRMNode* slave; + std::unique_ptr slave; std::string savefile; bool release, die, firstSlave, saveFileToggle, takeSnapshot, doSaveDelta, standalone, printOnly; messageqcpp::ByteStream delta; - idbdatafile::IDBDataFile* currentSaveFile; + std::unique_ptr currentSaveFile; std::string journalName; - idbdatafile::IDBDataFile* journalh; + std::unique_ptr journalh; int64_t snapshotInterval, journalCount; struct timespec MSG_TIMEOUT; }; diff --git a/versioning/BRM/slavenode.cpp b/versioning/BRM/slavenode.cpp index 071132a30..1e835dd25 100644 --- a/versioning/BRM/slavenode.cpp +++ b/versioning/BRM/slavenode.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include "slavedbrmnode.h" @@ -39,7 +40,7 @@ using namespace BRM; using namespace std; -SlaveComm* comm; +std::unique_ptr comm; bool die = false; boost::thread_group monitorThreads; @@ -120,12 +121,11 @@ int ServiceWorkerNode::Child() { setupChildSignalHandlers(); - SlaveDBRMNode slave; ShmKeys keys; try { - comm = new SlaveComm(std::string(m_nodename), &slave); + comm = std::make_unique(std::string(m_nodename)); NotifyServiceStarted(); } catch (exception& e) @@ -139,12 +139,12 @@ int ServiceWorkerNode::Child() } /* Start 4 threads to monitor write lock state */ - monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMFLLockStatus(), keys.KEYRANGE_EMFREELIST_BASE)); - monitorThreads.create_thread(RWLockMonitor(&die, slave.getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE)); - monitorThreads.create_thread(RWLockMonitor(&die, slave.getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE)); - monitorThreads.create_thread(RWLockMonitor(&die, slave.getVSSLockStatus(), keys.KEYRANGE_VSS_BASE)); + monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getEMFLLockStatus(), keys.KEYRANGE_EMFREELIST_BASE)); + monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getEMLockStatus(), keys.KEYRANGE_EXTENTMAP_BASE)); + monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getVBBMLockStatus(), keys.KEYRANGE_VBBM_BASE)); + monitorThreads.create_thread(RWLockMonitor(&die, comm->getSlaveNode().getVSSLockStatus(), keys.KEYRANGE_VSS_BASE)); monitorThreads.create_thread( - RWLockMonitor(&die, slave.getEMIndexLockStatus(), keys.KEYRANGE_EXTENTMAP_INDEX_BASE)); + RWLockMonitor(&die, comm->getSlaveNode().getEMIndexLockStatus(), keys.KEYRANGE_EXTENTMAP_INDEX_BASE)); try { diff --git a/writeengine/build/startdbrm b/writeengine/build/startdbrm deleted file mode 100755 index 3b4a68ca7..000000000 --- a/writeengine/build/startdbrm +++ /dev/null @@ -1,2 +0,0 @@ -nohup controlernode & -nohup workernode DBRM_Worker1 & diff --git a/writeengine/build/we_rules.mak b/writeengine/build/we_rules.mak deleted file mode 100644 index bc750d7f6..000000000 --- a/writeengine/build/we_rules.mak +++ /dev/null @@ -1,48 +0,0 @@ -IPCS_CLEANUP=$(EXPORT_ROOT)/bin/dbrm stop; ipcs-pat -d > /dev/null; sleep 2; $(EXPORT_ROOT)/bin/dbrm start; - -CPPFLAGS=-I. -I$(EXPORT_ROOT)/include -I/usr/include/libxml2 - -CXXFLAGS+=$(DEBUG_FLAGS) -D_FILE_OFFSET_BITS=64 -Wall -fpic -LLIBS=-L. -L$(EXPORT_ROOT)/lib -lxml2 -lrwlock -lconfigcpp -lmessageqcpp -lbrm -lboost_thread \ - -lloggingcpp -lboost_date_time -lboost_filesystem -TLIBS=$(LLIBS) -lcppunit -ldl -GLIBS=$(TLIBS:-lcppunit=) - -LIBDIR=../obj -LOBJS_SHARED= \ - $(LIBDIR)/we_blockop.o \ - $(LIBDIR)/we_brm.o \ - $(LIBDIR)/we_bulkrollbackmgr.o \ - $(LIBDIR)/we_bulkrollbackfile.o \ - $(LIBDIR)/we_bulkrollbackfilecompressed.o \ - $(LIBDIR)/we_bulkrollbackfilecompressedhdfs.o \ - $(LIBDIR)/we_cache.o \ - $(LIBDIR)/we_chunkmanager.o \ - $(LIBDIR)/we_config.o \ - $(LIBDIR)/we_confirmhdfsdbfile.o \ - $(LIBDIR)/we_convertor.o \ - $(LIBDIR)/we_dbfileop.o \ - $(LIBDIR)/we_dbrootextenttracker.o \ - $(LIBDIR)/we_define.o \ - $(LIBDIR)/we_fileop.o \ - $(LIBDIR)/we_log.o \ - $(LIBDIR)/we_rbmetawriter.o \ - $(LIBDIR)/we_simplesyslog.o \ - $(LIBDIR)/we_stats.o - -LOBJS_INDEX= \ - $(LIBDIR)/we_freemgr.o \ - $(LIBDIR)/we_indexlist.o \ - $(LIBDIR)/we_indexlist_common.o \ - $(LIBDIR)/we_indexlist_find_delete.o \ - $(LIBDIR)/we_indexlist_multiple_narray.o \ - $(LIBDIR)/we_indexlist_narray.o \ - $(LIBDIR)/we_indexlist_update_hdr_sub.o \ - $(LIBDIR)/we_indextree.o - -LOBJS_DCTNRY= $(LIBDIR)/we_dctnry.o -LOBJS_XML= $(LIBDIR)/we_xmlop.o \ - $(LIBDIR)/we_xmljob.o \ - $(LIBDIR)/we_xmlgendata.o \ - $(LIBDIR)/we_xmlgenproc.o - diff --git a/writeengine/bulk/cpimport.rc b/writeengine/bulk/cpimport.rc deleted file mode 100644 index d0ef7a6fb..000000000 --- a/writeengine/bulk/cpimport.rc +++ /dev/null @@ -1,100 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (United States) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "CompanyName", "InfiniDB" - VALUE "FileDescription", "InfiniDB Bulk Loader" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "cpimport" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "cpimport.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (United States) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 9af6e8e23..098f3dbb2 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -2415,12 +2415,10 @@ void BulkLoadBuffer::tokenize(const boost::ptr_vector& columnsInfo, //------------------------------------------------------------------ case FLD_PARSE_ENCLOSED_STATE: { - char next = *(p + 1); - if ((p + 1 < pEndOfData) && - (((c == ESCAPE_CHAR) && ((next == STRING_ENCLOSED_CHAR) || (next == ESCAPE_CHAR) || - (next == LINE_FEED) || (next == CARRIAGE_RETURN))) || - ((c == STRING_ENCLOSED_CHAR) && (next == STRING_ENCLOSED_CHAR)))) + (((c == ESCAPE_CHAR) && ((*(p + 1) == STRING_ENCLOSED_CHAR) || (*(p + 1) == ESCAPE_CHAR) || + (*(p + 1) == LINE_FEED) || (*(p + 1) == CARRIAGE_RETURN))) || + ((c == STRING_ENCLOSED_CHAR) && (*(p + 1) == STRING_ENCLOSED_CHAR)))) { // Create/save original data before stripping out bytes if (rawDataRowLength == 0) @@ -2515,7 +2513,7 @@ void BulkLoadBuffer::tokenize(const boost::ptr_vector& columnsInfo, // cout << "triming ... " << endl; char* tmp = p; - while (*(--tmp) == ' ') + while (tmp != lastRowHead && *(--tmp) == ' ') { // cout << "offset is " << offset < FileOp::m_DbRootAddExtentMutexes; +/*static*/ std::map FileOp::m_DbRootAddExtentMutexes; // in 1 call to fwrite(), during initialization // StopWatch timer; @@ -993,7 +993,7 @@ int FileOp::initColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, c Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_COL_EXTENT); #endif - boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); + boost::mutex::scoped_lock lk(m_DbRootAddExtentMutexes[dbRoot]); #ifdef PROFILE if (bExpandExtent) @@ -1714,7 +1714,7 @@ int FileOp::initDctnryExtent(IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, u Stats::startParseEvent(WE_STATS_WAIT_TO_CREATE_DCT_EXTENT); #endif - boost::mutex::scoped_lock lk(*m_DbRootAddExtentMutexes[dbRoot]); + boost::mutex::scoped_lock lk(m_DbRootAddExtentMutexes[dbRoot]); #ifdef PROFILE if (bExpandExtent) @@ -1809,33 +1809,13 @@ void FileOp::initDbRootExtentMutexes() for (size_t i = 0; i < rootIds.size(); i++) { - boost::mutex* pM = new boost::mutex; - m_DbRootAddExtentMutexes[rootIds[i]] = pM; + m_DbRootAddExtentMutexes.emplace(std::piecewise_construct, + std::forward_as_tuple(rootIds[i]), + std::forward_as_tuple()); } } } -/*********************************************************** - * DESCRIPTION: - * Cleans up memory allocated to the DBRoot extent mutexes. Calling - * this function is not necessary, but it is provided for completeness, - * to complement initDbRootExtentMutexes(), and to provide a way to - * free up memory at the end of program execution. - ***********************************************************/ -/* static */ -void FileOp::removeDbRootExtentMutexes() -{ - boost::mutex::scoped_lock lk(m_createDbRootMutexes); - - std::map::iterator k = m_DbRootAddExtentMutexes.begin(); - - while (k != m_DbRootAddExtentMutexes.end()) - { - delete k->second; - ++k; - } -} - /*********************************************************** * DESCRIPTION: * Write out (reinitialize) a partial extent in a column file. diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 7a5d8a03a..e93d15897 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -443,7 +443,6 @@ class FileOp : public BlockOp, public WeUIDGID execplan::CalpontSystemCatalog::ColDataType colDataType); static void initDbRootExtentMutexes(); - static void removeDbRootExtentMutexes(); int writeInitialCompColumnChunk(IDBDataFile* pFile, int nBlocksAllocated, int nRows, const uint8_t* emptyVal, int width, BRM::LBID_t lbid, @@ -457,7 +456,7 @@ class FileOp : public BlockOp, public WeUIDGID static boost::mutex m_createDbRootMutexes; // Mutexes used to serialize extent creation within each DBRoot - static std::map m_DbRootAddExtentMutexes; + static std::map m_DbRootAddExtentMutexes; // protect race condition in creating directories static boost::mutex m_mkdirMutex; diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index dca7c05db..9214165f7 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -399,8 +399,8 @@ struct JobColumn /** @brief Job Column Structure */ int compressionType; /** @brief compression type */ bool autoIncFlag; /** @brief auto increment flag */ DctnryStruct dctnry; /** @brief dictionary structure */ - int64_t fMinIntSat; /** @brief For integer type, the min saturation value */ - uint64_t fMaxIntSat; /** @brief For integer type, the max saturation value */ + int128_t fMinIntSat; /** @brief For integer type, the min saturation value */ + uint128_t fMaxIntSat; /** @brief For integer type, the max saturation value */ double fMinDblSat; /** @brief for float/double, the min saturation value */ double fMaxDblSat; /** @brief for float/double, the max saturation value */ bool fWithDefault; /** @brief With default */ diff --git a/writeengine/splitter/splitter.rc b/writeengine/splitter/splitter.rc deleted file mode 100644 index ff6051a6f..000000000 --- a/writeengine/splitter/splitter.rc +++ /dev/null @@ -1,101 +0,0 @@ -// Microsoft Visual C++ generated resource script. -// -#include "resource.h" - -#define APSTUDIO_READONLY_SYMBOLS -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 2 resource. -// -#include "afxres.h" - -///////////////////////////////////////////////////////////////////////////// -#undef APSTUDIO_READONLY_SYMBOLS - -///////////////////////////////////////////////////////////////////////////// -// English (U.S.) resources - -#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) -#ifdef _WIN32 -LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US -#pragma code_page(1252) -#endif //_WIN32 - -#ifdef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// TEXTINCLUDE -// - -1 TEXTINCLUDE -BEGIN - "resource.h\0" -END - -2 TEXTINCLUDE -BEGIN - "#include ""afxres.h""\r\n" - "\0" -END - -3 TEXTINCLUDE -BEGIN - "\r\n" - "\0" -END - -#endif // APSTUDIO_INVOKED - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION 4,6,0,0 - PRODUCTVERSION 4,6,0,0 - FILEFLAGSMASK 0x17L -#ifdef _DEBUG - FILEFLAGS 0x1L -#else - FILEFLAGS 0x0L -#endif - FILEOS 0x4L - FILETYPE 0x1L - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "FileDescription", "InfiniDB Bulk Loader" - VALUE "FileVersion", "4.6.0-0" - VALUE "InternalName", "splitter" - VALUE "LegalCopyright", "Copyright (C) 2014" - VALUE "OriginalFilename", "cpimport.exe" - VALUE "ProductName", "InfiniDB" - VALUE "ProductVersion", "4.6.0.0 Beta" - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -#endif // English (U.S.) resources -///////////////////////////////////////////////////////////////////////////// - - - -#ifndef APSTUDIO_INVOKED -///////////////////////////////////////////////////////////////////////////// -// -// Generated from the TEXTINCLUDE 3 resource. -// - - -///////////////////////////////////////////////////////////////////////////// -#endif // not APSTUDIO_INVOKED - diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 27efb1d37..32d126346 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -464,7 +464,7 @@ bool WECmdArgs::str2PmList(std::string& PmList, VecInts& V) if (aLen > 0) { - strncpy(aBuff, PmList.c_str(), BUFLEN); + strncpy(aBuff, PmList.c_str(), BUFLEN - 1); aBuff[BUFLEN - 1] = 0; } else diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 3723e7137..ba52b6198 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -65,6 +65,7 @@ using namespace idbdatafile; #include "dataconvert.h" #include "string_prefixes.h" +#include "mcs_decimal.h" namespace WriteEngine //#define PROFILE 1 @@ -917,7 +918,7 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, Column refCol; ColType newColType; ColType refColType; - boost::scoped_array defVal(new char[MAX_COLUMN_BOUNDARY]); + boost::scoped_array defVal(new char[datatypes::MAXDECIMALWIDTH]); ColumnOp* colOpNewCol = m_colOp[op(compressionType)]; ColumnOp* refColOp = m_colOp[op(refCompressionType)]; Dctnry* dctnry = m_dctnry[op(compressionType)]; @@ -992,6 +993,29 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, return rc; } +// TODO: Get rid of this +void emptyValueToAny(boost::any* any, const uint8_t* emptyValue, int colWidth) +{ + switch (colWidth) + { + case 16: + *any = *(uint128_t*)emptyValue; + break; + case 8: + *any = *(uint64_t*)emptyValue; + break; + case 4: + *any = *(uint32_t*)emptyValue; + break; + case 2: + *any = *(uint16_t*)emptyValue; + break; + default: + *any = *emptyValue; + } +} + + int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector& colExtentsColType, vector& colExtentsStruct, vector& colOldValueList, vector& ridLists, const int32_t tableOid, bool hasAUXCol) @@ -1032,10 +1056,7 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector const uint8_t* emptyVal = m_colOp[op(curColStruct.fCompressionType)]->getEmptyRowValue( curColStruct.colDataType, curColStruct.colWidth); - if (curColStruct.colWidth == datatypes::MAXDECIMALWIDTH) - curTuple.data = *(int128_t*)emptyVal; - else - curTuple.data = *(int64_t*)emptyVal; + emptyValueToAny(&curTuple.data, emptyVal, curColStruct.colWidth); curTupleList.push_back(curTuple); colValueList.push_back(curTupleList); diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index f5bd25e3f..deb29b26b 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -21,6 +21,7 @@ *******************************************************************************/ /** @file */ +#include "mcs_basic_types.h" #define WRITEENGINEXMLJOB_DLLEXPORT #include "we_xmljob.h" #undef WRITEENGINEXMLJOB_DLLEXPORT @@ -46,28 +47,6 @@ using namespace execplan; namespace WriteEngine { -// Maximum saturation value for DECIMAL types based on precision -// TODO MCOL-641 add support here. see dataconvert.cpp -const long long columnstore_precision[19] = {0, - 9, - 99, - 999, - 9999, - 99999, - 999999, - 9999999, - 99999999, - 999999999, - 9999999999LL, - 99999999999LL, - 999999999999LL, - 9999999999999LL, - 99999999999999LL, - 999999999999999LL, - 9999999999999999LL, - 99999999999999999LL, - 999999999999999999LL}; - //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ @@ -695,13 +674,14 @@ void XMLJob::initSatLimits(JobColumn& curColumn) const } else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::DECIMAL]) { - curColumn.fMinIntSat = -columnstore_precision[curColumn.precision]; - curColumn.fMaxIntSat = columnstore_precision[curColumn.precision]; + curColumn.fMaxIntSat = dataconvert::decimalRangeUp(curColumn.precision); + curColumn.fMinIntSat = -curColumn.fMaxIntSat; + } else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::UDECIMAL]) { curColumn.fMinIntSat = 0; - curColumn.fMaxIntSat = columnstore_precision[curColumn.precision]; + curColumn.fMaxIntSat = dataconvert::decimalRangeUp(curColumn.precision); } else if (curColumn.typeName == ColDataTypeStr[CalpontSystemCatalog::FLOAT]) {