From 098e49bd01bfc0ae07da480bb092c716f337bd1d Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 23 Jan 2019 10:29:12 -0600 Subject: [PATCH 1/7] MCOL-2001 Make the parameters for redistribute case insensitive --- oamapps/mcsadmin/mcsadmin.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index f2ea97596..8b31353ac 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -32,6 +32,7 @@ extern int h_errno; #include "boost/filesystem/path.hpp" #include "boost/scoped_ptr.hpp" #include "boost/tokenizer.hpp" +#include "boost/algorithm/string/predicate.hpp" #include "sessionmanager.h" #include "dbrm.h" #include "messagequeue.h" @@ -677,7 +678,7 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - if (arguments[1] == "start") + if (boost::iequals(arguments[1], "start")) { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -688,7 +689,7 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (arguments[2] == "remove") + if (boost::iequals(arguments[2], "remove")) { int dbroot; bool error = false; @@ -792,7 +793,7 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (arguments[1] == "stop") + else if (boost::iequals(arguments[1], "stop")) { ByteStream bs; // message WES ID, sequence #, action id @@ -802,7 +803,7 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (arguments[1] == "status") + else if (boost::iequals(arguments[1], "status")) { ByteStream bs; // message WES ID, sequence #, action id From bf4a215cc8d721be09cd2c311ab95c951ae8cef9 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 4 Feb 2019 12:32:04 -0600 Subject: [PATCH 2/7] MCOL-2001 make redistribute parameters case insensitive. --- oamapps/mcsadmin/mcsadmin.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 8b31353ac..dd7262d1e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -32,7 +32,6 @@ extern int h_errno; #include "boost/filesystem/path.hpp" #include "boost/scoped_ptr.hpp" #include "boost/tokenizer.hpp" -#include "boost/algorithm/string/predicate.hpp" #include "sessionmanager.h" #include "dbrm.h" #include "messagequeue.h" @@ -678,7 +677,11 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - if (boost::iequals(arguments[1], "start")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "start", 5) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "start", 5) == 0) +#endif { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -689,7 +692,11 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (boost::iequals(arguments[2], "remove")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "remove", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "remove", 6) == 0) +#endif { int dbroot; bool error = false; @@ -793,7 +800,11 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (boost::iequals(arguments[1], "stop")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id @@ -803,7 +814,11 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (boost::iequals(arguments[1], "status")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "status", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "status", 6) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id From b81455744d938c442d248b09b10ebbee65e4e2ee Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Feb 2019 13:43:21 -0600 Subject: [PATCH 3/7] Revert "Kill Jemalloc" This reverts commit e03f7dc88503c7044ebc5befdd432238e31d3f16. --- CMakeLists.txt | 8 ++++++- FindJeMalloc.cmake | 44 ++++++++++++++++++++++++++++++++++++++ build/infinidb.spec.in | 1 + build/infinidb.spec.in.4.5 | 1 + 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 FindJeMalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0932307ed..2645adc5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,12 @@ if (NOT LIBXML2_FOUND) MESSAGE(FATAL_ERROR "Could not find a usable libxml2 development environment!") endif() +INCLUDE (FindJeMalloc.cmake) +if (NOT JEMALLOC_FOUND) + message(NOTICE "jemalloc not found! Compiling with standard malloc") + SET(JEMALLOC_LIBRARIES "") +endif() + FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -123,7 +129,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt ${JEMALLOC_LIBRARIES}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk dataconvert common compress mysqlcl_idb querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) diff --git a/FindJeMalloc.cmake b/FindJeMalloc.cmake new file mode 100644 index 000000000..c97bb1b9f --- /dev/null +++ b/FindJeMalloc.cmake @@ -0,0 +1,44 @@ +# - Try to find jemalloc headers and libraries. +# +# Usage of this module as follows: +# +# find_package(JeMalloc) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# JEMALLOC_ROOT_DIR Set this variable to the root installation of +# jemalloc if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# JEMALLOC_FOUND System has jemalloc libs/headers +# JEMALLOC_LIBRARIES The jemalloc library/libraries +# JEMALLOC_INCLUDE_DIR The location of jemalloc headers + +find_path(JEMALLOC_ROOT_DIR + NAMES include/jemalloc/jemalloc.h +) + +find_library(JEMALLOC_LIBRARIES + NAMES jemalloc + HINTS ${JEMALLOC_ROOT_DIR}/lib +) + +find_path(JEMALLOC_INCLUDE_DIR + NAMES jemalloc/jemalloc.h + HINTS ${JEMALLOC_ROOT_DIR}/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(JeMalloc DEFAULT_MSG + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) + +mark_as_advanced( + JEMALLOC_ROOT_DIR + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) diff --git a/build/infinidb.spec.in b/build/infinidb.spec.in index 20a44e834..2531044a0 100644 --- a/build/infinidb.spec.in +++ b/build/infinidb.spec.in @@ -336,6 +336,7 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/mariadb/columnstore/lib/libmysqlcl_idb.so.1.0.0 /usr/local/mariadb/columnstore/lib/libquerystats.so.1.0.0 /usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1.0.0 +/usr/local/mariadb/columnstore/lib/libjemalloc.so.3.3.0 /usr/local/mariadb/columnstore/lib/libidbdatafile.so.1.0.0 /usr/local/mariadb/columnstore/lib/hdfs-20.so /usr/local/mariadb/columnstore/lib/hdfs-12.so diff --git a/build/infinidb.spec.in.4.5 b/build/infinidb.spec.in.4.5 index 966f2a73a..b8145d580 100755 --- a/build/infinidb.spec.in.4.5 +++ b/build/infinidb.spec.in.4.5 @@ -338,6 +338,7 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/Calpont/lib/libmysqlcl_idb.so.1.0.0 /usr/local/Calpont/lib/libquerystats.so.1.0.0 /usr/local/Calpont/lib/libwriteengineredistribute.so.1.0.0 +/usr/local/Calpont/lib/libjemalloc.so.3.3.0 /usr/local/Calpont/lib/libidbdatafile.so.1.0.0 /usr/local/Calpont/lib/hdfs-20.so /usr/local/Calpont/lib/hdfs-12.so From 52a741e6efdd7d22b29701ab1cce0ae86278886e Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Feb 2019 17:18:02 -0600 Subject: [PATCH 4/7] MCOL-2136: Force use of jemalloc for build and install. --- CMakeLists.txt | 2 +- build/infinidb.spec.in | 1 - build/infinidb.spec.in.4.5 | 1 - cpackEngineDEB.cmake | 2 +- cpackEngineRPM.cmake | 4 ++-- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2645adc5c..c72a74d61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,7 +94,7 @@ endif() INCLUDE (FindJeMalloc.cmake) if (NOT JEMALLOC_FOUND) - message(NOTICE "jemalloc not found! Compiling with standard malloc") + message(FATAL_ERROR "jemalloc not found!") SET(JEMALLOC_LIBRARIES "") endif() diff --git a/build/infinidb.spec.in b/build/infinidb.spec.in index 2531044a0..20a44e834 100644 --- a/build/infinidb.spec.in +++ b/build/infinidb.spec.in @@ -336,7 +336,6 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/mariadb/columnstore/lib/libmysqlcl_idb.so.1.0.0 /usr/local/mariadb/columnstore/lib/libquerystats.so.1.0.0 /usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1.0.0 -/usr/local/mariadb/columnstore/lib/libjemalloc.so.3.3.0 /usr/local/mariadb/columnstore/lib/libidbdatafile.so.1.0.0 /usr/local/mariadb/columnstore/lib/hdfs-20.so /usr/local/mariadb/columnstore/lib/hdfs-12.so diff --git a/build/infinidb.spec.in.4.5 b/build/infinidb.spec.in.4.5 index b8145d580..966f2a73a 100755 --- a/build/infinidb.spec.in.4.5 +++ b/build/infinidb.spec.in.4.5 @@ -338,7 +338,6 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/Calpont/lib/libmysqlcl_idb.so.1.0.0 /usr/local/Calpont/lib/libquerystats.so.1.0.0 /usr/local/Calpont/lib/libwriteengineredistribute.so.1.0.0 -/usr/local/Calpont/lib/libjemalloc.so.3.3.0 /usr/local/Calpont/lib/libidbdatafile.so.1.0.0 /usr/local/Calpont/lib/hdfs-20.so /usr/local/Calpont/lib/hdfs-12.so diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 30d3d0ab9..13865c05b 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -60,7 +60,7 @@ SET(CPACK_DEBIAN_LIBS_PACKAGE_PROVIDES "mariadb-columnstore-libs") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_PROVIDES "mariadb-columnstore-platform") SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_PROVIDES "mariadb-columnstore-storage-engine") -SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs") +SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs, libjemalloc1") SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 98974325d..12da14498 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -90,9 +90,9 @@ if (${REDHAT_VERSION_NUMBER} EQUAL 6) # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "jemalloc") else () - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "jemalloc") endif() SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") From f805478eb91173676012c481e8dc81511707963e Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 8 Feb 2019 17:55:50 +0000 Subject: [PATCH 5/7] MCOL-2149 Fix cpimport decimal saturation If we saturate int64_t during string -> decimal conversion then end processing there instead of continuing. This preserves a good saturation value. --- writeengine/shared/we_convertor.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 8ad344ce3..7dcad3986 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -182,19 +182,26 @@ long long Convertor::convertDecimalString( { long double dval = strtold(field, NULL); long long ret = 0; - + // move scale digits to the left of the decimal point for (int i = 0; i < scale; i++) dval *= 10; - + // range check against int64 - if (dval > LLONG_MAX || dval < LLONG_MIN) + if (dval > LLONG_MAX) + { errno = ERANGE; - else - errno = 0; - + return LLONG_MAX; + } + if (dval < LLONG_MIN) + { + errno = ERANGE; + return LLONG_MIN; + } + errno = 0; + ret = dval; - + // get the fractional part of what's left & round ret up or down. dval -= ret; if (dval >= 0.5 && ret < LLONG_MAX) From 9c71f3d48086b5963a020cc9bd9d2316ca22a0cd Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 13 Feb 2019 21:23:08 +0000 Subject: [PATCH 6/7] Bump version to 1.0.17 --- README.md | 8 ++++---- VERSION | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3ad645174..bac34bdd2 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.16 -MariaDB ColumnStore 1.0.16 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.17 +MariaDB ColumnStore 1.0.17 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.16 is an GA release. +#MariaDB ColumnStore 1.0.17 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. diff --git a/VERSION b/VERSION index b13a94d3f..7e6391a76 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=16 +COLUMNSTORE_VERSION_PATCH=17 COLUMNSTORE_VERSION_RELEASE=1 From 620519384593c0b2ada9dcb24ce52b9a6e1463c8 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 13 Feb 2019 21:50:44 +0000 Subject: [PATCH 7/7] Disable jemalloc on SLES --- CMakeLists.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 387d34f3a..2053bbad7 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,11 +112,16 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() -INCLUDE (FindJeMalloc.cmake) -if (NOT JEMALLOC_FOUND) - message(FATAL_ERROR "jemalloc not found!") +# Jemalloc has issues with SLES 12, so disable for now +IF (EXISTS "/etc/SuSE-release") SET(JEMALLOC_LIBRARIES "") -endif() +ELSE () + INCLUDE (FindJeMalloc.cmake) + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "jemalloc not found!") + SET(JEMALLOC_LIBRARIES "") + endif() +ENDIF () FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE)