From 6c79c40dc645d856e0da3c7b0d5d93c762d2c210 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 8 Nov 2019 15:41:33 -0500 Subject: [PATCH 01/35] Fixed a couple paths in the top-level cmakelists file. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce18bec3f..db0919358 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -273,8 +273,8 @@ SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/x SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") SET (ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre") -SET (ENGINE_SERVER_WSREP_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../wsrep-lib/include") -SET (ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../wsrep-lib/wsrep-API/v26/") +SET (ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include") +SET (ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/") SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk") SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) From dba7220ad3c40830c45a739298a120f1c4bfb6c2 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 20 Nov 2019 11:49:46 +0000 Subject: [PATCH 02/35] Fix a few cppcheck issues Found the following: * Potential stack explosions with alloca() usage on potentially large strings * Memory leaks in WriteEngineServer * Stack usage out of scope in dataconvert * A typo in an 'if' statement in dataconvert --- dbcon/execplan/calpontsystemcatalog.h | 4 ++-- dbcon/joblist/pdictionaryscan.cpp | 5 +---- primitives/linux-port/dictionary.cpp | 3 ++- storage-manager/src/SMLogging.cpp | 1 + utils/dataconvert/dataconvert.cpp | 3 +-- utils/dataconvert/dataconvert.h | 8 +++++--- utils/funcexp/func_char_length.cpp | 4 ++-- utils/funcexp/func_concat_ws.cpp | 12 ++++++++---- utils/funcexp/func_lcase.cpp | 9 ++++++--- utils/funcexp/func_left.cpp | 9 ++++++--- utils/funcexp/func_lpad.cpp | 13 ++++++++----- utils/funcexp/func_ltrim.cpp | 13 ++++++++----- utils/funcexp/func_repeat.cpp | 7 ++++--- utils/funcexp/func_right.cpp | 9 ++++++--- utils/funcexp/func_rpad.cpp | 13 ++++++++----- utils/funcexp/func_rtrim.cpp | 13 ++++++++----- utils/funcexp/func_substr.cpp | 9 ++++++--- utils/funcexp/func_trim.cpp | 13 ++++++++----- utils/funcexp/func_ucase.cpp | 9 ++++++--- utils/funcexp/utils_utf8.h | 16 +++++++++++----- writeengine/bulk/we_colbuf.cpp | 4 ++-- writeengine/bulk/we_columninfocompressed.cpp | 1 + 22 files changed, 110 insertions(+), 68 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 50248eb13..81b6c2218 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -430,9 +430,9 @@ public: struct TableAliasName { TableAliasName (): fisColumnStore (true) {} - TableAliasName (std::string sch, std::string tb, std::string al) : + TableAliasName (const std::string &sch, const std::string &tb, const std::string &al) : schema (sch), table (tb), alias (al), fisColumnStore(true) {} - TableAliasName (std::string sch, std::string tb, std::string al, std::string v) : + TableAliasName (const std::string &sch, const std::string &tb, const std::string &al, const std::string &v) : schema (sch), table (tb), alias (al), view(v), fisColumnStore(true) {} std::string schema; std::string table; diff --git a/dbcon/joblist/pdictionaryscan.cpp b/dbcon/joblist/pdictionaryscan.cpp index b8545b728..39bb561d0 100644 --- a/dbcon/joblist/pdictionaryscan.cpp +++ b/dbcon/joblist/pdictionaryscan.cpp @@ -310,11 +310,8 @@ void pDictionaryScan::addFilter(int8_t COP, const string& value) } else { - uint8_t* s = (uint8_t*)alloca(value.size() * sizeof(uint8_t)); - - memcpy(s, value.data(), value.size()); fFilterString << (uint16_t) value.size(); - fFilterString.append(s, value.size()); + fFilterString.append((const uint8_t*)value.data(), value.size()); } } diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index e5a334436..b1405589f 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -682,7 +682,7 @@ inline bool PrimitiveProcessor::isEscapedChar(char c) int PrimitiveProcessor::convertToRegexp(idb_regex_t* regex, const p_DataValue* str) { //In the worst case, every char is quadrupled, plus some leading/trailing cruft... - char* cBuf = (char*)alloca(((4 * str->len) + 3) * sizeof(char)); + char* cBuf = new char[(4 * str->len) + 3]; char c; int i, cBufIdx = 0; // translate to regexp symbols @@ -763,6 +763,7 @@ int PrimitiveProcessor::convertToRegexp(idb_regex_t* regex, const p_DataValue* s regex->regex = cBuf; #endif regex->used = true; + delete [] cBuf; return 0; } diff --git a/storage-manager/src/SMLogging.cpp b/storage-manager/src/SMLogging.cpp index 134d55e86..d029f19d3 100644 --- a/storage-manager/src/SMLogging.cpp +++ b/storage-manager/src/SMLogging.cpp @@ -65,6 +65,7 @@ void SMLogging::log(int priority,const char *format, ...) va_copy(args2, args); vfprintf(stderr, format, args2); fprintf(stderr, "\n"); + va_end(args2); #endif vsyslog(priority, format, args); diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index d9b0543e6..fb347253f 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -2011,7 +2011,7 @@ int64_t DataConvert::convertColumnTimestamp( unsigned int dataOrgLen, const std::string& timeZone ) { - + char tmbuf[64]; std::string dataOrgTemp = dataOrg; if (dataOrgTemp.substr(0, 19) == "0000-00-00 00:00:00") { @@ -2023,7 +2023,6 @@ int64_t DataConvert::convertColumnTimestamp( if (strcmp(dataOrg, "current_timestamp() ON UPDATE current_timestamp()") == 0) { struct timeval tv; - char tmbuf[64]; gettimeofday(&tv, 0); MySQLTime time; gmtSecToMySQLTime(tv.tv_sec, time, timeZone); diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 03d5e0830..efa4f5353 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -268,7 +268,7 @@ bool isDateValid ( int day, int month, int year) { bool valid = true; - if ( year == 0 && month == 0 && year == 0 ) + if ( day == 0 && month == 0 && year == 0 ) { return true; } @@ -1319,7 +1319,7 @@ inline void DataConvert::trimWhitespace(int64_t& charData) inline std::string DataConvert::constructRegexp(const std::string& str) { //In the worst case, every char is quadrupled, plus some leading/trailing cruft... - char* cBuf = (char*)alloca(((4 * str.length()) + 3) * sizeof(char)); + char* cBuf = new char[(4 * str.length()) + 3]; char c; uint32_t i, cBufIdx = 0; // translate to regexp symbols @@ -1392,7 +1392,9 @@ inline std::string DataConvert::constructRegexp(const std::string& str) #ifdef VERBOSE cerr << "regexified string is " << cBuf << endl; #endif - return cBuf; + std::string ret(cBuf); + delete [] cBuf; + return ret; } } // namespace dataconvert diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index bacef974c..c7c382c94 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -83,9 +83,9 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, return 0; size_t strwclen = utf8::idb_mbstowcs(0, tstr.c_str(), 0) + 1; - wchar_t* wcbuf = (wchar_t*)alloca(strwclen * sizeof(wchar_t)); + wchar_t* wcbuf = new wchar_t[strwclen]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen); - + delete [] wcbuf; return (int64_t)strwclen; } diff --git a/utils/funcexp/func_concat_ws.cpp b/utils/funcexp/func_concat_ws.cpp index d5519fd4f..af3da5245 100644 --- a/utils/funcexp/func_concat_ws.cpp +++ b/utils/funcexp/func_concat_ws.cpp @@ -57,7 +57,7 @@ string Func_concat_ws::getStrVal(Row& row, #ifdef STRCOLL_ENH__ wstring wstr; size_t strwclen = utf8::idb_mbstowcs(0, delim.c_str(), 0) + 1; - wchar_t* wcbuf = (wchar_t*)alloca(strwclen * sizeof(wchar_t)); + wchar_t* wcbuf = new wchar_t[strwclen]; strwclen = utf8::idb_mbstowcs(wcbuf, delim.c_str(), strwclen); wstring wdelim(wcbuf, strwclen); @@ -75,14 +75,15 @@ string Func_concat_ws::getStrVal(Row& row, wstr += wdelim; size_t strwclen1 = utf8::idb_mbstowcs(0, tstr.c_str(), 0) + 1; - wchar_t* wcbuf1 = (wchar_t*)alloca(strwclen1 * sizeof(wchar_t)); + wchar_t* wcbuf1 = new wchar_t[strwclen1]; strwclen1 = utf8::idb_mbstowcs(wcbuf1, tstr.c_str(), strwclen1); wstring str1(wcbuf1, strwclen1); wstr += str1; + delete [] wcbuf1; } size_t strmblen = utf8::idb_wcstombs(0, wstr.c_str(), 0) + 1; - char* outbuf = (char*)alloca(strmblen * sizeof(char)); + char* outbuf = new char[strmblen]; strmblen = utf8::idb_wcstombs(outbuf, wstr.c_str(), strmblen); if (strmblen == 0) @@ -90,7 +91,10 @@ string Func_concat_ws::getStrVal(Row& row, else isNull = false; - return string(outbuf, strmblen); + std::string ret(outbuf, strmblen); + delete [] outbuf; + delete [] wcbuf; + return ret; #else string str; diff --git a/utils/funcexp/func_lcase.cpp b/utils/funcexp/func_lcase.cpp index 8f0d164b1..ae399c986 100644 --- a/utils/funcexp/func_lcase.cpp +++ b/utils/funcexp/func_lcase.cpp @@ -68,7 +68,7 @@ std::string Func_lcase::getStrVal(rowgroup::Row& row, return ""; size_t strwclen = utf8::idb_mbstowcs(0, tstr.c_str(), 0) + 1; - wchar_t* wcbuf = (wchar_t*)alloca(strwclen * sizeof(wchar_t)); + wchar_t* wcbuf = new wchar_t[strwclen]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen); wstring wstr(wcbuf, strwclen); @@ -76,9 +76,12 @@ std::string Func_lcase::getStrVal(rowgroup::Row& row, wstr[i] = std::towlower(wstr[i]); size_t strmblen = utf8::idb_wcstombs(0, wstr.c_str(), 0) + 1; - char* outbuf = (char*)alloca(strmblen * sizeof(char)); + char* outbuf = new char[strmblen]; strmblen = utf8::idb_wcstombs(outbuf, wstr.c_str(), strmblen); - return string(outbuf, strmblen); + std::string ret(outbuf, strmblen); + delete [] outbuf; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/func_left.cpp b/utils/funcexp/func_left.cpp index 3f964850d..3fc0ea403 100644 --- a/utils/funcexp/func_left.cpp +++ b/utils/funcexp/func_left.cpp @@ -56,7 +56,7 @@ std::string Func_left::getStrVal(rowgroup::Row& row, return ""; size_t strwclen = utf8::idb_mbstowcs(0, tstr.c_str(), 0) + 1; - wchar_t* wcbuf = (wchar_t*)alloca(strwclen * sizeof(wchar_t)); + wchar_t* wcbuf = new wchar_t[strwclen]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen); wstring str(wcbuf, strwclen); @@ -70,9 +70,12 @@ std::string Func_left::getStrVal(rowgroup::Row& row, wstring out = str.substr(0, pos + 1); size_t strmblen = utf8::idb_wcstombs(0, out.c_str(), 0) + 1; - char* outbuf = (char*)alloca(strmblen * sizeof(char)); + char* outbuf = new char[strmblen]; strmblen = utf8::idb_wcstombs(outbuf, out.c_str(), strmblen); - return string(outbuf, strmblen); + std::string ret(outbuf, strmblen); + delete [] outbuf; + delete [] wcbuf; + return ret; // return str.substr(0, pos+1); } diff --git a/utils/funcexp/func_lpad.cpp b/utils/funcexp/func_lpad.cpp index fbb5ceca2..8a40f21d2 100644 --- a/utils/funcexp/func_lpad.cpp +++ b/utils/funcexp/func_lpad.cpp @@ -155,10 +155,10 @@ std::string Func_lpad::getStrVal(rowgroup::Row& row, if (strwclen > len) alen = strwclen; - size_t bufsize = (alen + 1) * sizeof(wchar_t); + size_t bufsize = alen + 1; // Convert to wide characters. Do all further work in wide characters - wchar_t* wcbuf = (wchar_t*)alloca(bufsize); + wchar_t* wcbuf = new wchar_t[bufsize]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen + 1); size_t strSize = strwclen; // The number of significant characters @@ -190,8 +190,8 @@ std::string Func_lpad::getStrVal(rowgroup::Row& row, // Convert the pad string to wide padwclen = pad->length(); // A guess to start. - size_t padbufsize = (padwclen + 1) * sizeof(wchar_t); - wchar_t* wcpad = (wchar_t*)alloca(padbufsize); + size_t padbufsize = padwclen + 1; + wchar_t* wcpad = new wchar_t[padbufsize]; // padwclen+1 is for giving count for the terminating null size_t padlen = utf8::idb_mbstowcs(wcpad, pad->c_str(), padwclen + 1); @@ -234,7 +234,10 @@ std::string Func_lpad::getStrVal(rowgroup::Row& row, wstring padded = wstring(wcbuf, len); // Turn back to a string - return utf8::wstring_to_utf8(padded.c_str()); + std::string ret(utf8::wstring_to_utf8(padded.c_str())); + delete [] wcpad; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/func_ltrim.cpp b/utils/funcexp/func_ltrim.cpp index 9ffef4bca..77db579b8 100644 --- a/utils/funcexp/func_ltrim.cpp +++ b/utils/funcexp/func_ltrim.cpp @@ -74,10 +74,10 @@ std::string Func_ltrim::getStrVal(rowgroup::Row& row, // determine the size of buffer to allocate, we can be sure the wide // char string won't be longer than: strwclen = tstr.length(); // a guess to start with. This will be >= to the real count. - int bufsize = (strwclen + 1) * sizeof(wchar_t); + int bufsize = strwclen + 1; // Convert the string to wide characters. Do all further work in wide characters - wchar_t* wcbuf = (wchar_t*)alloca(bufsize); + wchar_t* wcbuf = new wchar_t[bufsize]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen + 1); // idb_mbstowcs can return -1 if there is bad mbs char in tstr @@ -86,8 +86,8 @@ std::string Func_ltrim::getStrVal(rowgroup::Row& row, // Convert the trim string to wide trimwclen = trim.length(); // A guess to start. - int trimbufsize = (trimwclen + 1) * sizeof(wchar_t); - wchar_t* wctrim = (wchar_t*)alloca(trimbufsize); + int trimbufsize = trimwclen + 1; + wchar_t* wctrim = new wchar_t[trimbufsize]; size_t trimlen = utf8::idb_mbstowcs(wctrim, trim.c_str(), trimwclen + 1); // idb_mbstowcs can return -1 if there is bad mbs char in tstr @@ -123,7 +123,10 @@ std::string Func_ltrim::getStrVal(rowgroup::Row& row, size_t aLen = strwclen - (aPtr - oPtr); wstring trimmed = wstring(aPtr, aLen); // Turn back to a string - return utf8::wstring_to_utf8(trimmed.c_str()); + std::string ret(utf8::wstring_to_utf8(trimmed.c_str())); + delete [] wctrim; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/func_repeat.cpp b/utils/funcexp/func_repeat.cpp index d79d16de1..65a6b45e2 100644 --- a/utils/funcexp/func_repeat.cpp +++ b/utils/funcexp/func_repeat.cpp @@ -81,8 +81,7 @@ std::string Func_repeat::getStrVal(rowgroup::Row& row, int size = str.length() * count; //allocate memory - char* result = NULL; - result = (char*) alloca(size * sizeof(char) + 1); + char* result = new char[size + 1]; if (result == NULL) { @@ -97,7 +96,9 @@ std::string Func_repeat::getStrVal(rowgroup::Row& row, return ""; } - return result; + std::string res(result); + delete [] result; + return res; } diff --git a/utils/funcexp/func_right.cpp b/utils/funcexp/func_right.cpp index 0d1b222d6..f7a21faed 100644 --- a/utils/funcexp/func_right.cpp +++ b/utils/funcexp/func_right.cpp @@ -65,7 +65,7 @@ std::string Func_right::getStrVal(rowgroup::Row& row, size_t strwclen = utf8::idb_mbstowcs(0, tstr.c_str(), 0) + 1; //wchar_t wcbuf[strwclen]; - wchar_t* wcbuf = (wchar_t*)alloca(strwclen * sizeof(wchar_t)); + wchar_t* wcbuf = new wchar_t[strwclen]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen); wstring str(wcbuf, strwclen); @@ -75,9 +75,12 @@ std::string Func_right::getStrVal(rowgroup::Row& row, wstring out = str.substr(strwclen - pos, strwclen); size_t strmblen = utf8::idb_wcstombs(0, out.c_str(), 0) + 1; //char outbuf[strmblen]; - char* outbuf = (char*)alloca(strmblen * sizeof(char)); + char* outbuf = new char[strmblen]; strmblen = utf8::idb_wcstombs(outbuf, out.c_str(), strmblen); - return string(outbuf, strmblen); + std::string ret(outbuf, strmblen); + delete [] outbuf; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/func_rpad.cpp b/utils/funcexp/func_rpad.cpp index 7a6b94e0c..37e1d8791 100644 --- a/utils/funcexp/func_rpad.cpp +++ b/utils/funcexp/func_rpad.cpp @@ -155,10 +155,10 @@ std::string Func_rpad::getStrVal(rowgroup::Row& row, if (strwclen > len) alen = strwclen; - int bufsize = (alen + 1) * sizeof(wchar_t); + int bufsize = alen + 1; // Convert to wide characters. Do all further work in wide characters - wchar_t* wcbuf = (wchar_t*)alloca(bufsize); + wchar_t* wcbuf = new wchar_t[bufsize]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen + 1); unsigned int strSize = strwclen; // The number of significant characters @@ -190,8 +190,8 @@ std::string Func_rpad::getStrVal(rowgroup::Row& row, // Convert the pad string to wide padwclen = pad->length(); // A guess to start. - int padbufsize = (padwclen + 1) * sizeof(wchar_t); - wchar_t* wcpad = (wchar_t*)alloca(padbufsize); + int padbufsize = padwclen + 1; + wchar_t* wcpad = new wchar_t[padbufsize]; size_t padlen = utf8::idb_mbstowcs(wcpad, pad->c_str(), padwclen + 1); // How many chars do we need? @@ -221,7 +221,10 @@ std::string Func_rpad::getStrVal(rowgroup::Row& row, wstring padded = wstring(wcbuf, len); // Bug 5110 : strings were getting truncated since enough bytes not allocated. - return utf8::wstring_to_utf8(padded.c_str()); + std::string ret(utf8::wstring_to_utf8(padded.c_str())); + delete [] wcpad; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/func_rtrim.cpp b/utils/funcexp/func_rtrim.cpp index c069c5d1f..513567d6d 100644 --- a/utils/funcexp/func_rtrim.cpp +++ b/utils/funcexp/func_rtrim.cpp @@ -73,10 +73,10 @@ std::string Func_rtrim::getStrVal(rowgroup::Row& row, // determine the size of buffer to allocate, we can be sure the wide // char string won't be longer than: strwclen = tstr.length(); // a guess to start with. This will be >= to the real count. - int bufsize = (strwclen + 1) * sizeof(wchar_t); + int bufsize = strwclen + 1; // Convert the string to wide characters. Do all further work in wide characters - wchar_t* wcbuf = (wchar_t*)alloca(bufsize); + wchar_t* wcbuf = new wchar_t[bufsize]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen + 1); // utf8::idb_mbstowcs could return -1 if there is bad chars @@ -85,8 +85,8 @@ std::string Func_rtrim::getStrVal(rowgroup::Row& row, // Convert the trim string to wide trimwclen = trim.length(); // A guess to start. - int trimbufsize = (trimwclen + 1) * sizeof(wchar_t); - wchar_t* wctrim = (wchar_t*)alloca(trimbufsize); + int trimbufsize = trimwclen + 1; + wchar_t* wctrim = new wchar_t[trimbufsize]; size_t trimlen = utf8::idb_mbstowcs(wctrim, trim.c_str(), trimwclen + 1); // idb_mbstowcs could return -1 if there is bad chars @@ -128,7 +128,10 @@ std::string Func_rtrim::getStrVal(rowgroup::Row& row, size_t aLen = strwclen - trimCnt; wstring trimmed = wstring(aPtr, aLen); // Turn back to a string - return utf8::wstring_to_utf8(trimmed.c_str()); + std::string ret(utf8::wstring_to_utf8(trimmed.c_str())); + delete [] wctrim; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/func_substr.cpp b/utils/funcexp/func_substr.cpp index 17ee8724d..e99287af8 100644 --- a/utils/funcexp/func_substr.cpp +++ b/utils/funcexp/func_substr.cpp @@ -59,7 +59,7 @@ std::string Func_substr::getStrVal(rowgroup::Row& row, return ""; size_t strwclen = utf8::idb_mbstowcs(0, tstr.c_str(), 0) + 1; - wchar_t* wcbuf = (wchar_t*)alloca(strwclen * sizeof(wchar_t)); + wchar_t* wcbuf = new wchar_t[strwclen]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen); wstring str(wcbuf, strwclen); @@ -98,9 +98,12 @@ std::string Func_substr::getStrVal(rowgroup::Row& row, wstring out = str.substr(start, n); size_t strmblen = utf8::idb_wcstombs(0, out.c_str(), 0) + 1; - char* outbuf = (char*)alloca(strmblen * sizeof(char)); + char* outbuf = new char[strmblen]; strmblen = utf8::idb_wcstombs(outbuf, out.c_str(), strmblen); - return string(outbuf, strmblen); + std::string ret(outbuf, strmblen); + delete [] outbuf; + delete [] wcbuf; + return ret; #else const string& str = fp[0]->data()->getStrVal(row, isNull); diff --git a/utils/funcexp/func_trim.cpp b/utils/funcexp/func_trim.cpp index db6160245..83cbe6707 100644 --- a/utils/funcexp/func_trim.cpp +++ b/utils/funcexp/func_trim.cpp @@ -72,10 +72,10 @@ std::string Func_trim::getStrVal(rowgroup::Row& row, // determine the size of buffer to allocate, we can be sure the wide // char string won't be longer than: strwclen = tstr.length(); // a guess to start with. This will be >= to the real count. - int bufsize = (strwclen + 1) * sizeof(wchar_t); + int bufsize = strwclen + 1; // Convert the string to wide characters. Do all further work in wide characters - wchar_t* wcbuf = (wchar_t*)alloca(bufsize); + wchar_t* wcbuf = new wchar_t[bufsize]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen + 1); // Bad char in mbc can return -1 @@ -84,8 +84,8 @@ std::string Func_trim::getStrVal(rowgroup::Row& row, // Convert the trim string to wide trimwclen = trim.length(); // A guess to start. - int trimbufsize = (trimwclen + 1) * sizeof(wchar_t); - wchar_t* wctrim = (wchar_t*)alloca(trimbufsize); + int trimbufsize = trimwclen + 1; + wchar_t* wctrim = new wchar_t[trimbufsize]; size_t trimlen = utf8::idb_mbstowcs(wctrim, trim.c_str(), trimwclen + 1); // Bad char in mbc can return -1 @@ -144,7 +144,10 @@ std::string Func_trim::getStrVal(rowgroup::Row& row, size_t aLen = strwclen - trimCnt; wstring trimmed = wstring(aPtr, aLen); // Turn back to a string - return utf8::wstring_to_utf8(trimmed.c_str()); + std::string ret(utf8::wstring_to_utf8(trimmed.c_str())); + delete [] wctrim; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/func_ucase.cpp b/utils/funcexp/func_ucase.cpp index ac38deecc..f032de880 100644 --- a/utils/funcexp/func_ucase.cpp +++ b/utils/funcexp/func_ucase.cpp @@ -67,7 +67,7 @@ std::string Func_ucase::getStrVal(rowgroup::Row& row, return ""; size_t strwclen = utf8::idb_mbstowcs(0, tstr.c_str(), 0) + 1; - wchar_t* wcbuf = (wchar_t*)alloca(strwclen * sizeof(wchar_t)); + wchar_t* wcbuf = new wchar_t[strwclen]; strwclen = utf8::idb_mbstowcs(wcbuf, tstr.c_str(), strwclen); wstring wstr(wcbuf, strwclen); @@ -75,9 +75,12 @@ std::string Func_ucase::getStrVal(rowgroup::Row& row, wstr[i] = std::towupper(wstr[i]); size_t strmblen = utf8::idb_wcstombs(0, wstr.c_str(), 0) + 1; - char* outbuf = (char*)alloca(strmblen * sizeof(char)); + char* outbuf = new char[strmblen]; strmblen = utf8::idb_wcstombs(outbuf, wstr.c_str(), strmblen); - return string(outbuf, strmblen); + std::string ret(outbuf, strmblen); + delete [] outbuf; + delete [] wcbuf; + return ret; } diff --git a/utils/funcexp/utils_utf8.h b/utils/funcexp/utils_utf8.h index 23d41ce7e..abcb74ef7 100644 --- a/utils/funcexp/utils_utf8.h +++ b/utils/funcexp/utils_utf8.h @@ -217,10 +217,10 @@ size_t idb_wcstombs(char* dest, const wchar_t* src, size_t max) inline std::wstring utf8_to_wstring (const std::string& str) { - int bufsize = (int)((str.length() + 1) * sizeof(wchar_t)); + size_t bufsize = str.length() + 1; // Convert to wide characters. Do all further work in wide characters - wchar_t* wcbuf = (wchar_t*)alloca(bufsize); + wchar_t* wcbuf = new wchar_t[bufsize]; // Passing +1 so that windows is happy to see extra position to place NULL size_t strwclen = idb_mbstowcs(wcbuf, str.c_str(), str.length() + 1); @@ -229,7 +229,10 @@ std::wstring utf8_to_wstring (const std::string& str) if ( strwclen == static_cast(-1) ) strwclen = 0; - return std::wstring(wcbuf, strwclen); + std::wstring ret(wcbuf, strwclen); + + delete [] wcbuf; + return ret; } @@ -237,7 +240,7 @@ std::wstring utf8_to_wstring (const std::string& str) inline std::string wstring_to_utf8 (const std::wstring& str) { - char* outbuf = (char*)alloca((str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1); + char* outbuf = new char[(str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1]; // Passing +1 so that windows is happy to see extra position to place NULL size_t strmblen = idb_wcstombs(outbuf, str.c_str(), str.length() * MAX_UTF8_BYTES_PER_CHAR + 1); @@ -246,7 +249,10 @@ std::string wstring_to_utf8 (const std::wstring& str) if ( strmblen == static_cast(-1) ) strmblen = 0; - return std::string(outbuf, strmblen); + std::string ret(outbuf, strmblen); + + delete [] outbuf; + return ret; } inline diff --git a/writeengine/bulk/we_colbuf.cpp b/writeengine/bulk/we_colbuf.cpp index da25704af..7be21a6ec 100644 --- a/writeengine/bulk/we_colbuf.cpp +++ b/writeengine/bulk/we_colbuf.cpp @@ -133,7 +133,7 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmptie if (nitems != 1) { - delete newBuf; + delete [] newBuf; return ERR_FILE_WRITE; } @@ -141,7 +141,7 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmptie Stats::stopParseEvent(WE_STATS_WRITE_COL); #endif - delete newBuf; + delete [] newBuf; return NO_ERROR; } diff --git a/writeengine/bulk/we_columninfocompressed.cpp b/writeengine/bulk/we_columninfocompressed.cpp index 9ec34f7cb..25af8bc22 100644 --- a/writeengine/bulk/we_columninfocompressed.cpp +++ b/writeengine/bulk/we_columninfocompressed.cpp @@ -411,6 +411,7 @@ int ColumnInfoCompressed::truncateDctnryStore( fLog->logMsg( oss.str(), rc, MSGLVL_ERROR ); fTruncateDctnryFileOp.closeFile( dFile ); + delete [] pointerHdr; return rc; } From 0d26dc447cc23c186a14b424615737d9b815130c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 21 Nov 2019 14:41:00 -0500 Subject: [PATCH 03/35] Squash merge of the threaded UM hash table construction feature. Conflicts: oam/etc/Columnstore.xml.singleserver --- dbcon/joblist/tuplehashjoin.cpp | 364 +++++++++------- dbcon/joblist/tuplehashjoin.h | 24 +- exemgr/main.cpp | 5 +- oam/etc/Columnstore.xml | 2 +- oam/etc/Columnstore.xml.singleserver | 3 +- utils/common/fixedallocator.cpp | 47 +- utils/common/fixedallocator.h | 52 ++- utils/common/spinlock.h | 27 ++ utils/joiner/tuplejoiner.cpp | 627 ++++++++++++++++++++------- utils/joiner/tuplejoiner.h | 57 ++- utils/threadpool/threadpool.cpp | 4 + 11 files changed, 837 insertions(+), 375 deletions(-) create mode 100644 utils/common/spinlock.h diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index c3a667b36..2ad3c495b 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -54,6 +54,7 @@ using namespace funcexp; using namespace querytele; #include "atomicops.h" +#include "spinlock.h" namespace joblist { @@ -73,8 +74,6 @@ TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) : fTupleId2(-1), fCorrelatedSide(0), resourceManager(jobInfo.rm), - totalUMMemoryUsage(0), - rgDataSize(0), runRan(false), joinRan(false), largeSideIndex(1), @@ -84,7 +83,8 @@ TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) : fTokenJoin(-1), fStatsMutexPtr(new boost::mutex()), fFunctionJoinKeys(jobInfo.keyInfo->functionJoinKeys), - sessionMemLimit(jobInfo.umMemLimit) + sessionMemLimit(jobInfo.umMemLimit), + rgdLock(false) { /* Need to figure out how much memory these use... Overhead storing 16 byte elements is about 32 bytes. That @@ -116,11 +116,13 @@ TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) : else allowDJS = false; + numCores = sysconf(_SC_NPROCESSORS_ONLN); + if (numCores <= 0) + numCores = 8; /* Debugging, rand() is used to simulate failures time_t t = time(NULL); srand(t); */ - } TupleHashJoinStep::~TupleHashJoinStep() @@ -130,8 +132,8 @@ TupleHashJoinStep::~TupleHashJoinStep() if (ownsOutputDL) delete outputDL; - if (totalUMMemoryUsage != 0) - resourceManager->returnMemory(totalUMMemoryUsage, sessionMemLimit); + for (uint i = 0 ; i < smallDLs.size(); i++) + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); //cout << "deallocated THJS, UM memory available: " << resourceManager.availableMemory() << endl; } @@ -200,140 +202,237 @@ void TupleHashJoinStep::join() } } -/* Index is which small input to read. */ -void TupleHashJoinStep::smallRunnerFcn(uint32_t index) +// simple sol'n. Poll mem usage of Joiner once per second. Request mem +// increase after the fact. Failure to get mem will be detected and handled by +// the threads inserting into Joiner. +void TupleHashJoinStep::trackMem(uint index) { - uint64_t i; - bool more, flippedUMSwitch = false, gotMem; - RGData oneRG; - //shared_array oneRG; - Row r; + boost::shared_ptr joiner = joiners[index]; + ssize_t memBefore = 0, memAfter = 0; + bool gotMem; + + boost::unique_lock scoped(memTrackMutex); + while (!stopMemTracking) + { + memAfter = joiner->getMemUsage(); + if (memAfter != memBefore) + { + gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, false); + atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); + memBefore = memAfter; + if (!gotMem) + return; + } + memTrackDone.timed_wait(scoped, boost::posix_time::seconds(1)); + } + + // one more iteration to capture mem usage since last poll, for this one + // raise an error if mem went over the limit + memAfter = joiner->getMemUsage(); + if (memAfter == memBefore) + return; + gotMem = resourceManager->getMemory(memAfter - memBefore, sessionMemLimit, false); + atomicops::atomicAdd(&memUsedByEachJoin[index], memAfter - memBefore); + if (!gotMem) + { + if (!joinIsTooBig && (isDML || !allowDJS || (fSessionId & 0x80000000) || + (tableOid() < 3000 && tableOid() >= 1000))) + { + joinIsTooBig = true; + fLogger->logMessage(logging::LOG_TYPE_INFO, logging::ERR_JOIN_TOO_BIG); + errorMessage(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG)); + status(logging::ERR_JOIN_TOO_BIG); + cout << "Join is too big, raise the UM join limit for now (monitor thread)" << endl; + abort(); + } + } +} + +void TupleHashJoinStep::startSmallRunners(uint index) +{ + utils::setThreadName("HJSStartSmall"); + string extendedInfo; JoinType jt; - RowGroupDL* smallDL; - uint32_t smallIt; - RowGroup smallRG; boost::shared_ptr joiner; - string extendedInfo; - extendedInfo += toString(); // is each small side supposed to have the whole THJS info? - - smallDL = smallDLs[index]; - smallIt = smallIts[index]; - smallRG = smallRGs[index]; jt = joinTypes[index]; + extendedInfo += toString(); - //cout << " smallRunner " << index << " sees jointype " << jt << " joinTypes has " << joinTypes.size() - // << " elements" << endl; if (typelessJoin[index]) { - joiner.reset(new TupleJoiner(smallRG, largeRG, smallSideKeys[index], - largeSideKeys[index], jt)); + joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index], + largeSideKeys[index], jt, &jobstepThreadPool)); } else { - joiner.reset(new TupleJoiner(smallRG, largeRG, smallSideKeys[index][0], - largeSideKeys[index][0], jt)); + joiner.reset(new TupleJoiner(smallRGs[index], largeRG, smallSideKeys[index][0], + largeSideKeys[index][0], jt, &jobstepThreadPool)); } joiner->setUniqueLimit(uniqueLimit); joiner->setTableName(smallTableNames[index]); joiners[index] = joiner; + /* check for join types unsupported on the PM. */ + if (!largeBPS || !isExeMgr) + joiner->setInUM(rgData[index]); + /* - read the small side into a TupleJoiner - send the TupleJoiner to the large side TBPS - start the large TBPS - read the large side, write to the output + start the small runners + join them + check status + handle abort, out of memory, etc */ - smallRG.initRow(&r); -// cout << "reading smallDL" << endl; - more = smallDL->next(smallIt, &oneRG); + boost::posix_time::ptime end_time, start_time = + boost::posix_time::microsec_clock::universal_time(); + + stopMemTracking = false; + uint64_t jobs[numCores]; + uint64_t memMonitor = jobstepThreadPool.invoke([this, index] { this->trackMem(index); }); + // starting 1 thread when in PM mode, since it's only inserting into a + // vector of rows. The rest will be started when converted to UM mode. + if (joiner->inUM()) + for (int i = 0; i < numCores; i++) + jobs[i] = jobstepThreadPool.invoke([this, i, index, &jobs] { this->smallRunnerFcn(index, i, jobs); }); + else + jobs[0] = jobstepThreadPool.invoke([this, index, &jobs] { this->smallRunnerFcn(index, 0, jobs); }); + + // wait for the first thread to join, then decide whether the others exist and need joining + jobstepThreadPool.join(jobs[0]); + if (joiner->inUM()) + for (int i = 1; i < numCores; i++) + jobstepThreadPool.join(jobs[i]); + + // stop the monitor thread + memTrackMutex.lock(); + stopMemTracking = true; + memTrackDone.notify_one(); + memTrackMutex.unlock(); + jobstepThreadPool.join(memMonitor); + + /* If there was an error or an abort, drain the input DL, + do endOfInput on the output */ + if (cancelled()) + { +// cout << "HJ stopping... status is " << status() << endl; + if (largeBPS) + largeBPS->abort(); + + bool more = true; + RGData oneRG; + while (more) + more = smallDLs[index]->next(smallIts[index], &oneRG); + } + //joiner->doneInserting(); + + end_time = boost::posix_time::microsec_clock::universal_time(); + if (!(fSessionId & 0x80000000)) + cout << "hash table construction time = " << end_time - start_time << + " size = " << joiner->size() << endl; + + extendedInfo += "\n"; + ostringstream oss; + if (joiner->inPM()) + { + oss << "PM join (" << index << ")" << endl; + #ifdef JLF_DEBUG + cout << oss.str(); + #endif + extendedInfo += oss.str(); + } + else if (joiner->inUM() && !joiner->onDisk()) + { + oss << "UM join (" << index << ")" << endl; + #ifdef JLF_DEBUG + cout << oss.str(); + #endif + extendedInfo += oss.str(); + } + + /* Trying to get the extended info to match the original version + It's kind of kludgey at the moment, need to clean it up at some point */ + if (!joiner->onDisk()) + { + joiner->doneInserting(); + boost::mutex::scoped_lock lk(*fStatsMutexPtr); + fExtendedInfo += extendedInfo; + formatMiniStats(index); + } +} + +/* Index is which small input to read. */ +void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t *jobs) +{ + utils::setThreadName("HJSmallRunner"); + bool more = true; + RGData oneRG; + Row r; + RowGroupDL* smallDL; + uint32_t smallIt; + RowGroup smallRG; + boost::shared_ptr joiner = joiners[index]; + + smallDL = smallDLs[index]; + smallIt = smallIts[index]; + smallRG = smallRGs[index]; + + smallRG.initRow(&r); try { - /* check for join types unsupported on the PM. */ - if (!largeBPS || !isExeMgr) - { - flippedUMSwitch = true; - oss << "UM join (" << index << ")"; -#ifdef JLF_DEBUG - cout << oss.str() << endl; -#endif - extendedInfo += oss.str(); - joiner->setInUM(); - } - - resourceManager->getMemory(joiner->getMemUsage(), sessionMemLimit, false); - (void)atomicops::atomicAdd(&totalUMMemoryUsage, joiner->getMemUsage()); - memUsedByEachJoin[index] += joiner->getMemUsage(); - + ssize_t rgSize; + bool gotMem; + goto next; while (more && !cancelled()) { - uint64_t memUseBefore, memUseAfter; - smallRG.setData(&oneRG); - if (smallRG.getRowCount() == 0) goto next; - smallRG.getRow(0, &r); - - memUseBefore = joiner->getMemUsage() + rgDataSize; - // TupleHJ owns the row memory + utils::getSpinlock(rgdLock); rgData[index].push_back(oneRG); - rgDataSize += smallRG.getSizeWithStrings(); + utils::releaseSpinlock(rgdLock); - for (i = 0; i < smallRG.getRowCount(); i++, r.nextRow()) + rgSize = smallRG.getSizeWithStrings(); + atomicops::atomicAdd(&memUsedByEachJoin[index], rgSize); + gotMem = resourceManager->getMemory(rgSize, sessionMemLimit, false); + if (!gotMem) { - //cout << "inserting " << r.toString() << endl; - joiner->insert(r); - } - - memUseAfter = joiner->getMemUsage() + rgDataSize; - - if (UNLIKELY(!flippedUMSwitch && (memUseAfter >= pmMemLimit))) - { - flippedUMSwitch = true; - oss << "UM join (" << index << ") "; -#ifdef JLF_DEBUG - cout << oss.str() << endl; -#endif - extendedInfo += oss.str(); - joiner->setInUM(); - memUseAfter = joiner->getMemUsage() + rgDataSize; - } - - gotMem = resourceManager->getMemory(memUseAfter - memUseBefore, sessionMemLimit, false); - atomicops::atomicAdd(&totalUMMemoryUsage, memUseAfter - memUseBefore); - memUsedByEachJoin[index] += memUseAfter - memUseBefore; - - /* This is kind of kludgy and overlaps with segreateJoiners() atm. - If this join won't be converted to disk-based, this fcn needs to abort the same as - it did before. If it will be converted, it should just return. */ - if (UNLIKELY(!gotMem)) - { - if (isDML || !allowDJS || (fSessionId & 0x80000000) || - (tableOid() < 3000 && tableOid() >= 1000)) + boost::unique_lock sl(saneErrMsg); + if (!joinIsTooBig && (isDML || !allowDJS || (fSessionId & 0x80000000) || + (tableOid() < 3000 && tableOid() >= 1000))) { joinIsTooBig = true; fLogger->logMessage(logging::LOG_TYPE_INFO, logging::ERR_JOIN_TOO_BIG); errorMessage(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_JOIN_TOO_BIG)); status(logging::ERR_JOIN_TOO_BIG); - cout << "Join is too big, raise the UM join limit for now" << endl; + cout << "Join is too big, raise the UM join limit for now (small runner)" << endl; abort(); break; } else + { + joiner->setConvertToDiskJoin(); return; + } } + joiner->insertRGData(smallRG, threadID); + + if (!joiner->inUM() && (memUsedByEachJoin[index] > pmMemLimit)) + { + joiner->setInUM(rgData[index]); + for (int i = 1; i < numCores; i++) + jobs[i] = jobstepThreadPool.invoke([this, i, index, jobs] + { this->smallRunnerFcn(index, i, jobs); }); + } next: -// cout << "inserted one rg into the joiner, rowcount = " << -// smallRG.getRowCount() << endl; + dlMutex.lock(); more = smallDL->next(smallIt, &oneRG); + dlMutex.unlock(); } } catch (boost::exception& e) @@ -356,34 +455,8 @@ next: status(logging::ERR_EXEMGR_MALFUNCTION); } - if (!flippedUMSwitch && !cancelled()) - { - oss << "PM join (" << index << ")"; -#ifdef JLF_DEBUG - cout << oss.str() << endl; -#endif - extendedInfo += oss.str(); + if (!joiner->inUM()) joiner->setInPM(); - } - - /* If there was an error or an abort drain the input DL, - do endOfInput on the output */ - if (cancelled()) - { -// cout << "HJ stopping... status is " << status() << endl; - if (largeBPS) - largeBPS->abort(); - - while (more) - more = smallDL->next(smallIt, &oneRG); - } - - joiner->doneInserting(); - extendedInfo += "\n"; - - boost::mutex::scoped_lock lk(*fStatsMutexPtr); - fExtendedInfo += extendedInfo; - formatMiniStats(index); } void TupleHashJoinStep::forwardCPData() @@ -517,24 +590,6 @@ void TupleHashJoinStep::djsReaderFcn(int index) processFE2(l_outputRG, l_fe2RG, fe2InRow, fe2OutRow, &v_rgData, &l_fe); processDupList(0, (fe2 ? l_fe2RG : l_outputRG), &v_rgData); -#if 0 - - if (!v_rgData.empty() && fSessionId < 0x80000000) - { - if (fe2) - { - l_fe2RG.setData(&v_rgData[0]); - cout << "fully processed rowgroup: " << l_fe2RG.toString() << endl; - } - else - { - l_outputRG.setData(&v_rgData[0]); - cout << "fully processed rowgroup: " << l_outputRG.toString() << endl; - } - } - - cout << "ownsOutputDL = " << (int) ownsOutputDL << " fDelivery = " << (int) fDelivery << endl; -#endif sendResult(v_rgData); } @@ -553,6 +608,7 @@ void TupleHashJoinStep::djsReaderFcn(int index) void TupleHashJoinStep::hjRunner() { uint32_t i; + std::vector smallRunners; // thread handles from thread pool if (cancelled()) { @@ -581,7 +637,7 @@ void TupleHashJoinStep::hjRunner() /* Start the small-side runners */ rgData.reset(new vector[smallDLs.size()]); - memUsedByEachJoin.reset(new uint64_t[smallDLs.size()]); + memUsedByEachJoin.reset(new ssize_t[smallDLs.size()]); for (i = 0; i < smallDLs.size(); i++) memUsedByEachJoin[i] = 0; @@ -680,7 +736,7 @@ void TupleHashJoinStep::hjRunner() { vector empty; resourceManager->returnMemory(memUsedByEachJoin[djsJoinerMap[i]], sessionMemLimit); - atomicops::atomicSub(&totalUMMemoryUsage, memUsedByEachJoin[djsJoinerMap[i]]); + memUsedByEachJoin[djsJoinerMap[i]] = 0; djs[i].loadExistingData(rgData[djsJoinerMap[i]]); rgData[djsJoinerMap[i]].swap(empty); } @@ -792,8 +848,11 @@ void TupleHashJoinStep::hjRunner() joiners.clear(); tbpsJoiners.clear(); rgData.reset(); - resourceManager->returnMemory(totalUMMemoryUsage, sessionMemLimit); - totalUMMemoryUsage = 0; + for (uint i = 0; i < smallDLs.size(); i++) + { + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + memUsedByEachJoin[i] = 0; + } } } } @@ -840,7 +899,7 @@ void TupleHashJoinStep::hjRunner() #ifdef JLF_DEBUG cout << "moving join " << i << " to UM (PM join can't follow a UM join)\n"; #endif - tbpsJoiners[i]->setInUM(); + tbpsJoiners[i]->setInUM(rgData[i]); } } @@ -880,12 +939,10 @@ void TupleHashJoinStep::hjRunner() } #ifdef JLF_DEBUG - if (runFE2onPM) cout << "PM runs FE2\n"; else cout << "UM runs FE2\n"; - #endif largeBPS->setFcnExpGroup2(fe2, fe2Output, runFE2onPM); } @@ -971,8 +1028,11 @@ uint32_t TupleHashJoinStep::nextBand(messageqcpp::ByteStream& bs) joiners.clear(); rgData.reset(); - resourceManager->returnMemory(totalUMMemoryUsage, sessionMemLimit); - totalUMMemoryUsage = 0; + for (uint i = 0; i < smallDLs.size(); i++) + { + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + memUsedByEachJoin[i] = 0; + } return 0; } @@ -992,8 +1052,11 @@ uint32_t TupleHashJoinStep::nextBand(messageqcpp::ByteStream& bs) cout << " -- returning error status " << deliveredRG->getStatus() << endl; deliveredRG->serializeRGData(bs); - resourceManager->returnMemory(totalUMMemoryUsage, sessionMemLimit); - totalUMMemoryUsage = 0; + for (uint i = 0; i < smallDLs.size(); i++) + { + resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + memUsedByEachJoin[i] = 0; + } return 0; } @@ -1850,7 +1913,8 @@ void TupleHashJoinStep::segregateJoiners() return; } - /* Debugging code, this makes all eligible joins disk-based. + #if 0 + // Debugging code, this makes all eligible joins disk-based. else { cout << "making all joins disk-based" << endl; for (i = 0; i < smallSideCount; i++) { @@ -1860,7 +1924,7 @@ void TupleHashJoinStep::segregateJoiners() } return; } - */ + #endif /* For now if there is no largeBPS all joins need to either be DJS or not, not mixed */ if (!largeBPS) diff --git a/dbcon/joblist/tuplehashjoin.h b/dbcon/joblist/tuplehashjoin.h index 27154946f..164f1f541 100644 --- a/dbcon/joblist/tuplehashjoin.h +++ b/dbcon/joblist/tuplehashjoin.h @@ -75,6 +75,8 @@ public: void tableOid1(execplan::CalpontSystemCatalog::OID tableOid1) { fTableOID1 = tableOid1; + if (fTableOID1 < 3000) + numCores = 1; // syscat query, no need for more than 1 thread } void tableOid2(execplan::CalpontSystemCatalog::OID tableOid2) { @@ -425,7 +427,6 @@ private: std::vector > smallSideKeys; ResourceManager* resourceManager; - volatile uint64_t totalUMMemoryUsage; struct JoinerSorter { @@ -436,16 +437,14 @@ private: } }; std::vector > joiners; - boost::scoped_array > rgData; TupleBPS* largeBPS; rowgroup::RowGroup largeRG, outputRG; std::vector smallRGs; - uint64_t pmMemLimit; - uint64_t rgDataSize; + ssize_t pmMemLimit; void hjRunner(); - void smallRunnerFcn(uint32_t index); + void smallRunnerFcn(uint32_t index, uint threadID, uint64_t *threads); struct HJRunner { @@ -462,15 +461,13 @@ private: SmallRunner(TupleHashJoinStep* hj, uint32_t i) : HJ(hj), index(i) { } void operator()() { - utils::setThreadName("HJSSmallSide"); - HJ->smallRunnerFcn(index); + HJ->startSmallRunners(index); } TupleHashJoinStep* HJ; uint32_t index; }; int64_t mainRunner; // thread handle from thread pool - std::vector smallRunners; // thread handles from thread pool // for notify TupleAggregateStep PM hashjoin // Ideally, hashjoin and delivery communicate with RowGroupDL, @@ -614,10 +611,19 @@ private: std::vector > tbpsJoiners; std::vector > djsJoiners; std::vector djsJoinerMap; - boost::scoped_array memUsedByEachJoin; + boost::scoped_array memUsedByEachJoin; boost::mutex djsLock; boost::shared_ptr sessionMemLimit; + /* Threaded UM join support */ + int numCores; + boost::mutex dlMutex, memTrackMutex, saneErrMsg; + boost::condition memTrackDone; + std::atomic rgdLock; + bool stopMemTracking; + void trackMem(uint index); + void startSmallRunners(uint index); + friend class DiskJoinStep; }; diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 4b43f71bc..288d2bada 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1305,7 +1305,7 @@ int8_t setupCwd(joblist::ResourceManager* rm) if (rc < 0 || access(".", W_OK) != 0) rc = chdir("/tmp"); - + return (rc < 0) ? -5 : rc; } @@ -1359,7 +1359,7 @@ void cleanTempDir() return; if (tmpPrefix.empty()) - tmpPrefix = "/tmp/infinidb"; + tmpPrefix = "/tmp/cs-diskjoin"; tmpPrefix += "/"; @@ -1636,4 +1636,3 @@ int main(int argc, char* argv[]) return 0; } // vim:ts=4 sw=4: - diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 78ecc633e..ea4cf6857 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -495,7 +495,7 @@ + /tmp/cs-diskjoin --> Y diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver index de91668f0..4c3286c40 100644 --- a/oam/etc/Columnstore.xml.singleserver +++ b/oam/etc/Columnstore.xml.singleserver @@ -241,6 +241,7 @@ 128M 10 /tmp/columnstore_tmp_files + $INSTALLDIR 10 3 10 @@ -486,7 +487,7 @@ - /var/lib/columnstore/tmp + /var/lib/columnstore/tmp/cs-diskjoin Y diff --git a/utils/common/fixedallocator.cpp b/utils/common/fixedallocator.cpp index d2f705f25..5bb8bc7c4 100644 --- a/utils/common/fixedallocator.cpp +++ b/utils/common/fixedallocator.cpp @@ -50,6 +50,9 @@ FixedAllocator::FixedAllocator(const FixedAllocator& f) tmpSpace = f.tmpSpace; capacityRemaining = 0; currentlyStored = 0; + useLock = f.useLock; + lock = false; + } FixedAllocator& FixedAllocator::operator=(const FixedAllocator& f) @@ -57,10 +60,22 @@ FixedAllocator& FixedAllocator::operator=(const FixedAllocator& f) elementCount = f.elementCount; elementSize = f.elementSize; tmpSpace = f.tmpSpace; + useLock = f.useLock; + lock = false; deallocateAll(); return *this; } +void FixedAllocator::setUseLock(bool useIt) +{ + useLock = useIt; +} + +void FixedAllocator::setAllocSize(uint allocSize) +{ + elementSize = allocSize; +} + void FixedAllocator::newBlock() { shared_array next; @@ -80,39 +95,15 @@ void FixedAllocator::newBlock() } } -void* FixedAllocator::allocate() -{ - void* ret; - - if (capacityRemaining < elementSize) - newBlock(); - - ret = nextAlloc; - nextAlloc += elementSize; - capacityRemaining -= elementSize; - currentlyStored += elementSize; - return ret; -} - -void* FixedAllocator::allocate(uint32_t len) -{ - void* ret; - - if (capacityRemaining < len) - newBlock(); - - ret = nextAlloc; - nextAlloc += len; - capacityRemaining -= len; - currentlyStored += len; - return ret; -} - void FixedAllocator::truncateBy(uint32_t amt) { + if (useLock) + getSpinlock(lock); nextAlloc -= amt; capacityRemaining += amt; currentlyStored -= amt; + if (useLock) + releaseSpinlock(lock); } void FixedAllocator::deallocateAll() diff --git a/utils/common/fixedallocator.h b/utils/common/fixedallocator.h index 31e7b4d75..9e716a133 100644 --- a/utils/common/fixedallocator.h +++ b/utils/common/fixedallocator.h @@ -38,6 +38,8 @@ #include #include #include +#include +#include "spinlock.h" #if defined(_MSC_VER) && defined(xxxFIXEDALLOCATOR_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -55,11 +57,13 @@ public: EXPORT FixedAllocator() : capacityRemaining(0), - elementCount(std::numeric_limits::max()), + elementCount(DEFAULT_NUM_ELEMENTS), elementSize(0), currentlyStored(0), tmpSpace(false), - nextAlloc(0) {} + nextAlloc(0), + useLock(false), + lock(false) {} EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false, unsigned long numElements = DEFAULT_NUM_ELEMENTS) : capacityRemaining(0), @@ -67,7 +71,9 @@ public: elementSize(allocSize), currentlyStored(0), tmpSpace(isTmpSpace), - nextAlloc(0) {} + nextAlloc(0), + useLock(false), + lock(false) {} EXPORT FixedAllocator(const FixedAllocator&); EXPORT FixedAllocator& operator=(const FixedAllocator&); virtual ~FixedAllocator() {} @@ -78,6 +84,8 @@ public: void deallocate() { } // does nothing EXPORT void deallocateAll(); // drops all memory in use EXPORT uint64_t getMemUsage() const; + void setUseLock(bool); + void setAllocSize(uint); private: void newBlock(); @@ -89,10 +97,46 @@ private: uint64_t currentlyStored; bool tmpSpace; uint8_t* nextAlloc; + bool useLock; + std::atomic lock; }; +inline void* FixedAllocator::allocate() +{ + void* ret; + + if (useLock) + getSpinlock(lock); + if (capacityRemaining < elementSize) + newBlock(); + + ret = nextAlloc; + nextAlloc += elementSize; + capacityRemaining -= elementSize; + currentlyStored += elementSize; + if (useLock) + releaseSpinlock(lock); + return ret; +} + +inline void* FixedAllocator::allocate(uint32_t len) +{ + void* ret; + + if (useLock) + getSpinlock(lock); + if (capacityRemaining < len) + newBlock(); + + ret = nextAlloc; + nextAlloc += len; + capacityRemaining -= len; + currentlyStored += len; + if (useLock) + releaseSpinlock(lock); + return ret; } #undef EXPORT - +} // namespace #endif diff --git a/utils/common/spinlock.h b/utils/common/spinlock.h new file mode 100644 index 000000000..1dc84391f --- /dev/null +++ b/utils/common/spinlock.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace utils +{ + +inline void getSpinlock(std::atomic &lock) +{ + bool _false = false; + while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire)) + _false = false; +} + +inline bool trySpinlock(std::atomic &lock) +{ + bool _false = false; + bool ret = lock.compare_exchange_weak(_false, true, std::memory_order_acquire); + return ret; +} + +inline void releaseSpinlock(std::atomic &lock) +{ + lock.store(false, std::memory_order_release); +} + +} diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 88bab4bb0..34d3702e6 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -27,6 +27,7 @@ #endif #include "hasher.h" #include "lbidlist.h" +#include "spinlock.h" using namespace std; using namespace rowgroup; @@ -42,30 +43,49 @@ TupleJoiner::TupleJoiner( const rowgroup::RowGroup& largeInput, uint32_t smallJoinColumn, uint32_t largeJoinColumn, - JoinType jt) : + JoinType jt, + threadpool::ThreadPool *jsThreadPool) : smallRG(smallInput), largeRG(largeInput), joinAlg(INSERTING), joinType(jt), - threadCount(1), typelessJoin(false), bSignedUnsignedJoin(false), uniqueLimit(100), finished(false) + threadCount(1), typelessJoin(false), bSignedUnsignedJoin(false), uniqueLimit(100), finished(false), + jobstepThreadPool(jsThreadPool), _convertToDiskJoin(false) { + uint i; + + getBucketCount(); + m_bucketLocks.reset(new boost::mutex[bucketCount]); + if (smallRG.getColTypes()[smallJoinColumn] == CalpontSystemCatalog::LONGDOUBLE) { - STLPoolAllocator > alloc; - _pool = alloc.getPoolAllocator(); - - ld.reset(new ldhash_t(10, hasher(), ldhash_t::key_equal(), alloc)); + ld.reset(new boost::scoped_ptr[bucketCount]); + _pool.reset(new boost::shared_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) + { + STLPoolAllocator > alloc; + _pool[i] = alloc.getPoolAllocator(); + ld[i].reset(new ldhash_t(10, hasher(), ldhash_t::key_equal(), alloc)); + } } else if (smallRG.usesStringTable()) { - STLPoolAllocator > alloc; - _pool = alloc.getPoolAllocator(); - - sth.reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); + sth.reset(new boost::scoped_ptr[bucketCount]); + _pool.reset(new boost::shared_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) + { + STLPoolAllocator > alloc; + _pool[i] = alloc.getPoolAllocator(); + sth[i].reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); + } } else { - STLPoolAllocator > alloc; - _pool = alloc.getPoolAllocator(); - - h.reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); + h.reset(new boost::scoped_ptr[bucketCount]); + _pool.reset(new boost::shared_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) + { + STLPoolAllocator > alloc; + _pool[i] = alloc.getPoolAllocator(); + h[i].reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); + } } smallRG.initRow(&smallNullRow); @@ -106,16 +126,28 @@ TupleJoiner::TupleJoiner( const rowgroup::RowGroup& largeInput, const vector& smallJoinColumns, const vector& largeJoinColumns, - JoinType jt) : + JoinType jt, + threadpool::ThreadPool *jsThreadPool) : smallRG(smallInput), largeRG(largeInput), joinAlg(INSERTING), joinType(jt), threadCount(1), typelessJoin(true), smallKeyColumns(smallJoinColumns), largeKeyColumns(largeJoinColumns), - bSignedUnsignedJoin(false), uniqueLimit(100), finished(false) + bSignedUnsignedJoin(false), uniqueLimit(100), finished(false), + jobstepThreadPool(jsThreadPool), _convertToDiskJoin(false) { - STLPoolAllocator > alloc; - _pool = alloc.getPoolAllocator(); + uint i; + + getBucketCount(); + + _pool.reset(new boost::shared_ptr[bucketCount]); + ht.reset(new boost::scoped_ptr[bucketCount]); + for (i = 0; i < bucketCount; i++) + { + STLPoolAllocator > alloc; + _pool[i] = alloc.getPoolAllocator(); + ht[i].reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); + } + m_bucketLocks.reset(new boost::mutex[bucketCount]); - ht.reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); smallRG.initRow(&smallNullRow); if (smallOuterJoin() || largeOuterJoin() || semiJoin() || antiJoin()) @@ -126,7 +158,7 @@ TupleJoiner::TupleJoiner( smallNullRow.initToNull(); } - for (uint32_t i = keyLength = 0; i < smallKeyColumns.size(); i++) + for (i = keyLength = 0; i < smallKeyColumns.size(); i++) { if (smallRG.getColTypes()[smallKeyColumns[i]] == CalpontSystemCatalog::CHAR || smallRG.getColTypes()[smallKeyColumns[i]] == CalpontSystemCatalog::VARCHAR @@ -155,12 +187,16 @@ TupleJoiner::TupleJoiner( } } - storedKeyAlloc = FixedAllocator(keyLength); + // note, 'numcores' is implied by tuplehashjoin on calls to insertRGData(). + // TODO: make it explicit to avoid future confusion. + storedKeyAlloc.reset(new FixedAllocator[numCores]); + for (i = 0; i < (uint) numCores; i++) + storedKeyAlloc[i].setAllocSize(keyLength); discreteValues.reset(new bool[smallKeyColumns.size()]); cpValues.reset(new vector[smallKeyColumns.size()]); - for (uint32_t i = 0; i < smallKeyColumns.size(); i++) + for (i = 0; i < smallKeyColumns.size(); i++) { discreteValues[i] = false; if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) @@ -199,6 +235,166 @@ bool TupleJoiner::operator<(const TupleJoiner& tj) const return size() < tj.size(); } +void TupleJoiner::getBucketCount() +{ + // get the # of cores, round up to nearest power of 2 + // make the bucket mask + numCores = sysconf(_SC_NPROCESSORS_ONLN); + if (numCores <= 0) + numCores = 8; + bucketCount = (numCores == 1 ? 1 : (1 << (32 - __builtin_clz(numCores - 1)))); + bucketMask = bucketCount - 1; +} + +template +void TupleJoiner::bucketsToTables(buckets_t *buckets, hash_table_t *tables) +{ + uint i; + + bool done = false, wasProductive; + while (!done) + { + done = true; + wasProductive = false; + for (i = 0; i < bucketCount; i++) + { + if (buckets[i].empty()) + continue; + bool gotIt = m_bucketLocks[i].try_lock(); + if (!gotIt) + { + done = false; + continue; + } + for (auto &element : buckets[i]) + tables[i]->insert(element); + m_bucketLocks[i].unlock(); + wasProductive = true; + buckets[i].clear(); + } + if (!done && !wasProductive) + ::usleep(1000 * numCores); + } +} + +void TupleJoiner::um_insertTypeless(uint threadID, uint rowCount, Row &r) +{ + TypelessData td[rowCount]; + vector > v[bucketCount]; + uint i; + FixedAllocator *alloc = &storedKeyAlloc[threadID]; + + for (i = 0; i < rowCount; i++, r.nextRow()) + { + td[i] = makeTypelessKey(r, smallKeyColumns, keyLength, alloc, + largeRG, largeKeyColumns); + if (td[i].len == 0) + continue; + uint bucket = bucketPicker((char *) td[i].data, td[i].len, bpSeed) & bucketMask; + v[bucket].push_back(pair(td[i], r.getPointer())); + } + bucketsToTables(&v[0], ht.get()); +} + +void TupleJoiner::um_insertLongDouble(uint rowCount, Row &r) +{ + vector > v[bucketCount]; + uint i; + uint smallKeyColumn = smallKeyColumns[0]; + + for (i = 0; i < rowCount; i++, r.nextRow()) + { + long double smallKey = r.getLongDoubleField(smallKeyColumn); + uint bucket = bucketPicker((char *) &smallKey, 10, bpSeed) & bucketMask; // change if we decide to support windows again + if (UNLIKELY(smallKey == joblist::LONGDOUBLENULL)) + v[bucket].push_back(pair(joblist::LONGDOUBLENULL, r.getPointer())); + else + v[bucket].push_back(pair(smallKey, r.getPointer())); + } + bucketsToTables(&v[0], ld.get()); +} + +void TupleJoiner::um_insertInlineRows(uint rowCount, Row &r) +{ + uint i; + int64_t smallKey; + vector > v[bucketCount]; + uint smallKeyColumn = smallKeyColumns[0]; + + for (i = 0; i < rowCount; i++, r.nextRow()) + { + if (!r.isUnsigned(smallKeyColumn)) + smallKey = r.getIntField(smallKeyColumn); + else + smallKey = (int64_t) r.getUintField(smallKeyColumn); + uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + v[bucket].push_back(pair(getJoinNullValue(), r.getData())); + else + v[bucket].push_back(pair(smallKey, r.getData())); + } + bucketsToTables(&v[0], h.get()); +} + +void TupleJoiner::um_insertStringTable(uint rowCount, Row &r) +{ + int64_t smallKey; + uint i; + vector > v[bucketCount]; + uint smallKeyColumn = smallKeyColumns[0]; + + for (i = 0; i < rowCount; i++, r.nextRow()) + { + if (!r.isUnsigned(smallKeyColumn)) + smallKey = r.getIntField(smallKeyColumn); + else + smallKey = (int64_t) r.getUintField(smallKeyColumn); + uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + v[bucket].push_back(pair(getJoinNullValue(), r.getPointer())); + else + v[bucket].push_back(pair(smallKey, r.getPointer())); + } + bucketsToTables(&v[0], sth.get()); +} + +void TupleJoiner::insertRGData(RowGroup &rg, uint threadID) +{ + uint i, rowCount; + Row r; + + rg.initRow(&r); + rowCount = rg.getRowCount(); + + rg.getRow(0, &r); + m_cpValuesLock.lock(); + for (i = 0; i < rowCount; i++, r.nextRow()) + { + updateCPData(r); + r.zeroRid(); + } + m_cpValuesLock.unlock(); + rg.getRow(0, &r); + + if (joinAlg == UM) + { + if (typelessJoin) + um_insertTypeless(threadID, rowCount, r); + else if (r.getColType(smallKeyColumns[0]) == execplan::CalpontSystemCatalog::LONGDOUBLE) + um_insertLongDouble(rowCount, r); + else if (!smallRG.usesStringTable()) + um_insertInlineRows(rowCount, r); + else + um_insertStringTable(rowCount, r); + } + else + { + // while in PM-join mode, inserting is single-threaded + for (i = 0; i < rowCount; i++, r.nextRow()) + rows.push_back(r.getPointer()); + } +} + void TupleJoiner::insert(Row& r, bool zeroTheRid) { /* when doing a disk-based join, only the first iteration on the large side @@ -212,49 +408,54 @@ void TupleJoiner::insert(Row& r, bool zeroTheRid) { if (typelessJoin) { - TypelessData td = makeTypelessKey(r, smallKeyColumns, keyLength, &storedKeyAlloc, + TypelessData td = makeTypelessKey(r, smallKeyColumns, keyLength, &storedKeyAlloc[0], largeRG, largeKeyColumns); if (td.len > 0) { - ht->insert(pair(td, r.getPointer())); + uint bucket = bucketPicker((char *) td.data, td.len, bpSeed) & bucketMask; + ht[bucket]->insert(pair(td, r.getPointer())); } } else if (r.getColType(smallKeyColumns[0]) == execplan::CalpontSystemCatalog::LONGDOUBLE) { long double smallKey = r.getLongDoubleField(smallKeyColumns[0]); + uint bucket = bucketPicker((char *) &smallKey, 10, bpSeed) & bucketMask; // change if we decide to support windows again if (UNLIKELY(smallKey == joblist::LONGDOUBLENULL)) - ld->insert(pair(joblist::LONGDOUBLENULL, r.getPointer())); + ld[bucket]->insert(pair(joblist::LONGDOUBLENULL, r.getPointer())); else - ld->insert(pair(smallKey, r.getPointer())); + ld[bucket]->insert(pair(smallKey, r.getPointer())); } else if (!smallRG.usesStringTable()) { int64_t smallKey; - if (r.isUnsigned(smallKeyColumns[0])) - smallKey = (int64_t)(r.getUintField(smallKeyColumns[0])); - else + if (!r.isUnsigned(smallKeyColumns[0])) smallKey = r.getIntField(smallKeyColumns[0]); - - if (UNLIKELY(smallKey == nullValueForJoinColumn)) - h->insert(pair(getJoinNullValue(), r.getData())); else - h->insert(pair(smallKey, r.getData())); // Normal path for integers + smallKey = (int64_t) r.getUintField(smallKeyColumns[0]); + uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + h[bucket]->insert(pair(getJoinNullValue(), r.getData())); + else + h[bucket]->insert(pair(smallKey, r.getData())); // Normal path for integers } else { - int64_t smallKey = r.getIntField(smallKeyColumns[0]); + int64_t smallKey; - if (UNLIKELY(smallKey == nullValueForJoinColumn)) - sth->insert(pair(getJoinNullValue(), r.getPointer())); + if (!r.isUnsigned(smallKeyColumns[0])) + smallKey = r.getIntField(smallKeyColumns[0]); else - sth->insert(pair(smallKey, r.getPointer())); + smallKey = (int64_t) r.getUintField(smallKeyColumns[0]); + uint bucket = bucketPicker((char *) &smallKey, sizeof(smallKey), bpSeed) & bucketMask; + if (UNLIKELY(smallKey == nullValueForJoinColumn)) + sth[bucket]->insert(pair(getJoinNullValue(), r.getPointer())); + else + sth[bucket]->insert(pair(smallKey, r.getPointer())); } } else - { rows.push_back(r.getPointer()); - } } void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uint32_t threadID, @@ -262,7 +463,6 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin { uint32_t i; bool isNull = hasNullJoinColumn(largeSideRow); - matches->clear(); if (inPM()) @@ -286,19 +486,14 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin pair range; largeKey = makeTypelessKey(largeSideRow, largeKeyColumns, keyLength, &tmpKeyAlloc[threadID], smallRG, smallKeyColumns); - if (largeKey.len > 0) - { - it = ht->find(largeKey); - } - else - { - return; - } - - if (it == ht->end() && !(joinType & (LARGEOUTER | MATCHNULLS))) + if (largeKey.len == 0) return; - range = ht->equal_range(largeKey); + uint bucket = bucketPicker((char *) largeKey.data, largeKey.len, bpSeed) & bucketMask; + range = ht[bucket]->equal_range(largeKey); + + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) + return; for (; range.first != range.second; ++range.first) matches->push_back(range.first->second); @@ -313,13 +508,11 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin Row r; largeKey = largeSideRow.getLongDoubleField(largeKeyColumns[0]); - it = ld->find(largeKey); + uint bucket = bucketPicker((char *) &largeKey, 10, bpSeed) & bucketMask; + range = ld[bucket]->equal_range(largeKey); - if (it == ld->end() && !(joinType & (LARGEOUTER | MATCHNULLS))) + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) return; - - range = ld->equal_range(largeKey); - for (; range.first != range.second; ++range.first) { matches->push_back(range.first->second); @@ -328,9 +521,6 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin else if (!smallRG.usesStringTable()) { int64_t largeKey; - iterator it; - pair range; - Row r; if (largeSideRow.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) { @@ -348,62 +538,39 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin if (ld) { // Compare against long double - ldIterator it; - pair range; long double ldKey = largeKey; -// ldKey = 6920; - it = ld->find(ldKey); + uint bucket = bucketPicker((char *) &ldKey, 10, bpSeed) & bucketMask; + auto range = ld[bucket]->equal_range(ldKey); - if (it == ld->end() && !(joinType & (LARGEOUTER | MATCHNULLS))) + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) return; - range = ld->equal_range(ldKey); - for (; range.first != range.second; ++range.first) - { matches->push_back(range.first->second); - } } else { - it = h->find(largeKey); + uint bucket = bucketPicker((char *) &largeKey, sizeof(largeKey), bpSeed) & bucketMask; + auto range = h[bucket]->equal_range(largeKey); - if (it == h->end() && !(joinType & (LARGEOUTER | MATCHNULLS))) + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) return; - range = h->equal_range(largeKey); - - //smallRG.initRow(&r); for (; range.first != range.second; ++range.first) - { - //r.setData(range.first->second); - //cerr << "matched small side row: " << r.toString() << endl; matches->push_back(range.first->second); - } } } else { - int64_t largeKey; - sthash_t::iterator it; - pair range; - Row r; + int64_t largeKey = largeSideRow.getIntField(largeKeyColumns[0]); + uint bucket = bucketPicker((char *) &largeKey, sizeof(largeKey), bpSeed) & bucketMask; + auto range = sth[bucket]->equal_range(largeKey); - largeKey = largeSideRow.getIntField(largeKeyColumns[0]); - it = sth->find(largeKey); - - if (it == sth->end() && !(joinType & (LARGEOUTER | MATCHNULLS))) + if (range.first == range.second && !(joinType & (LARGEOUTER | MATCHNULLS))) return; - range = sth->equal_range(largeKey); - - //smallRG.initRow(&r); for (; range.first != range.second; ++range.first) - { - //r.setPointer(range.first->second); - //cerr << "matched small side row: " << r.toString() << endl; matches->push_back(range.first->second); - } } } @@ -417,21 +584,27 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin { if (smallRG.getColType(largeKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) { - pair range = ld->equal_range(joblist::LONGDOUBLENULL); + uint bucket = bucketPicker((char *) &(joblist::LONGDOUBLENULL), + sizeof(joblist::LONGDOUBLENULL), bpSeed) & bucketMask; + pair range = ld[bucket]->equal_range(joblist::LONGDOUBLENULL); for (; range.first != range.second; ++range.first) matches->push_back(range.first->second); } else if (!smallRG.usesStringTable()) { - pair range = h->equal_range(getJoinNullValue()); + auto nullVal = getJoinNullValue(); + uint bucket = bucketPicker((char *) &nullVal, sizeof(nullVal), bpSeed) & bucketMask; + pair range = h[bucket]->equal_range(nullVal); for (; range.first != range.second; ++range.first) matches->push_back(range.first->second); } else { - pair range = sth->equal_range(getJoinNullValue()); + auto nullVal = getJoinNullValue(); + uint bucket = bucketPicker((char *) &nullVal, sizeof(nullVal), bpSeed) & bucketMask; + pair range = sth[bucket]->equal_range(nullVal); for (; range.first != range.second; ++range.first) matches->push_back(range.first->second); @@ -448,30 +621,34 @@ void TupleJoiner::match(rowgroup::Row& largeSideRow, uint32_t largeRowIndex, uin { ldIterator it; - for (it = ld->begin(); it != ld->end(); ++it) - matches->push_back(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = ld[i]->begin(); it != ld[i]->end(); ++it) + matches->push_back(it->second); } else if (!smallRG.usesStringTable()) { iterator it; - for (it = h->begin(); it != h->end(); ++it) - matches->push_back(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = h[i]->begin(); it != h[i]->end(); ++it) + matches->push_back(it->second); } else { sthash_t::iterator it; - for (it = sth->begin(); it != sth->end(); ++it) - matches->push_back(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = sth[i]->begin(); it != sth[i]->end(); ++it) + matches->push_back(it->second); } } else { thIterator it; - for (it = ht->begin(); it != ht->end(); ++it) - matches->push_back(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = ht[i]->begin(); it != ht[i]->end(); ++it) + matches->push_back(it->second); } } } @@ -516,16 +693,17 @@ void TupleJoiner::doneInserting() rowCount = size(); + uint bucket = 0; if (joinAlg == PM) pmpos = 0; else if (typelessJoin) - thit = ht->begin(); + thit = ht[bucket]->begin(); else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) - ldit = ld->begin(); + ldit = ld[bucket]->begin(); else if (!smallRG.usesStringTable()) - hit = h->begin(); + hit = h[bucket]->begin(); else - sthit = sth->begin(); + sthit = sth[bucket]->begin(); for (i = 0; i < rowCount; i++) { @@ -533,21 +711,29 @@ void TupleJoiner::doneInserting() smallRow.setPointer(rows[pmpos++]); else if (typelessJoin) { + while (thit == ht[bucket]->end()) + thit = ht[++bucket]->begin(); smallRow.setPointer(thit->second); ++thit; } else if (smallRG.getColType(smallKeyColumns[col]) == CalpontSystemCatalog::LONGDOUBLE) { + while (ldit == ld[bucket]->end()) + ldit = ld[++bucket]->begin(); smallRow.setPointer(ldit->second); ++ldit; } else if (!smallRG.usesStringTable()) { + while (hit == h[bucket]->end()) + hit = h[++bucket]->begin(); smallRow.setPointer(hit->second); ++hit; } else { + while (sthit == sth[bucket]->end()) + sthit = sth[++bucket]->begin(); smallRow.setPointer(sthit->second); ++sthit; } @@ -565,13 +751,12 @@ void TupleJoiner::doneInserting() uniquer.insert(*(int64_t*)&dval); } default: - { + { uniquer.insert((int64_t)dval); } } } - else - if (smallRow.isUnsigned(smallKeyColumns[col])) + else if (smallRow.isUnsigned(smallKeyColumns[col])) { uniquer.insert((int64_t)smallRow.getUintField(smallKeyColumns[col])); } @@ -599,6 +784,18 @@ void TupleJoiner::setInPM() joinAlg = PM; } +void TupleJoiner::umJoinConvert(size_t begin, size_t end) +{ + Row smallRow; + smallRG.initRow(&smallRow); + + while (begin < end) + { + smallRow.setPointer(rows[begin++]); + insert(smallRow); + } +} + void TupleJoiner::setInUM() { vector empty; @@ -610,16 +807,17 @@ void TupleJoiner::setInUM() joinAlg = UM; size = rows.size(); - smallRG.initRow(&smallRow); -#ifdef TJ_DEBUG - cout << "converting array to hash, size = " << size << "\n"; -#endif + size_t chunkSize = ((size / numCores) + 1 < 50000 ? 50000 : (size / numCores) + 1); // don't start a thread to process < 50k rows - for (i = 0; i < size; i++) - { - smallRow.setPointer(rows[i]); - insert(smallRow); - } + uint64_t jobs[numCores]; + i = 0; + for (size_t firstRow = 0; i < (uint) numCores && firstRow < size; i++, firstRow += chunkSize) + jobs[i] = jobstepThreadPool->invoke([this, firstRow, chunkSize, size] { + this->umJoinConvert(firstRow, (firstRow + chunkSize < size ? firstRow + chunkSize : size)); + } ); + + for (uint j = 0; j < i; j++) + jobstepThreadPool->join(jobs[j]); #ifdef TJ_DEBUG cout << "done\n"; @@ -635,6 +833,57 @@ void TupleJoiner::setInUM() } } +void TupleJoiner::umJoinConvert(uint threadID, vector &rgs, size_t begin, size_t end) +{ + RowGroup l_smallRG(smallRG); + + while (begin < end) + { + l_smallRG.setData(&(rgs[begin++])); + insertRGData(l_smallRG, threadID); + } +} + +void TupleJoiner::setInUM(vector &rgs) +{ + Row smallRow; + uint32_t i, size; + + if (joinAlg == UM) + return; + + { // don't need rows anymore, free the mem + vector empty; + rows.swap(empty); + } + + joinAlg = UM; + size = rgs.size(); + size_t chunkSize = ((size / numCores) + 1 < 10 ? 10 : (size / numCores) + 1); // don't issue jobs for < 10 rowgroups + + uint64_t jobs[numCores]; + i = 0; + for (size_t firstRow = 0; i < (uint) numCores && firstRow < size; i++, firstRow += chunkSize) + jobs[i] = jobstepThreadPool->invoke([this, firstRow, chunkSize, size, i, &rgs] { + this->umJoinConvert(i, rgs, firstRow, (firstRow + chunkSize < size ? firstRow + chunkSize : size)); + } ); + + for (uint j = 0; j < i; j++) + jobstepThreadPool->join(jobs[j]); + +#ifdef TJ_DEBUG + cout << "done\n"; +#endif + + if (typelessJoin) + { + tmpKeyAlloc.reset(new FixedAllocator[threadCount]); + + for (i = 0; i < threadCount; i++) + tmpKeyAlloc[i] = FixedAllocator(keyLength, true); + } +} + void TupleJoiner::setPMJoinResults(boost::shared_array > jr, uint32_t threadID) { @@ -727,49 +976,53 @@ void TupleJoiner::getUnmarkedRows(vector* out) { typelesshash_t::iterator it; - for (it = ht->begin(); it != ht->end(); ++it) - { - smallR.setPointer(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = ht[i]->begin(); it != ht[i]->end(); ++it) + { + smallR.setPointer(it->second); - if (!smallR.isMarked()) - out->push_back(it->second); - } + if (!smallR.isMarked()) + out->push_back(it->second); + } } else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) { ldIterator it; - for (it = ld->begin(); it != ld->end(); ++it) - { - smallR.setPointer(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = ld[i]->begin(); it != ld[i]->end(); ++it) + { + smallR.setPointer(it->second); - if (!smallR.isMarked()) - out->push_back(it->second); - } + if (!smallR.isMarked()) + out->push_back(it->second); + } } else if (!smallRG.usesStringTable()) { iterator it; - for (it = h->begin(); it != h->end(); ++it) - { - smallR.setPointer(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = h[i]->begin(); it != h[i]->end(); ++it) + { + smallR.setPointer(it->second); - if (!smallR.isMarked()) - out->push_back(it->second); - } + if (!smallR.isMarked()) + out->push_back(it->second); + } } else { sthash_t::iterator it; - for (it = sth->begin(); it != sth->end(); ++it) - { - smallR.setPointer(it->second); + for (uint i = 0; i < bucketCount; i++) + for (it = sth[i]->begin(); it != sth[i]->end(); ++it) + { + smallR.setPointer(it->second); - if (!smallR.isMarked()) - out->push_back(it->second); - } + if (!smallR.isMarked()) + out->push_back(it->second); + } } } } @@ -777,9 +1030,21 @@ void TupleJoiner::getUnmarkedRows(vector* out) uint64_t TupleJoiner::getMemUsage() const { if (inUM() && typelessJoin) - return _pool->getMemUsage() + storedKeyAlloc.getMemUsage(); + { + size_t ret = 0; + for (uint i = 0; i < bucketCount; i++) + ret += _pool[i]->getMemUsage(); + for (int i = 0; i < numCores; i++) + ret += storedKeyAlloc[i].getMemUsage(); + return ret; + } else if (inUM()) - return _pool->getMemUsage(); + { + size_t ret = 0; + for (uint i = 0; i < bucketCount; i++) + ret += _pool[i]->getMemUsage(); + return ret; + } else return (rows.size() * sizeof(Row::Pointer)); } @@ -841,7 +1106,7 @@ void TupleJoiner::updateCPData(const Row& r) uval = *(uint64_t*)&dval; } default: - { + { uval = (uint64_t)dval; } } @@ -873,7 +1138,7 @@ void TupleJoiner::updateCPData(const Row& r) val = *(int64_t*)&dval; } default: - { + { val = (int64_t)dval; } } @@ -896,14 +1161,17 @@ size_t TupleJoiner::size() const { if (joinAlg == UM || joinAlg == INSERTING) { - if (UNLIKELY(typelessJoin)) - return ht->size(); - else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) - return ld->size(); - else if (!smallRG.usesStringTable()) - return h->size(); - else - return sth->size(); + size_t ret = 0; + for (uint i = 0; i < bucketCount; i++) + if (UNLIKELY(typelessJoin)) + ret += ht[i]->size(); + else if (smallRG.getColType(smallKeyColumns[0]) == CalpontSystemCatalog::LONGDOUBLE) + ret += ld[i]->size(); + else if (!smallRG.usesStringTable()) + ret += h[i]->size(); + else + ret += sth[i]->size(); + return ret; } return rows.size(); @@ -1050,7 +1318,7 @@ TypelessData makeTypelessKey(const Row& r, const vector& keyCols, break; } default: - { + { if (off + 8 > keylen) goto toolong; if (r.isUnsigned(keyCols[i]) && keyld > MAX_UBIGINT) @@ -1178,7 +1446,7 @@ TypelessData makeTypelessKey(const Row& r, const vector& keyCols, Pool break; } default: - { + { if (r.isUnsigned(keyCols[i]) && keyld > MAX_UBIGINT) { ret.len = 0; @@ -1352,15 +1620,29 @@ void TupleJoiner::setTableName(const string& tname) void TupleJoiner::clearData() { - STLPoolAllocator > alloc; - _pool = alloc.getPoolAllocator(); - + _pool.reset(new boost::shared_ptr[bucketCount]); if (typelessJoin) - ht.reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); + ht.reset(new boost::scoped_ptr[bucketCount]); + else if (smallRG.getColTypes()[smallKeyColumns[0]] == CalpontSystemCatalog::LONGDOUBLE) + ld.reset(new boost::scoped_ptr[bucketCount]); else if (smallRG.usesStringTable()) - sth.reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); + sth.reset(new boost::scoped_ptr[bucketCount]); else - h.reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); + h.reset(new boost::scoped_ptr[bucketCount]); + + for (uint i = 0; i < bucketCount; i++) + { + STLPoolAllocator > alloc; + _pool[i] = alloc.getPoolAllocator(); + if (typelessJoin) + ht[i].reset(new typelesshash_t(10, hasher(), typelesshash_t::key_equal(), alloc)); + else if (smallRG.getColTypes()[smallKeyColumns[0]] == CalpontSystemCatalog::LONGDOUBLE) + ld[i].reset(new ldhash_t(10, hasher(), ldhash_t::key_equal(), alloc)); + else if (smallRG.usesStringTable()) + sth[i].reset(new sthash_t(10, hasher(), sthash_t::key_equal(), alloc)); + else + h[i].reset(new hash_t(10, hasher(), hash_t::key_equal(), alloc)); + } std::vector empty; rows.swap(empty); @@ -1406,7 +1688,16 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() } if (typelessJoin) - ret->storedKeyAlloc = FixedAllocator(keyLength); + { + ret->storedKeyAlloc.reset(new FixedAllocator[numCores]); + for (int i = 0; i < numCores; i++) + ret->storedKeyAlloc[i].setAllocSize(keyLength); + } + + ret->numCores = numCores; + ret->bucketCount = bucketCount; + ret->bucketMask = bucketMask; + ret->jobstepThreadPool = jobstepThreadPool; ret->setThreadCount(1); ret->clearData(); @@ -1414,9 +1705,9 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() return ret; } - - - - +void TupleJoiner::setConvertToDiskJoin() +{ + _convertToDiskJoin = true; +} }; diff --git a/utils/joiner/tuplejoiner.h b/utils/joiner/tuplejoiner.h index 83d68dfca..b0d38364d 100644 --- a/utils/joiner/tuplejoiner.h +++ b/utils/joiner/tuplejoiner.h @@ -38,6 +38,7 @@ #include "../funcexp/funcexpwrapper.h" #include "stlpoolallocator.h" #include "hasher.h" +#include "threadpool.h" namespace joiner { @@ -147,7 +148,8 @@ public: const rowgroup::RowGroup& largeInput, uint32_t smallJoinColumn, uint32_t largeJoinColumn, - joblist::JoinType jt); + joblist::JoinType jt, + threadpool::ThreadPool *jsThreadPool); /* ctor to use for string & compound join */ TupleJoiner( @@ -155,12 +157,14 @@ public: const rowgroup::RowGroup& largeInput, const std::vector& smallJoinColumns, const std::vector& largeJoinColumns, - joblist::JoinType jt); + joblist::JoinType jt, + threadpool::ThreadPool *jsThreadPool); ~TupleJoiner(); size_t size() const; - void insert(rowgroup::Row& r, bool zeroTheRid = true); + void insert(rowgroup::Row& r, bool zeroTheRid = true); // not thread-safe + void insertRGData(rowgroup::RowGroup &rg, uint threadID); void doneInserting(); /* match() returns the small-side rows that match the large-side row. @@ -187,8 +191,20 @@ public: { return joinAlg == UM; } + inline bool onDisk() const + { + return _convertToDiskJoin; + } void setInPM(); + void setInUM(std::vector &rgs); + void umJoinConvert(uint threadID, std::vector &rgs, size_t begin, size_t end); + + // TODO: these are currently in use by edge cases, ex, converting to disk + // join. Would be nice to make those cases use the rgdata variants + // above. void setInUM(); + void umJoinConvert(size_t begin, size_t end); + void setThreadCount(uint32_t cnt); void setPMJoinResults(boost::shared_array >, uint32_t threadID); @@ -325,6 +341,7 @@ public: { return finished; } + void setConvertToDiskJoin(); private: typedef std::tr1::unordered_multimap, @@ -344,13 +361,13 @@ private: TupleJoiner(); TupleJoiner(const TupleJoiner&); TupleJoiner& operator=(const TupleJoiner&); + void getBucketCount(); rowgroup::RGData smallNullMemory; - - boost::scoped_ptr h; // used for UM joins on ints - boost::scoped_ptr sth; // used for UM join on ints where the backing table uses a string table - boost::scoped_ptr ld; // used for UM join on long double + boost::scoped_array > h; // used for UM joins on ints + boost::scoped_array > sth; // used for UM join on ints where the backing table uses a string table + boost::scoped_array > ld; // used for UM join on long double std::vector rows; // used for PM join /* This struct is rough. The BPP-JL stores the parsed results for @@ -372,16 +389,16 @@ private: }; JoinAlg joinAlg; joblist::JoinType joinType; - boost::shared_ptr _pool; // pool for the table and nodes + boost::shared_array > _pool; // pools for the table and nodes uint32_t threadCount; std::string tableName; /* vars, & fcns for typeless join */ bool typelessJoin; std::vector smallKeyColumns, largeKeyColumns; - boost::scoped_ptr ht; // used for UM join on strings + boost::scoped_array > ht; // used for UM join on strings uint32_t keyLength; - utils::FixedAllocator storedKeyAlloc; + boost::scoped_array storedKeyAlloc; boost::scoped_array tmpKeyAlloc; bool bSignedUnsignedJoin; // Set if we have a signed vs unsigned compare in a join. When not set, we can save checking for the signed bit. @@ -398,9 +415,27 @@ private: boost::scoped_array > cpValues; // if !discreteValues, [0] has min, [1] has max uint32_t uniqueLimit; bool finished; + + // multithreaded UM hash table construction + int numCores; + uint bucketCount; + uint bucketMask; + boost::scoped_array m_bucketLocks; + boost::mutex m_typelessLock, m_cpValuesLock; + utils::Hasher_r bucketPicker; + const uint32_t bpSeed = 0x4545e1d7; // an arbitrary random # + threadpool::ThreadPool *jobstepThreadPool; + void um_insertTypeless(uint threadID, uint rowcount, rowgroup::Row &r); + void um_insertLongDouble(uint rowcount, rowgroup::Row &r); + void um_insertInlineRows(uint rowcount, rowgroup::Row &r); + void um_insertStringTable(uint rowcount, rowgroup::Row &r); + + template + void bucketsToTables(buckets_t *, hash_table_t *); + + bool _convertToDiskJoin; }; } #endif - diff --git a/utils/threadpool/threadpool.cpp b/utils/threadpool/threadpool.cpp index e62de7769..da2b5f24f 100644 --- a/utils/threadpool/threadpool.cpp +++ b/utils/threadpool/threadpool.cpp @@ -29,6 +29,7 @@ using namespace std; using namespace logging; #include "threadpool.h" +#include "threadnaming.h" #include #include #include "boost/date_time/posix_time/posix_time_types.hpp" @@ -321,6 +322,7 @@ uint64_t ThreadPool::invoke(const Functor_T& threadfunc) void ThreadPool::beginThread() throw() { + utils::setThreadName("Idle"); try { boost::mutex::scoped_lock lock1(fMutex); @@ -386,6 +388,7 @@ void ThreadPool::beginThread() throw() lock1.unlock(); + utils::setThreadName("Unspecified"); try { todo->functor(); @@ -405,6 +408,7 @@ void ThreadPool::beginThread() throw() #endif } + utils::setThreadName("Idle"); lock1.lock(); --fIssued; --waitingFunctorsSize; From c866715efe70544b0e42544ae9e6457b61e3f488 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 21 Nov 2019 14:51:46 -0500 Subject: [PATCH 04/35] Removed CalpontHome config param. --- oam/etc/Columnstore.xml.singleserver | 1 - 1 file changed, 1 deletion(-) diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver index 4c3286c40..37909c361 100644 --- a/oam/etc/Columnstore.xml.singleserver +++ b/oam/etc/Columnstore.xml.singleserver @@ -241,7 +241,6 @@ 128M 10 /tmp/columnstore_tmp_files - $INSTALLDIR 10 3 10 From 5bbd21b8e0d03bf28264324a7a50f1fdd92dea0d Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 20 Nov 2019 11:57:55 -0600 Subject: [PATCH 05/35] MCOL-3577: Add functionality to sync S3 storage for suspendDatabaseWrites. --- procmgr/processmanager.cpp | 12 +++++ storage-manager/CMakeLists.txt | 1 + storage-manager/include/messageFormat.h | 3 +- storage-manager/src/ProcessTask.cpp | 4 ++ storage-manager/src/SyncTask.cpp | 62 +++++++++++++++++++++++++ storage-manager/src/SyncTask.h | 41 ++++++++++++++++ storage-manager/src/Synchronizer.cpp | 36 ++++++++++++++ storage-manager/src/Synchronizer.h | 6 +-- utils/cloudio/SMComm.cpp | 14 ++++++ utils/cloudio/SMComm.h | 2 + utils/cloudio/SMFileSystem.cpp | 5 ++ utils/cloudio/SMFileSystem.h | 2 + utils/idbdatafile/IDBFileSystem.h | 5 ++ 13 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 storage-manager/src/SyncTask.cpp create mode 100644 storage-manager/src/SyncTask.h diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index b2315f5f4..be3ae085e 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2712,6 +2712,8 @@ void processMSG(messageqcpp::IOSocket* cfIos) ByteStream::byte ackResponse = API_FAILURE; log.writeLog(__LINE__, "MSG RECEIVED: suspend database writes"); + string storageType = Config::makeConfig()->getConfig("Installation", "DBRootStorageType"); + // GRACEFUL_WAIT means that we are Suspending writes, but waiting for all // transactions to finish or rollback as commanded. This is only set if there // are, in fact, transactions active (or cpimport). @@ -2785,6 +2787,16 @@ void processMSG(messageqcpp::IOSocket* cfIos) dbrm.setSystemSuspended(false); } + if (storageType == "storagemanager") + { + string DBRMroot; + oam.getSystemConfig("DBRMRoot", DBRMroot); + + string currentFileName = DBRMroot + "_current"; + IDBFileSystem &fs = IDBPolicy::getFs(currentFileName.c_str()); + fs.filesystemSync(); + } + ackMsg.reset(); ackMsg << (ByteStream::byte) oam::ACK; ackMsg << actionType; diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index ff1138aa5..36fcfc796 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -34,6 +34,7 @@ set(storagemanager_SRCS src/Utilities.cpp src/Ownership.cpp src/PrefixCache.cpp + src/SyncTask.cpp ../utils/common/crashtrace.cpp ) diff --git a/storage-manager/include/messageFormat.h b/storage-manager/include/messageFormat.h index d0a20ca96..aefcea4da 100755 --- a/storage-manager/include/messageFormat.h +++ b/storage-manager/include/messageFormat.h @@ -82,7 +82,8 @@ enum Opcodes { TRUNCATE, LIST_DIRECTORY, PING, - COPY + COPY, + SYNC }; /* diff --git a/storage-manager/src/ProcessTask.cpp b/storage-manager/src/ProcessTask.cpp index feb79ca33..c6cdc337f 100644 --- a/storage-manager/src/ProcessTask.cpp +++ b/storage-manager/src/ProcessTask.cpp @@ -33,6 +33,7 @@ #include "TruncateTask.h" #include "UnlinkTask.h" #include "WriteTask.h" +#include "SyncTask.h" #include "SessionManager.h" #include "SMLogging.h" @@ -109,6 +110,9 @@ void ProcessTask::operator()() case PING: task.reset(new PingTask(sock, length)); break; + case SYNC: + task.reset(new SyncTask(sock, length)); + break; case COPY: task.reset(new CopyTask(sock, length)); break; diff --git a/storage-manager/src/SyncTask.cpp b/storage-manager/src/SyncTask.cpp new file mode 100644 index 000000000..a962f9151 --- /dev/null +++ b/storage-manager/src/SyncTask.cpp @@ -0,0 +1,62 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + + +#include "SyncTask.h" +#include "Synchronizer.h" +#include "messageFormat.h" +#include + +namespace storagemanager +{ + +SyncTask::SyncTask(int sock, uint len) : PosixTask(sock, len) +{ +} + +SyncTask::~SyncTask() +{ +} + +bool SyncTask::run() +{ + // force flush synchronizer + uint8_t buf; + + if (getLength() > 1) + { + handleError("SyncTask", E2BIG); + return true; + } + // consume the msg + bool success = read(&buf, getLength()); + if (!success) + { + handleError("SyncTask", errno); + return false; + } + + Synchronizer::get()->syncNow(); + + // send generic success response + sm_response ret; + ret.returnCode = 0; + success = write(ret, 0); + return success; +} + +} diff --git a/storage-manager/src/SyncTask.h b/storage-manager/src/SyncTask.h new file mode 100644 index 000000000..c6b3a4abe --- /dev/null +++ b/storage-manager/src/SyncTask.h @@ -0,0 +1,41 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + + + +#ifndef SYNCTASK_H_ +#define SYNCTASK_H_ + +#include "PosixTask.h" + +namespace storagemanager +{ + +class SyncTask : public PosixTask +{ + public: + SyncTask(int sock, uint length); + virtual ~SyncTask(); + + bool run(); + + private: + SyncTask(); +}; + +} +#endif diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index 979c1ac94..017a78d61 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -185,6 +185,10 @@ void Synchronizer::deletedObjects(const bf::path &prefix, const vector & void Synchronizer::flushObject(const bf::path &prefix, const string &_key) { string key = (prefix/_key).string(); + + while (blockNewJobs) + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + boost::unique_lock s(mutex); // if there is something to do on key, it should be either in pendingOps or opsInProgress @@ -307,13 +311,45 @@ void Synchronizer::syncNow(const bf::path &prefix) if (job.first.find(prefix.string()) == 0) makeJob(job.first); uncommittedJournalSize[prefix] = 0; + lock.unlock(); threadPool.reset(new ThreadPool()); threadPool->setMaxThreads(maxUploads); + lock.lock(); blockNewJobs = false; } + +void Synchronizer::syncNow() +{ + boost::unique_lock lock(mutex); + + // this is pretty hacky. when time permits, implement something better. + // + // Issue all of the pendingOps for the given prefix + // recreate the threadpool (dtor returns once all jobs have finished) + // resume normal operation + + blockNewJobs = true; + while (pendingOps.size() != 0) + { + for (auto &job : pendingOps) + makeJob(job.first); + for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) + it->second = 0; + lock.unlock(); + while (opsInProgress.size() > 0) + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + if (pendingOps.size() != 0) + logger->log(LOG_DEBUG,"Synchronizer syncNow pendingOps not empty."); + lock.lock(); + } + blockNewJobs = false; +} + + void Synchronizer::forceFlush() { boost::unique_lock lock(mutex); + syncThread.interrupt(); } diff --git a/storage-manager/src/Synchronizer.h b/storage-manager/src/Synchronizer.h index 2ed4192a5..908a4e5e8 100644 --- a/storage-manager/src/Synchronizer.h +++ b/storage-manager/src/Synchronizer.h @@ -53,8 +53,8 @@ class Synchronizer : public boost::noncopyable , public ConfigListener void deletedObjects(const boost::filesystem::path &firstDir, const std::vector &keys); void flushObject(const boost::filesystem::path &firstDir, const std::string &key); void forceFlush(); // ideally, make a version of this that takes a firstDir parameter - - + void syncNow(); // synchronous version of force for SyncTask + void newPrefix(const boost::filesystem::path &p); void dropPrefix(const boost::filesystem::path &p); @@ -118,7 +118,7 @@ class Synchronizer : public boost::noncopyable , public ConfigListener bool blockNewJobs; void syncNow(const boost::filesystem::path &prefix); // a synchronous version of forceFlush() - + // some KPIs size_t numBytesRead, numBytesWritten, numBytesUploaded, numBytesDownloaded, flushesTriggeredBySize, flushesTriggeredByTimer, journalsMerged, objectsSyncedWithNoJournal, diff --git a/utils/cloudio/SMComm.cpp b/utils/cloudio/SMComm.cpp index 912596f83..59d93c78f 100644 --- a/utils/cloudio/SMComm.cpp +++ b/utils/cloudio/SMComm.cpp @@ -247,6 +247,20 @@ int SMComm::ping() common_exit(command, response, err); } +int SMComm::sync() +{ + ByteStream *command = buffers.getByteStream(); + ByteStream *response = buffers.getByteStream(); + ssize_t err; + + *command << (uint8_t) storagemanager::SYNC; + err = sockets.send_recv(*command, response); + if (err) + common_exit(command, response, err); + check_for_error(command, response, err); + common_exit(command, response, err); +} + int SMComm::copyFile(const string &file1, const string &file2) { ByteStream *command = buffers.getByteStream(); diff --git a/utils/cloudio/SMComm.h b/utils/cloudio/SMComm.h index 8ae7a853d..d7240e73a 100644 --- a/utils/cloudio/SMComm.h +++ b/utils/cloudio/SMComm.h @@ -59,6 +59,8 @@ class SMComm : public boost::noncopyable // the specified S3 bucket. Need to define specific error codes. int ping(); + int sync(); + int copyFile(const std::string &file1, const std::string &file2); virtual ~SMComm(); diff --git a/utils/cloudio/SMFileSystem.cpp b/utils/cloudio/SMFileSystem.cpp index 5cf1ec453..fcd12b073 100644 --- a/utils/cloudio/SMFileSystem.cpp +++ b/utils/cloudio/SMFileSystem.cpp @@ -110,4 +110,9 @@ bool SMFileSystem::filesystemIsUp() const return (comm->ping() == 0); } +bool SMFileSystem::filesystemSync() const +{ + SMComm *comm = SMComm::get(); + return (comm->sync() == 0); +} } diff --git a/utils/cloudio/SMFileSystem.h b/utils/cloudio/SMFileSystem.h index 80a189dfb..4b136a2fd 100644 --- a/utils/cloudio/SMFileSystem.h +++ b/utils/cloudio/SMFileSystem.h @@ -44,6 +44,8 @@ class SMFileSystem : public IDBFileSystem, boost::noncopyable bool isDir(const char* pathname) const; int copyFile(const char* srcPath, const char* destPath) const; bool filesystemIsUp() const; + bool filesystemSync() const; + }; } diff --git a/utils/idbdatafile/IDBFileSystem.h b/utils/idbdatafile/IDBFileSystem.h index 5979d822f..27ef09be7 100644 --- a/utils/idbdatafile/IDBFileSystem.h +++ b/utils/idbdatafile/IDBFileSystem.h @@ -129,6 +129,11 @@ public: return true; } + virtual bool filesystemSync() const + { + return true; + } + protected: IDBFileSystem( Types type ); From abd7444809b9929c4ea4acdcbabc2172127a3bab Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 20 Nov 2019 11:59:22 -0600 Subject: [PATCH 06/35] MCOL-3577: Found an issues with the order of dropPrefix Cache before Sync. --- storage-manager/src/Ownership.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage-manager/src/Ownership.cpp b/storage-manager/src/Ownership.cpp index d85571a88..569a3c86f 100644 --- a/storage-manager/src/Ownership.cpp +++ b/storage-manager/src/Ownership.cpp @@ -181,8 +181,8 @@ void Ownership::releaseOwnership(const bf::path &p, bool isDtor) // start flushing boost::thread xfer([this, &p, &done] { this->touchFlushing(p, &done); }); - Cache::get()->dropPrefix(p); Synchronizer::get()->dropPrefix(p); + Cache::get()->dropPrefix(p); done = true; xfer.interrupt(); xfer.join(); From 18d31fae82328715d5ef472bd497fdbaeaf63c75 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 20 Nov 2019 12:43:53 -0600 Subject: [PATCH 07/35] MCOL-3577: cleanup debug logging and comments from copy/paste. --- storage-manager/src/Synchronizer.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index 017a78d61..5425b60ed 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -322,11 +322,10 @@ void Synchronizer::syncNow() { boost::unique_lock lock(mutex); - // this is pretty hacky. when time permits, implement something better. - // - // Issue all of the pendingOps for the given prefix - // recreate the threadpool (dtor returns once all jobs have finished) - // resume normal operation + // This should ensure that all pendingOps have been added as jobs + // and waits for them to complete. until pendingOps is empty. + // Used by the mcsadmin command suspendDatabaseWrites. + // Leaving S3 storage and local metadata directories sync'd for snapshot backups. blockNewJobs = true; while (pendingOps.size() != 0) @@ -338,8 +337,6 @@ void Synchronizer::syncNow() lock.unlock(); while (opsInProgress.size() > 0) boost::this_thread::sleep_for(boost::chrono::seconds(1)); - if (pendingOps.size() != 0) - logger->log(LOG_DEBUG,"Synchronizer syncNow pendingOps not empty."); lock.lock(); } blockNewJobs = false; From 830490c84631ff74a8519d8239cc9956c68b8d18 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 21 Nov 2019 15:13:39 -0600 Subject: [PATCH 08/35] MCOL-3577: Make this run syncFS for storagemanager on all nodes when suspend writes is done from mcsadmin. --- oam/oamcpp/liboamcpp.cpp | 4 +++ oam/oamcpp/liboamcpp.h | 3 +- procmgr/processmanager.cpp | 73 +++++++++++++++++++++++++++++++++++++- procmon/processmonitor.cpp | 29 +++++++++++++++ procmon/processmonitor.h | 1 + 5 files changed, 108 insertions(+), 2 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 43b53b832..16ba7aec1 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -3232,6 +3232,10 @@ void Oam::SuspendWrites(GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag) cout << endl << " Suspension of database writes canceled" << endl << endl; break; + case API_FAILURE: + cout << endl << " Suspension of database writes failed: Filesystem sync failed" << endl << endl; + break; + default: exceptionControl("suspendWrites", returnStatus); break; diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index ecdd76d50..51317bb14 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -581,7 +581,8 @@ enum PROC_MGT_TYPE_REQUEST DISABLEREP, PROCGLUSTERASSIGN, PROCGLUSTERUNASSIGN, - CONFIGURE + CONFIGURE, + SYNCFSALL }; diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index be3ae085e..d20edc473 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2794,7 +2794,78 @@ void processMSG(messageqcpp::IOSocket* cfIos) string currentFileName = DBRMroot + "_current"; IDBFileSystem &fs = IDBPolicy::getFs(currentFileName.c_str()); - fs.filesystemSync(); + + if (!fs.filesystemSync()) + { + ackMsg << (ByteStream::byte) oam::ACK; + ackMsg << actionType; + ackMsg << target; + ackMsg << (ByteStream::byte) API_FAILURE; + + try + { + fIos.write(ackMsg); + } + catch (...) {} + + log.writeLog(__LINE__, "SUSPENDWRITES: API_FAILURE filestemSync()",LOG_TYPE_ERROR); + break; + } + //sync fs on all pm nodes if up + for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) + { + if ( systemmoduletypeconfig.moduletypeconfig[i].ModuleType != "pm" ) + continue; + + int moduleCount = systemmoduletypeconfig.moduletypeconfig[i].ModuleCount; + + if ( moduleCount == 0) + continue; + + DeviceNetworkList::iterator pt = systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.begin(); + + for ( ; pt != systemmoduletypeconfig.moduletypeconfig[i].ModuleNetworkList.end(); pt++) + { + int opState = oam::ACTIVE; + bool degraded; + + try + { + oam.getModuleStatus((*pt).DeviceName, opState, degraded); + } + catch (...) + { + log.writeLog(__LINE__, "EXCEPTION ERROR on getModuleStatus on module " + (*pt).DeviceName + ": Caught unknown exception!", LOG_TYPE_ERROR); + } + + if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) + continue; + + ByteStream msg; + ByteStream::byte requestID = SYNCFSALL; + + msg << requestID; + + int returnStatus = processManager.sendMsgProcMon( (*pt).DeviceName, msg, requestID ); + + if (returnStatus != API_SUCCESS) + { + ackMsg << (ByteStream::byte) oam::ACK; + ackMsg << actionType; + ackMsg << target; + ackMsg << (ByteStream::byte) API_FAILURE; + + try + { + fIos.write(ackMsg); + } + catch (...) {} + + log.writeLog(__LINE__, "SUSPENDWRITES: API_FAILURE filestemSync() on module " + (*pt).DeviceName,LOG_TYPE_ERROR); + break; + } + } + } } ackMsg.reset(); diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index d8445ab2b..08887c3ff 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -2072,6 +2072,19 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO break; } + case SYNCFSALL: + { + log.writeLog(__LINE__, "MSG RECEIVED: SYNC FileSystem..."); + int requestStatus = API_SUCCESS; + requestStatus = syncFS(); + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) SHUTDOWNMODULE; + ackMsg << (ByteStream::byte) API_SUCCESS; + mq.write(ackMsg); + + log.writeLog(__LINE__, "SYNCFSALL: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); + break; + } default: break; @@ -6188,6 +6201,22 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) return oam::API_SUCCESS; } + +int ProcessMonitor::syncFS() +{ + Oam oam; + + string DBRMroot; + oam.getSystemConfig("DBRMRoot", DBRMroot); + + string currentFileName = DBRMroot + "_current"; + IDBFileSystem &fs = IDBPolicy::getFs(currentFileName.c_str()); + bool success = fs.filesystemSync(); + if (!success) + return oam::API_FAILURE; + return oam::API_SUCCESS; +} + } //end of namespace // vim:ts=4 sw=4: diff --git a/procmon/processmonitor.h b/procmon/processmonitor.h index 7984ac80c..57e90b1f4 100644 --- a/procmon/processmonitor.h +++ b/procmon/processmonitor.h @@ -542,6 +542,7 @@ public: */ int glusterUnassign(std::string dbrootID); + int syncFS(); /** * return the process list */ From d377defa313e13f2bf1a992be93dac883b9122aa Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 21 Nov 2019 15:14:58 -0600 Subject: [PATCH 09/35] MCOL-3577: update syncNow with prefix to behave like syncNow no prefix. This should be corrected eventually but is safer now as previous implementation would possibly result in deadlock. --- storage-manager/src/Synchronizer.cpp | 31 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index 5425b60ed..7e86f8e2d 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -299,22 +299,23 @@ void Synchronizer::periodicSync() void Synchronizer::syncNow(const bf::path &prefix) { boost::unique_lock lock(mutex); - - // this is pretty hacky. when time permits, implement something better. - // - // Issue all of the pendingOps for the given prefix - // recreate the threadpool (dtor returns once all jobs have finished) - // resume normal operation - + + // This should ensure that all pendingOps have been added as jobs + // and waits for them to complete. until pendingOps is empty. + // this should be redone to only remove items of given prefix eventually + blockNewJobs = true; - for (auto &job : pendingOps) - if (job.first.find(prefix.string()) == 0) + while (pendingOps.size() != 0 && opsInProgress.size() != 0) + { + for (auto &job : pendingOps) makeJob(job.first); - uncommittedJournalSize[prefix] = 0; - lock.unlock(); - threadPool.reset(new ThreadPool()); - threadPool->setMaxThreads(maxUploads); - lock.lock(); + for (auto it = uncommittedJournalSize.begin(); it != uncommittedJournalSize.end(); ++it) + it->second = 0; + lock.unlock(); + while (opsInProgress.size() > 0) + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + lock.lock(); + } blockNewJobs = false; } @@ -328,7 +329,7 @@ void Synchronizer::syncNow() // Leaving S3 storage and local metadata directories sync'd for snapshot backups. blockNewJobs = true; - while (pendingOps.size() != 0) + while (pendingOps.size() != 0 && opsInProgress.size() != 0) { for (auto &job : pendingOps) makeJob(job.first); From f3dae6bf0d8a0bbc9659038513c9fac270800ea2 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 21 Nov 2019 18:50:05 -0600 Subject: [PATCH 10/35] MCOL-3577: Fix messaging for filesytem sync. --- procmgr/processmanager.cpp | 58 ++++++++++++++++++++------------------ procmgr/processmanager.h | 4 +++ procmon/processmonitor.cpp | 15 ++++++++-- 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index d20edc473..aa7877148 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2789,28 +2789,6 @@ void processMSG(messageqcpp::IOSocket* cfIos) if (storageType == "storagemanager") { - string DBRMroot; - oam.getSystemConfig("DBRMRoot", DBRMroot); - - string currentFileName = DBRMroot + "_current"; - IDBFileSystem &fs = IDBPolicy::getFs(currentFileName.c_str()); - - if (!fs.filesystemSync()) - { - ackMsg << (ByteStream::byte) oam::ACK; - ackMsg << actionType; - ackMsg << target; - ackMsg << (ByteStream::byte) API_FAILURE; - - try - { - fIos.write(ackMsg); - } - catch (...) {} - - log.writeLog(__LINE__, "SUSPENDWRITES: API_FAILURE filestemSync()",LOG_TYPE_ERROR); - break; - } //sync fs on all pm nodes if up for ( unsigned int i = 0 ; i < systemmoduletypeconfig.moduletypeconfig.size(); i++) { @@ -2841,12 +2819,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) if (opState == oam::MAN_DISABLED || opState == oam::AUTO_DISABLED) continue; - ByteStream msg; - ByteStream::byte requestID = SYNCFSALL; - - msg << requestID; - - int returnStatus = processManager.sendMsgProcMon( (*pt).DeviceName, msg, requestID ); + int returnStatus = processManager.syncFsAll( (*pt).DeviceName ); if (returnStatus != API_SUCCESS) { @@ -11193,6 +11166,35 @@ int ProcessManager::glusterUnassign(std::string moduleName, std::string dbroot) return returnStatus; } +/****************************************************************************************** +* @brief syncFsALL +* +* purpose: Sync filesystem for backup snapshots on suspenddatabasewrites +* +******************************************************************************************/ +int ProcessManager::syncFsAll(std::string moduleName) +{ + + ByteStream msg; + ByteStream::byte requestID = SYNCFSALL; + + msg << requestID; + + int returnStatus = sendMsgProcMon( moduleName, msg, requestID, 30 ); + + if ( returnStatus == API_SUCCESS) + { + //log the success event + log.writeLog(__LINE__, "syncFsALL Success: " + moduleName, LOG_TYPE_DEBUG); + } + else + { + //log the error event + log.writeLog(__LINE__, "syncFsALL FAILED: " + moduleName, LOG_TYPE_ERROR); + } + + return returnStatus; +} } //end of namespace // vim:ts=4 sw=4: diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 0663c3e4b..9c8e1378d 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -556,6 +556,10 @@ public: */ int glusterUnassign(std::string moduleName, std::string dbroot); + /** @brief sync filesystem for snapshot backups + */ + int syncFsAll(std::string moduleName); + private: diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 08887c3ff..121318758 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -2077,9 +2077,18 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO log.writeLog(__LINE__, "MSG RECEIVED: SYNC FileSystem..."); int requestStatus = API_SUCCESS; requestStatus = syncFS(); - ackMsg << (ByteStream::byte) ACK; - ackMsg << (ByteStream::byte) SHUTDOWNMODULE; - ackMsg << (ByteStream::byte) API_SUCCESS; + if (requestStatus == API_SUCCESS) + { + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) SYNCFSALL; + ackMsg << (ByteStream::byte) API_SUCCESS; + } + else + { + ackMsg << (ByteStream::byte) ACK; + ackMsg << (ByteStream::byte) SYNCFSALL; + ackMsg << (ByteStream::byte) API_FAILURE; + } mq.write(ackMsg); log.writeLog(__LINE__, "SYNCFSALL: ACK back to ProcMgr, return status = " + oam.itoa((int) API_SUCCESS)); From a76ceb5aa02dee56e6db85e9d0edc0b56610fd69 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 22 Nov 2019 01:34:29 +0000 Subject: [PATCH 11/35] Get the associated string values for charset()/collation() functions applied to a field. --- dbcon/mysql/ha_mcs_execplan.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 07fdb25f5..a9675921b 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3680,6 +3680,29 @@ ReturnedColumn* buildFunctionColumn( fc = buildCaseFunction(ifp, gwi, nonSupport); } + else if ((funcName == "charset" || funcName == "collation") && + ifp->argument_count() == 1 && + ifp->arguments()[0]->type() == Item::FIELD_ITEM) + { + Item_field *item = reinterpret_cast(ifp->arguments()[0]); + CHARSET_INFO* info = item->charset_for_protocol(); + ReturnedColumn* rc; + string val; + + if (funcName == "charset") + { + val = info->csname; + } + else // collation + { + val = info->name; + } + + rc = new ConstantColumn(val, ConstantColumn::LITERAL); + + return rc; + } + else if ((functor = funcExp->getFunctor(funcName))) { // where clause isnull still treated as predicate operator From acdd8ca9f1efb4495ef0c21c05688eccd9eedb8c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 22 Nov 2019 10:38:05 -0500 Subject: [PATCH 12/35] Removed a debugging printout, used RM to get core count. --- dbcon/joblist/tuplehashjoin.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 2ad3c495b..20967e15f 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -116,7 +116,7 @@ TupleHashJoinStep::TupleHashJoinStep(const JobInfo& jobInfo) : else allowDJS = false; - numCores = sysconf(_SC_NPROCESSORS_ONLN); + numCores = resourceManager->numCores(); if (numCores <= 0) numCores = 8; /* Debugging, rand() is used to simulate failures @@ -284,8 +284,10 @@ void TupleHashJoinStep::startSmallRunners(uint index) handle abort, out of memory, etc */ + /* To measure wall-time spent constructing the small-side tables... boost::posix_time::ptime end_time, start_time = boost::posix_time::microsec_clock::universal_time(); + */ stopMemTracking = false; uint64_t jobs[numCores]; @@ -324,12 +326,13 @@ void TupleHashJoinStep::startSmallRunners(uint index) while (more) more = smallDLs[index]->next(smallIts[index], &oneRG); } - //joiner->doneInserting(); + /* To measure wall-time spent constructing the small-side tables... end_time = boost::posix_time::microsec_clock::universal_time(); if (!(fSessionId & 0x80000000)) cout << "hash table construction time = " << end_time - start_time << " size = " << joiner->size() << endl; + */ extendedInfo += "\n"; From 627f8345ad1c5b53072b58dc5da0eacfaa7eb980 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 22 Nov 2019 14:36:34 +0000 Subject: [PATCH 13/35] MCOL-3564 Don't double-execute ProcMon It is possible for ProcMon can be left behind during a node shutdown. This patch makes sure it is killed and makes sure it is killed before a new one starts. --- oam/install_scripts/columnstore.in | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/oam/install_scripts/columnstore.in b/oam/install_scripts/columnstore.in index 339867d1e..858395063 100644 --- a/oam/install_scripts/columnstore.in +++ b/oam/install_scripts/columnstore.in @@ -56,6 +56,11 @@ start() { exit 0 fi + # Make sure ProcMon is down first + touch ${tmpDir}/StopColumnstore + pkill ProcMon + pkill ProcMgr + (mkdir -p $lockdir && touch $lockdir/columnstore) >/dev/null 2>&1 #checkInstallSetup @@ -70,7 +75,8 @@ start() { fi RETVAL=0 - echo "Starting MariaDB Columnstore Database Platform" + moduleName="$(cat /var/lib/columnstore/local/module)" + echo "Starting module $moduleName of MariaDB Columnstore Database Platform" rm -f ${tmpDir}/StopColumnstore exec columnstore_run.sh -l ${tmpDir} ProcMon > /dev/null 2>&1 & @@ -85,6 +91,8 @@ stop() { rm -f $lockdir/columnstore fuser -k 8604/tcp > /dev/null 2>&1 mysql-Columnstore stop > /dev/null 2>&1 + pkill ProcMon + pkill ProcMgr return $RETVAL } restart() { From 2e16fe674cea6112373401aa26fd0d9f4027eb0a Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 21 Nov 2019 15:32:29 +0000 Subject: [PATCH 14/35] 1. Implement bit_count() distributed function. 2. Handle hexadecimal string literals in buildReturnedColumn(). --- dbcon/mysql/ha_mcs_execplan.cpp | 9 +++++++ utils/funcexp/func_bitwise.cpp | 44 +++++++++++++++++++++++++++++++++ utils/funcexp/funcexp.cpp | 1 + utils/funcexp/functor_int.h | 17 +++++++++++++ 4 files changed, 71 insertions(+) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 07fdb25f5..4dd14cd4d 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3173,6 +3173,15 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp } case STRING_RESULT: { + // Special handling for 0xHHHH literals + const Type_handler *tph = item->type_handler(); + if (typeid(*tph) == typeid(Type_handler_hex_hybrid)) + { + Item_hex_hybrid *hip = reinterpret_cast(const_cast(item)); + rc = new ConstantColumn((int64_t)hip->val_int(), ConstantColumn::NUM); + break; + } + String val, *str = item->val_str(&val); string valStr; valStr.assign(str->ptr(), str->length()); diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 8b065e15f..028aedc1c 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -359,5 +359,49 @@ int64_t Func_bitxor::getIntVal(Row& row, } +// +// BIT COUNT +// + + +CalpontSystemCatalog::ColType Func_bit_count::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType ) +{ + return resultType; +} + +int64_t Func_bit_count::getIntVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& operationColType) +{ + if ( parm.size() != 1 ) + { + isNull = true; + return 0; + } + + uint64_t val = 0; + + if (!getUIntValFromParm(row, parm[0], val, isNull, fTimeZone)) + { + std::ostringstream oss; + oss << "bit_count: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + // Refer to Hacker's Delight Chapter 5 + // for the bit counting algo used here + val = val - ((val >> 1) & 0x5555555555555555); + val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333); + val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F; + val = val + (val >> 8); + val = val + (val >> 16); + val = val + (val >> 32); + + return (int64_t)(val & 0x000000000000007F); + +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index be50ca2cf..1c01458d7 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -86,6 +86,7 @@ FuncExp::FuncExp() fFuncMap["atan"] = new Func_atan(); fFuncMap["atan2"] = new Func_atan(); fFuncMap["between"] = new Func_between(); + fFuncMap["bit_count"] = new Func_bit_count(); fFuncMap["case_searched"] = new Func_searched_case(); fFuncMap["case_simple"] = new Func_simple_case(); fFuncMap["cast_as_signed"] = new Func_cast_signed(); //dlh diff --git a/utils/funcexp/functor_int.h b/utils/funcexp/functor_int.h index 78655a892..3e2f465cb 100644 --- a/utils/funcexp/functor_int.h +++ b/utils/funcexp/functor_int.h @@ -365,6 +365,23 @@ public: }; +/** @brief Func_bit_count class + */ +class Func_bit_count : public Func_Int +{ +public: + Func_bit_count() : Func_Int("bit_count") {} + virtual ~Func_bit_count() {} + + execplan::CalpontSystemCatalog::ColType operationType(FunctionParm& fp, execplan::CalpontSystemCatalog::ColType& resultType); + + int64_t getIntVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); +}; + + /** @brief Func_hour class */ class Func_hour : public Func_Int From 3a730a42221dbc9a7735a2e66cf398b3e9e50d7a Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 31 Oct 2019 14:53:27 -0500 Subject: [PATCH 15/35] MCOL-3563: fix compiler warnings / type errors. A previous commit made posix::read changed return value from bool to int. However some callers still expected bool and compared a bool<0 which is always false. --- storage-manager/src/CopyTask.cpp | 5 ++--- storage-manager/src/ListDirectoryTask.cpp | 10 +++++++--- storage-manager/src/ReadTask.cpp | 5 ++--- storage-manager/src/StatTask.cpp | 5 ++--- storage-manager/src/TruncateTask.cpp | 5 ++--- storage-manager/src/UnlinkTask.cpp | 5 ++--- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/storage-manager/src/CopyTask.cpp b/storage-manager/src/CopyTask.cpp index 7943d6cbc..b267b7ebb 100644 --- a/storage-manager/src/CopyTask.cpp +++ b/storage-manager/src/CopyTask.cpp @@ -43,7 +43,7 @@ CopyTask::~CopyTask() bool CopyTask::run() { - bool success; + int success; SMLogging* logger = SMLogging::get(); uint8_t buf[2048] = {0}; @@ -83,8 +83,7 @@ bool CopyTask::run() sm_response *resp = (sm_response *) buf; resp->returnCode = 0; - success = write(*resp, 0); - return success; + return write(*resp, 0); } } diff --git a/storage-manager/src/ListDirectoryTask.cpp b/storage-manager/src/ListDirectoryTask.cpp index c474868e9..e9e619b8a 100644 --- a/storage-manager/src/ListDirectoryTask.cpp +++ b/storage-manager/src/ListDirectoryTask.cpp @@ -37,7 +37,7 @@ ListDirectoryTask::~ListDirectoryTask() } #define check_error(msg, ret) \ - if (success<0) \ + if (!success) \ { \ handleError(msg, errno); \ return ret; \ @@ -85,8 +85,12 @@ bool ListDirectoryTask::run() return true; } - success = read(buf, getLength()); - check_error("ListDirectoryTask read", false); + err = read(buf, getLength()); + if (err<0) + { + handleError("ListDirectoryTask read", errno); + return false; + } listdir_cmd *cmd = (listdir_cmd *) buf; #ifdef SM_TRACE diff --git a/storage-manager/src/ReadTask.cpp b/storage-manager/src/ReadTask.cpp index 5ce3015d1..cae69bffd 100644 --- a/storage-manager/src/ReadTask.cpp +++ b/storage-manager/src/ReadTask.cpp @@ -54,7 +54,7 @@ bool ReadTask::run() return true; } - bool success; + int success; success = read(buf, getLength()); check_error("ReadTask read cmd", false); read_cmd *cmd = (read_cmd *) buf; @@ -103,8 +103,7 @@ bool ReadTask::run() } if (resp->returnCode >= 0) payloadLen = resp->returnCode; - success = write(*resp, payloadLen); - return success; + return write(*resp, payloadLen); } diff --git a/storage-manager/src/StatTask.cpp b/storage-manager/src/StatTask.cpp index 827189964..552b068c3 100644 --- a/storage-manager/src/StatTask.cpp +++ b/storage-manager/src/StatTask.cpp @@ -48,7 +48,7 @@ StatTask::~StatTask() bool StatTask::run() { SMLogging* logger = SMLogging::get(); - bool success; + int success; uint8_t buf[1024] = {0}; if (getLength() > 1023) { @@ -84,8 +84,7 @@ bool StatTask::run() payloadLen = 4; *((int32_t *) resp->payload) = errno; } - success = write(*resp, payloadLen); - return success; + return write(*resp, payloadLen); } } diff --git a/storage-manager/src/TruncateTask.cpp b/storage-manager/src/TruncateTask.cpp index e76975133..3bdc09a80 100644 --- a/storage-manager/src/TruncateTask.cpp +++ b/storage-manager/src/TruncateTask.cpp @@ -44,7 +44,7 @@ TruncateTask::~TruncateTask() bool TruncateTask::run() { SMLogging* logger = SMLogging::get(); - bool success; + int success; uint8_t buf[1024] = {0}; if (getLength() > 1023) { @@ -79,8 +79,7 @@ bool TruncateTask::run() sm_response *resp = (sm_response *) buf; resp->returnCode = 0; - success = write(*resp, 0); - return success; + return write(*resp, 0); } } diff --git a/storage-manager/src/UnlinkTask.cpp b/storage-manager/src/UnlinkTask.cpp index ccb777a47..9c441ebf5 100644 --- a/storage-manager/src/UnlinkTask.cpp +++ b/storage-manager/src/UnlinkTask.cpp @@ -45,7 +45,7 @@ UnlinkTask::~UnlinkTask() bool UnlinkTask::run() { SMLogging* logger = SMLogging::get(); - bool success; + int success; uint8_t buf[1024] = {0}; if (getLength() > 1023) { @@ -81,8 +81,7 @@ bool UnlinkTask::run() sm_response *resp = (sm_response *) buf; resp->returnCode = 0; - success = write(*resp, 0); - return success; + return write(*resp, 0); } } From 3e3e0ed89b79c3dbd90d63b86db315ecca87edde Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 14 Nov 2019 13:07:29 -0600 Subject: [PATCH 16/35] MCOL-3563: convert some cout to logging messages and default fd for ctrl socket. --- storage-manager/src/SessionManager.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/storage-manager/src/SessionManager.cpp b/storage-manager/src/SessionManager.cpp index 1c21d9229..5a93d9edd 100644 --- a/storage-manager/src/SessionManager.cpp +++ b/storage-manager/src/SessionManager.cpp @@ -51,6 +51,8 @@ namespace storagemanager SessionManager::SessionManager() { crp = ClientRequestProcessor::get(); + socketCtrl[0] = -1; + socketCtrl[1] = -1; } SessionManager::~SessionManager() @@ -164,7 +166,7 @@ int SessionManager::start() { //logger->log(LOG_DEBUG,"Error! revents = %d", fds[socketIncr].revents,); if (fds[socketIncr].fd == -1) - cout << "!= POLLIN, closing fd -1" << endl; + logger->log(LOG_DEBUG,"!= POLLIN, closing fd -1"); close(fds[socketIncr].fd); fds[socketIncr].fd = -1; continue; @@ -239,7 +241,7 @@ int SessionManager::start() if (fds[i].fd == socket) { if (socket == -1) - cout << "REMOVEFD told to remove fd -1" << endl; + logger->log(LOG_DEBUG,"REMOVEFD told to remove fd -1"); close(socket); fds[i].fd = -1; break; @@ -376,7 +378,7 @@ int SessionManager::start() if (closeConn) { if (fds[socketIncr].fd == -1) - cout << "closeConn closing fd -1" << endl; + logger->log(LOG_DEBUG,"closeConn closing fd -1"); close(fds[socketIncr].fd); fds[socketIncr].fd = -1; } From 8d6c2a33bf2b3a092cb507d96eefe18f29b98934 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 14 Nov 2019 13:08:55 -0600 Subject: [PATCH 17/35] MCOL-3563: Modify all *TASK tests to use processtask and test connection failures and short message error handling on disconnect. --- storage-manager/src/unit_tests.cpp | 297 +++++++++++++++++++++-------- 1 file changed, 219 insertions(+), 78 deletions(-) diff --git a/storage-manager/src/unit_tests.cpp b/storage-manager/src/unit_tests.cpp index f118b11a9..f66b77a1e 100644 --- a/storage-manager/src/unit_tests.cpp +++ b/storage-manager/src/unit_tests.cpp @@ -33,6 +33,7 @@ #include "S3Storage.h" #include "Utilities.h" #include "Synchronizer.h" +#include "ProcessTask.h" #include #include @@ -189,7 +190,7 @@ void makeConnection() t.join(); } -bool opentask(bool connectionTest=false) +bool opentask(bool connectionTest=false, bool errorHandle=false) { // going to rely on msgs being smaller than the buffer here int err=0; @@ -209,20 +210,39 @@ bool opentask(bool connectionTest=false) cout << "open file " << filename << endl; ::unlink(filename); - ssize_t result = ::write(sessionSock, cmd, hdr->payloadLen); - assert(result==(hdr->payloadLen)); - - OpenTask o(clientSock, hdr->payloadLen); - o.run(); + if (connectionTest) + hdr->payloadLen -= 2; + + size_t result = ::write(sessionSock, cmd, hdr->payloadLen); + assert(result==(hdr->payloadLen)); + + if (errorHandle) + clientSock = 9999999; + if (connectionTest) + hdr->payloadLen += 2; + + ProcessTask pt(clientSock, hdr->payloadLen); + boost::thread t(pt); + if (connectionTest) { - close(sessionSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else if (errorHandle) + { + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); } else { + t.join(); // read the response err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; @@ -237,13 +257,14 @@ bool opentask(bool connectionTest=false) assert(_stat->st_uid == getuid()); assert(_stat->st_gid == getgid()); assert(_stat->st_size == 0); - } - /* verify the file is there */ - string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path"); - assert(!metaPath.empty()); - metaPath += string("/" + prefix + "/" + testFile + ".meta"); + /* verify the file is there */ + string metaPath = Config::get()->getValue("ObjectStorage", "metadata_path"); + assert(!metaPath.empty()); + metaPath += string("/" + prefix + "/" + testFile + ".meta"); + + assert(boost::filesystem::exists(metaPath)); + } - assert(boost::filesystem::exists(metaPath)); cout << "opentask OK" << endl; return true; } @@ -486,7 +507,7 @@ bool appendtask() return true; } -void unlinktask(bool connectionTest=false) +void unlinktask(bool connectionTest=false, bool errorHandle=false) { int err=0; // make a meta file and delete it @@ -498,8 +519,10 @@ void unlinktask(bool connectionTest=false) IOCoordinator *ioc = IOCoordinator::get(); bf::path fullPathMeta = ioc->getMetadataPath()/(string(Metafilename) + ".meta"); bf::remove(fullPathMeta); - + MetadataFile meta(Metafilename); + meta.writeMetadata(); + assert(bf::exists(fullPathMeta)); uint8_t buf[1024]; @@ -509,19 +532,38 @@ void unlinktask(bool connectionTest=false) cmd->flen = strlen(filename); memcpy(&cmd->filename, filename, cmd->flen); - UnlinkTask u(clientSock, sizeof(unlink_cmd) + cmd->flen); + if (connectionTest) + cmd->flen -= 2; + size_t result = ::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen); assert(result==(sizeof(unlink_cmd) + cmd->flen)); - u.run(); - + + if (errorHandle) + clientSock = 9999999; + if (connectionTest) + cmd->flen += 2; + + ProcessTask pt(clientSock, sizeof(unlink_cmd) + cmd->flen); + boost::thread t(pt); + if (connectionTest) { - close(sessionSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else if (errorHandle) + { + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); } else { + t.join(); // read the response err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; @@ -530,11 +572,10 @@ void unlinktask(bool connectionTest=false) assert(resp->header.payloadLen == sizeof(ssize_t)); assert(resp->header.flags == 0); assert(resp->returnCode == 0); + // confirm it no longer exists + assert(!bf::exists(fullPathMeta)); } - // confirm it no longer exists - assert(!bf::exists(fullPathMeta)); - // delete it again, make sure we get an error message & reasonable error code // Interesting. boost::filesystem::remove() doesn't consider it an error if the file doesn't // exist. Need to look into the reasoning for that, and decide whether IOC @@ -545,7 +586,7 @@ void unlinktask(bool connectionTest=false) cmd->opcode = UNLINK; cmd->flen = strlen(filename); memcpy(&cmd->filename, filename, cmd->flen); - + UnlinkTask u2(clientSock, sizeof(unlink_cmd) + cmd->flen); ssize_t result = ::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen); assert(result==(sizeof(unlink_cmd) + cmd->flen)); @@ -567,7 +608,7 @@ void unlinktask(bool connectionTest=false) cout << "unlink task OK" << endl; } -bool stattask(bool connectionTest=false) +bool stattask(bool connectionTest=false, bool errorHandle=false) { int err=0; bf::path fullPath = homepath / prefix / "stattest1"; @@ -586,20 +627,39 @@ bool stattask(bool connectionTest=false) cmd->flen = filename.length(); strcpy((char *) cmd->filename, filename.c_str()); + if (connectionTest) + cmd->flen -= 2; + size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); assert(result==(sizeof(*cmd) + cmd->flen)); - StatTask s(clientSock, sizeof(*cmd) + cmd->flen); - s.run(); + if (errorHandle) + clientSock = 9999999; + if (connectionTest) + cmd->flen += 2; + + ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); + boost::thread t(pt); + if (connectionTest) { - close(sessionSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else if (errorHandle) + { + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); } else { + t.join(); // read the response err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; @@ -762,7 +822,7 @@ bool IOCTruncate() } -bool truncatetask(bool connectionTest=false) +bool truncatetask(bool connectionTest=false, bool errorHandle=false) { IOCoordinator *ioc = IOCoordinator::get(); Cache *cache = Cache::get(); @@ -786,20 +846,38 @@ bool truncatetask(bool connectionTest=false) cmd->flen = strlen(filename); strcpy((char *) cmd->filename, filename); + if (connectionTest) + cmd->flen -= 2; + size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); assert(result==(sizeof(*cmd) + cmd->flen)); - - TruncateTask t(clientSock, sizeof(*cmd) + cmd->flen); - t.run(); + if (errorHandle) + clientSock = 9999999; + if (connectionTest) + cmd->flen += 2; + + ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); + boost::thread t(pt); + if (connectionTest) { - close(sessionSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else if (errorHandle) + { + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); } else { + t.join(); // read the response err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; @@ -808,19 +886,18 @@ bool truncatetask(bool connectionTest=false) assert(resp->header.flags == 0); assert(resp->header.payloadLen == sizeof(ssize_t)); assert(resp->returnCode == 0); + // reload the metadata, check that it is 1000 bytes + meta = MetadataFile(Metafilename); + assert(meta.getLength() == 1000); } - // reload the metadata, check that it is 1000 bytes - meta = MetadataFile(Metafilename); - assert(meta.getLength() == 1000); - cache->reset(); ::unlink(metaFullName.c_str()); cout << "truncate task OK" << endl; return true; } -bool listdirtask(bool connectionTest=false) +bool listdirtask(bool connectionTest=false,bool errorHandle=false) { IOCoordinator *ioc = IOCoordinator::get(); const bf::path metaPath = ioc->getMetadataPath(); @@ -848,26 +925,46 @@ bool listdirtask(bool connectionTest=false) } uint8_t buf[8192]; + memset(buf,0,8192); listdir_cmd *cmd = (listdir_cmd *) buf; cmd->opcode = LIST_DIRECTORY; cmd->plen = strlen(relPath); memcpy(cmd->path, relPath, cmd->plen); - + + if (connectionTest) + cmd->plen -= 2; + size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->plen); assert(result==(sizeof(*cmd) + cmd->plen)); - ListDirectoryTask l(clientSock, sizeof(*cmd) + cmd->plen); - l.run(); + if (errorHandle) + clientSock = 9999999; + if (connectionTest) + cmd->plen += 2; + //ListDirectoryTask l(clientSock, sizeof(*cmd) + cmd->plen); + //l.run(); + ProcessTask pt(clientSock, sizeof(*cmd) + cmd->plen); + boost::thread t(pt); if (connectionTest) { + sleep(1); close(sessionSock); + close(clientSock); err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); assert(err == -1); + t.join(); + } + else if (errorHandle) + { + err = ::recv(sessionSock, buf, 8192, MSG_DONTWAIT); + assert(err == -1); + t.join(); } else { + t.join(); /* going to keep this simple. Don't run this in a big dir. */ /* maybe later I'll make a dir, put a file in it, and etc. For now run it in a small dir. */ err = ::recv(sessionSock, buf, 8192, MSG_DONTWAIT); @@ -895,30 +992,53 @@ bool listdirtask(bool connectionTest=false) } bf::remove_all(tmpPath); + + cout << "listdir task OK" << endl; + return true; } -void pingtask(bool connectionTest=false) +void pingtask(bool connectionTest=false, bool errorHandle=false) { int err=0; uint8_t buf[1024]; ping_cmd *cmd = (ping_cmd *) buf; cmd->opcode = PING; + size_t len = sizeof(*cmd); + + if (connectionTest) + len -= 2; + ssize_t result = ::write(sessionSock, cmd, sizeof(*cmd)); assert(result==(sizeof(*cmd))); - - PingTask p(clientSock, sizeof(*cmd)); - p.run(); + if (errorHandle) + clientSock = 9999999; + if (connectionTest) + len += 2; + + ProcessTask pt(clientSock, len); + boost::thread t(pt); + if (connectionTest) { - close(sessionSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else if (errorHandle) + { + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); } else { + t.join(); // read the response err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; @@ -932,7 +1052,7 @@ void pingtask(bool connectionTest=false) cout << "pingtask OK" << endl; } -bool copytask(bool connectionTest=false) +bool copytask(bool connectionTest=false, bool errorHandle=false) { /* make a file @@ -962,20 +1082,41 @@ bool copytask(bool connectionTest=false) strncpy(file2->filename, dest, file2->flen); uint len = (uint64_t) &file2->filename[file2->flen] - (uint64_t) buf; + + if (connectionTest) + len -= 2; + ssize_t result = ::write(sessionSock, buf, len); assert(result==len); - CopyTask c(clientSock, len); - c.run(); int err=0; + + if (errorHandle) + clientSock = 9999999; + if (connectionTest) + len += 2; + + ProcessTask pt(clientSock, len); + boost::thread t(pt); + if (connectionTest) { - close(sessionSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); + sleep(1); + close(sessionSock); + close(clientSock); + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); + } + else if (errorHandle) + { + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + assert(err == -1); + t.join(); } else { + t.join(); // read the response err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); sm_response *resp = (sm_response *) buf; @@ -984,13 +1125,11 @@ bool copytask(bool connectionTest=false) assert(resp->header.payloadLen == sizeof(ssize_t)); assert(resp->header.flags == 0); assert(resp->returnCode == 0); + // verify copytest2 is there + MetadataFile meta2(Metadest, MetadataFile::no_create_t(),true); + assert(meta2.exists()); } - - // verify copytest2 is there - MetadataFile meta2(Metadest, MetadataFile::no_create_t(),true); - assert(meta2.exists()); - bf::path metaPath = IOCoordinator::get()->getMetadataPath(); bf::remove(metaPath/(metaStrSrc + ".meta")); bf::remove(metaPath/(metaStrDest + ".meta")); @@ -1861,30 +2000,32 @@ int main(int argc, char* argv[]) //opentask(); opentask(true); - cout << "connecting" << endl; makeConnection(); - cout << "connected" << endl; + opentask(false,true); + makeConnection(); unlinktask(true); - cout << "connecting" << endl; makeConnection(); - cout << "connected" << endl; + unlinktask(false,true); + makeConnection(); stattask(true); - cout << "connecting" << endl; makeConnection(); - cout << "connected" << endl; + stattask(false,true); + makeConnection(); truncatetask(true); - cout << "connecting" << endl; makeConnection(); - cout << "connected" << endl; + truncatetask(false,true); + makeConnection(); listdirtask(true); - cout << "connecting" << endl; makeConnection(); - cout << "connected" << endl; + listdirtask(false,true); + makeConnection(); pingtask(true); - cout << "connecting" << endl; makeConnection(); - cout << "connected" << endl; + pingtask(false,true); + makeConnection(); copytask(true); + makeConnection(); + copytask(false,true); (Cache::get())->shutdown(); delete (Synchronizer::get()); From 3565d237edbcd4832e54b890965da59f56694d0d Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 22 Nov 2019 17:03:19 -0600 Subject: [PATCH 18/35] MCOL-3563: Fix OpenTask and PingTask for read returning int to a bool. --- storage-manager/src/OpenTask.cpp | 4 ++-- storage-manager/src/PingTask.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage-manager/src/OpenTask.cpp b/storage-manager/src/OpenTask.cpp index e94a8483d..19b3ae826 100644 --- a/storage-manager/src/OpenTask.cpp +++ b/storage-manager/src/OpenTask.cpp @@ -45,7 +45,7 @@ bool OpenTask::run() return the result */ SMLogging* logger = SMLogging::get(); - bool success; + int success; uint8_t buf[1024] = {0}; if (getLength() > 1023) @@ -55,7 +55,7 @@ bool OpenTask::run() } success = read(buf, getLength()); - if (!success) + if (success<0) { handleError("OpenTask read2", errno); return false; diff --git a/storage-manager/src/PingTask.cpp b/storage-manager/src/PingTask.cpp index 4adb748b7..73e460cea 100644 --- a/storage-manager/src/PingTask.cpp +++ b/storage-manager/src/PingTask.cpp @@ -43,8 +43,8 @@ bool PingTask::run() return true; } // consume the msg - bool success = read(&buf, getLength()); - if (!success) + int success = read(&buf, getLength()); + if (success<0) { handleError("PingTask", errno); return false; From a0ffc427ecec98a760ecde4afbb1604f3c08cc91 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 22 Nov 2019 18:36:53 -0600 Subject: [PATCH 19/35] MCOL-3563: remove the dumb errorHandle test. Misread what this was doing. described the connection test behavior. --- storage-manager/src/unit_tests.cpp | 144 +++++++++++++---------------- 1 file changed, 64 insertions(+), 80 deletions(-) diff --git a/storage-manager/src/unit_tests.cpp b/storage-manager/src/unit_tests.cpp index f66b77a1e..7a6c0baab 100644 --- a/storage-manager/src/unit_tests.cpp +++ b/storage-manager/src/unit_tests.cpp @@ -190,7 +190,7 @@ void makeConnection() t.join(); } -bool opentask(bool connectionTest=false, bool errorHandle=false) +bool opentask(bool connectionTest=false) { // going to rely on msgs being smaller than the buffer here int err=0; @@ -211,22 +211,28 @@ bool opentask(bool connectionTest=false, bool errorHandle=false) cout << "open file " << filename << endl; ::unlink(filename); + // set payload to be shorter than actual message lengh + // and send a shortened message. if (connectionTest) hdr->payloadLen -= 2; size_t result = ::write(sessionSock, cmd, hdr->payloadLen); assert(result==(hdr->payloadLen)); - if (errorHandle) - clientSock = 9999999; + // set payload to be correct length again if (connectionTest) hdr->payloadLen += 2; + // process task will look for the full length and + // will wait on the rest of the message. ProcessTask pt(clientSock, hdr->payloadLen); boost::thread t(pt); if (connectionTest) { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path sleep(1); close(sessionSock); close(clientSock); @@ -234,12 +240,6 @@ bool opentask(bool connectionTest=false, bool errorHandle=false) assert(err == -1); t.join(); } - else if (errorHandle) - { - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } else { t.join(); @@ -507,7 +507,7 @@ bool appendtask() return true; } -void unlinktask(bool connectionTest=false, bool errorHandle=false) +void unlinktask(bool connectionTest=false) { int err=0; // make a meta file and delete it @@ -532,22 +532,28 @@ void unlinktask(bool connectionTest=false, bool errorHandle=false) cmd->flen = strlen(filename); memcpy(&cmd->filename, filename, cmd->flen); + // set payload to be shorter than actual message lengh + // and send a shortened message. if (connectionTest) cmd->flen -= 2; size_t result = ::write(sessionSock, cmd, sizeof(unlink_cmd) + cmd->flen); assert(result==(sizeof(unlink_cmd) + cmd->flen)); - if (errorHandle) - clientSock = 9999999; + // set payload to be correct length again if (connectionTest) cmd->flen += 2; + // process task will look for the full length and + // will wait on the rest of the message. ProcessTask pt(clientSock, sizeof(unlink_cmd) + cmd->flen); boost::thread t(pt); if (connectionTest) { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path sleep(1); close(sessionSock); close(clientSock); @@ -555,12 +561,6 @@ void unlinktask(bool connectionTest=false, bool errorHandle=false) assert(err == -1); t.join(); } - else if (errorHandle) - { - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } else { t.join(); @@ -608,7 +608,7 @@ void unlinktask(bool connectionTest=false, bool errorHandle=false) cout << "unlink task OK" << endl; } -bool stattask(bool connectionTest=false, bool errorHandle=false) +bool stattask(bool connectionTest=false) { int err=0; bf::path fullPath = homepath / prefix / "stattest1"; @@ -627,23 +627,28 @@ bool stattask(bool connectionTest=false, bool errorHandle=false) cmd->flen = filename.length(); strcpy((char *) cmd->filename, filename.c_str()); + // set payload to be shorter than actual message lengh + // and send a shortened message. if (connectionTest) cmd->flen -= 2; size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); assert(result==(sizeof(*cmd) + cmd->flen)); - - if (errorHandle) - clientSock = 9999999; + // set payload to be correct length again if (connectionTest) cmd->flen += 2; + // process task will look for the full length and + // will wait on the rest of the message. ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); boost::thread t(pt); if (connectionTest) { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path sleep(1); close(sessionSock); close(clientSock); @@ -651,12 +656,6 @@ bool stattask(bool connectionTest=false, bool errorHandle=false) assert(err == -1); t.join(); } - else if (errorHandle) - { - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } else { t.join(); @@ -822,7 +821,7 @@ bool IOCTruncate() } -bool truncatetask(bool connectionTest=false, bool errorHandle=false) +bool truncatetask(bool connectionTest=false) { IOCoordinator *ioc = IOCoordinator::get(); Cache *cache = Cache::get(); @@ -846,22 +845,28 @@ bool truncatetask(bool connectionTest=false, bool errorHandle=false) cmd->flen = strlen(filename); strcpy((char *) cmd->filename, filename); + // set payload to be shorter than actual message lengh + // and send a shortened message. if (connectionTest) cmd->flen -= 2; size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->flen); assert(result==(sizeof(*cmd) + cmd->flen)); - if (errorHandle) - clientSock = 9999999; + // set payload to be correct length again if (connectionTest) cmd->flen += 2; + // process task will look for the full length and + // will wait on the rest of the message. ProcessTask pt(clientSock, sizeof(*cmd) + cmd->flen); boost::thread t(pt); if (connectionTest) { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path sleep(1); close(sessionSock); close(clientSock); @@ -869,12 +874,6 @@ bool truncatetask(bool connectionTest=false, bool errorHandle=false) assert(err == -1); t.join(); } - else if (errorHandle) - { - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } else { t.join(); @@ -897,7 +896,7 @@ bool truncatetask(bool connectionTest=false, bool errorHandle=false) return true; } -bool listdirtask(bool connectionTest=false,bool errorHandle=false) +bool listdirtask(bool connectionTest=false) { IOCoordinator *ioc = IOCoordinator::get(); const bf::path metaPath = ioc->getMetadataPath(); @@ -932,23 +931,28 @@ bool listdirtask(bool connectionTest=false,bool errorHandle=false) cmd->plen = strlen(relPath); memcpy(cmd->path, relPath, cmd->plen); + // set payload to be shorter than actual message lengh + // and send a shortened message. if (connectionTest) cmd->plen -= 2; size_t result = ::write(sessionSock, cmd, sizeof(*cmd) + cmd->plen); assert(result==(sizeof(*cmd) + cmd->plen)); - if (errorHandle) - clientSock = 9999999; + // set payload to be correct length again if (connectionTest) cmd->plen += 2; - //ListDirectoryTask l(clientSock, sizeof(*cmd) + cmd->plen); - //l.run(); + + // process task will look for the full length and + // will wait on the rest of the message. ProcessTask pt(clientSock, sizeof(*cmd) + cmd->plen); boost::thread t(pt); if (connectionTest) { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path sleep(1); close(sessionSock); close(clientSock); @@ -956,12 +960,6 @@ bool listdirtask(bool connectionTest=false,bool errorHandle=false) assert(err == -1); t.join(); } - else if (errorHandle) - { - err = ::recv(sessionSock, buf, 8192, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } else { t.join(); @@ -998,7 +996,7 @@ bool listdirtask(bool connectionTest=false,bool errorHandle=false) return true; } -void pingtask(bool connectionTest=false, bool errorHandle=false) +void pingtask(bool connectionTest=false) { int err=0; uint8_t buf[1024]; @@ -1007,22 +1005,28 @@ void pingtask(bool connectionTest=false, bool errorHandle=false) size_t len = sizeof(*cmd); + // set payload to be shorter than actual message lengh + // and send a shortened message. if (connectionTest) len -= 2; ssize_t result = ::write(sessionSock, cmd, sizeof(*cmd)); assert(result==(sizeof(*cmd))); - if (errorHandle) - clientSock = 9999999; + // set payload to be correct length again if (connectionTest) len += 2; + // process task will look for the full length and + // will wait on the rest of the message. ProcessTask pt(clientSock, len); boost::thread t(pt); if (connectionTest) { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path sleep(1); close(sessionSock); close(clientSock); @@ -1030,12 +1034,6 @@ void pingtask(bool connectionTest=false, bool errorHandle=false) assert(err == -1); t.join(); } - else if (errorHandle) - { - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } else { t.join(); @@ -1052,7 +1050,7 @@ void pingtask(bool connectionTest=false, bool errorHandle=false) cout << "pingtask OK" << endl; } -bool copytask(bool connectionTest=false, bool errorHandle=false) +bool copytask(bool connectionTest=false) { /* make a file @@ -1083,6 +1081,8 @@ bool copytask(bool connectionTest=false, bool errorHandle=false) uint len = (uint64_t) &file2->filename[file2->flen] - (uint64_t) buf; + // set payload to be shorter than actual message lengh + // and send a shortened message. if (connectionTest) len -= 2; @@ -1091,16 +1091,20 @@ bool copytask(bool connectionTest=false, bool errorHandle=false) int err=0; - if (errorHandle) - clientSock = 9999999; + // set payload to be correct length again if (connectionTest) len += 2; + // process task will look for the full length and + // will wait on the rest of the message. ProcessTask pt(clientSock, len); boost::thread t(pt); if (connectionTest) { + // make sure the thread is waiting for the rest of the data + // then kill the connection. This will trigger the task thread + // to exit on an error handling path sleep(1); close(sessionSock); close(clientSock); @@ -1108,12 +1112,6 @@ bool copytask(bool connectionTest=false, bool errorHandle=false) assert(err == -1); t.join(); } - else if (errorHandle) - { - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } else { t.join(); @@ -2001,31 +1999,17 @@ int main(int argc, char* argv[]) //opentask(); opentask(true); makeConnection(); - opentask(false,true); - makeConnection(); unlinktask(true); makeConnection(); - unlinktask(false,true); - makeConnection(); stattask(true); makeConnection(); - stattask(false,true); - makeConnection(); truncatetask(true); makeConnection(); - truncatetask(false,true); - makeConnection(); listdirtask(true); makeConnection(); - listdirtask(false,true); - makeConnection(); pingtask(true); makeConnection(); - pingtask(false,true); - makeConnection(); copytask(true); - makeConnection(); - copytask(false,true); (Cache::get())->shutdown(); delete (Synchronizer::get()); From 0ae4969a4d00d0ebe47b46d4c0d73ef7bcbba2ba Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 22 Nov 2019 18:42:49 -0600 Subject: [PATCH 20/35] MCOL-3563: new SyncTask was added also had this issue. --- storage-manager/src/SyncTask.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-manager/src/SyncTask.cpp b/storage-manager/src/SyncTask.cpp index a962f9151..4415822e1 100644 --- a/storage-manager/src/SyncTask.cpp +++ b/storage-manager/src/SyncTask.cpp @@ -43,8 +43,8 @@ bool SyncTask::run() return true; } // consume the msg - bool success = read(&buf, getLength()); - if (!success) + int success = read(&buf, getLength()); + if (success<0) { handleError("SyncTask", errno); return false; From 3fabf01e939e599b835f361b153d7ecd5c330595 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 13 Nov 2019 18:56:18 +0300 Subject: [PATCH 21/35] MCOL-3593 Disabled full optimizer run and enabled copy-pasted simplify_joins. Disabled 4th if block in buildOuterJoin to handle non-optimized MDB query structures. Broke getSelectPlan into pieces: processFrom, processWhere. MCOL-3593 UNION processing depends on two flags isUnion that comes as arg of getSelectPlan and unionSel that is a local variable in getSelectPlan. Modularization of getSelectPlan broke the mechanizm. This patch is supposed to partially fix it. MCOL-3593 Removed unused if condition from buildOuterJoin that allows unsupported construct subquery in ON expression. Fixed an improper if condition that ignors tableMap entries w/o condition in external_lock thus external_lock doesn't clean up when the query finishes. Fixed wrong logging for queries processed in tableMode. Now rnd_init properly sends queryText down to ExeMgr to be logged. MCOL-3593 Unused attribute FromSubQuery::fFromSub was removed. getSelectPlan has been modularized into: setExecutionParams, processFrom, processWhere. SELECT, HAVING, GROUP BY, ORDER BY still lives in getSelectPlan. Copied optimization function simplify_joins_ into our pushdown code to provide the plugin code with some rewrites from MDB it expects. The columnstore_processing_handlers_fallback session variable has been removed thus CS can't fallback from SH to partial execution paths, e.g. DH, GBH or plugin API. MCOL-3602 Moved MDB optimizer rewrites into a separate file. Add SELECT_LEX::optimize_unflattened_subqueries() call to fix IN into EXISTS rewrite for semi-JOINs with subqueries. disable_indices_for_CEJ() add index related hints to disable index access methods in Cross Engine Joins. create_SH() now flattens JOIN that has both physical tables and views. This fixes most of views related tests in the regression. --- dbcon/mysql/CMakeLists.txt | 1 + dbcon/mysql/ha_from_sub.cpp | 7 +- dbcon/mysql/ha_mcs.h | 1 + dbcon/mysql/ha_mcs_execplan.cpp | 321 +++++++++++++++------------- dbcon/mysql/ha_mcs_impl.cpp | 63 ++---- dbcon/mysql/ha_mcs_impl_if.h | 1 - dbcon/mysql/ha_mcs_opt_rewrites.cpp | 245 +++++++++++++++++++++ dbcon/mysql/ha_mcs_opt_rewrites.h | 26 +++ dbcon/mysql/ha_mcs_pushdown.cpp | 76 ++++--- dbcon/mysql/ha_mcs_pushdown.h | 2 + dbcon/mysql/ha_mcs_sysvars.cpp | 21 +- dbcon/mysql/ha_mcs_sysvars.h | 3 - dbcon/mysql/ha_subquery.h | 1 - dbcon/mysql/sm.cpp | 2 +- 14 files changed, 518 insertions(+), 252 deletions(-) create mode 100644 dbcon/mysql/ha_mcs_opt_rewrites.cpp create mode 100644 dbcon/mysql/ha_mcs_opt_rewrites.h diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 965b138ed..d99c75d9f 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -8,6 +8,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} SET ( libcalmysql_SRCS ha_mcs_sysvars.cpp ha_mcs_client_udfs.cpp + ha_mcs_opt_rewrites.cpp ha_mcs_pushdown.cpp ha_mcs.cpp ha_mcs_impl.cpp diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 014ded75e..12b2b24de 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -326,8 +326,7 @@ FromSubQuery::FromSubQuery(gp_walk_info& gwip, SELECT_LEX* sub, bool isPushdownHandler) : SubQuery(gwip), - fFromSub(sub), - fPushdownHand(isPushdownHandler) + fFromSub(sub) {} FromSubQuery::~FromSubQuery() @@ -349,9 +348,7 @@ SCSEP FromSubQuery::transform() csep->derivedTbAlias(fAlias); // always lower case csep->derivedTbView(fGwip.viewName.alias); - // DRRTUY isUnion - false. fPushdownHand could be safely set to true - // b/c only pushdowns get here. - if (getSelectPlan(gwi, *fFromSub, csep, false, fPushdownHand) != 0) + if (getSelectPlan(gwi, *fFromSub, csep, false, true) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index c2a9d6532..1a06b7449 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -1,5 +1,6 @@ /* Copyright (C) 2014 InfiniDB, Inc. Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2019 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 07fdb25f5..db2bb550d 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -16,11 +16,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -/* - * $Id: ha_mcs_execplan.cpp 9749 2013-08-15 04:00:39Z zzhu $ - */ - -/** @file */ //#define DEBUG_WALK_COND #include #include @@ -61,7 +56,6 @@ using namespace logging; #include "ha_mcs_impl_if.h" #include "ha_mcs_sysvars.h" #include "ha_subquery.h" -//#include "ha_view.h" using namespace cal_impl_if; #include "calpontselectexecutionplan.h" @@ -132,7 +126,6 @@ public: gp_walk_info* fgwip; }; -//#define OUTER_JOIN_DEBUG namespace { string lower(string str) @@ -1321,6 +1314,7 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) if (table_ptr->outer_join && table_ptr->on_expr) { + // inner tables block Item_cond* expr = reinterpret_cast(table_ptr->on_expr); gwi_outer.innerTables.insert(tan); @@ -1392,8 +1386,10 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) #endif expr->traverse_cond(gp_walk, &gwi_outer, Item::POSTFIX); } +// *DRRTUY This part is to be removed in 1.4.3 +#if 0 // @bug 2849 - else if (table_ptr->embedding && table_ptr->embedding->nested_join) + /*else if (table_ptr->embedding && table_ptr->embedding->nested_join) { // if this is dervied table process phase, mysql may have not developed the plan // completely. Return and let it finish. It will come to rnd_init again. @@ -1434,8 +1430,8 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) } } } - } - + } */ +#endif // Error out subquery in outer join on filter for now if (gwi_outer.hasSubSelect) { @@ -1444,9 +1440,8 @@ uint32_t buildOuterJoin(gp_walk_info& gwi, SELECT_LEX& select_lex) setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText); return -1; } - // build outerjoinon filter - ParseTree* filters = NULL, *ptp = NULL, /**rhs = NULL*/*lhs = NULL; + ParseTree* filters = NULL, *ptp = NULL, *lhs = NULL; while (!gwi_outer.ptWorkStack.empty()) { @@ -6064,69 +6059,15 @@ bool isMCSTable(TABLE* table_ptr) return false; } -int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, - SCSEP& csep, - bool isUnion, - bool isPushdownHand) +/*@brief set some runtime params to run the query */ +/*********************************************************** + * DESCRIPTION: + * This function just sets a number of runtime params that + * limits resource consumed. + ***********************************************************/ +void setExecutionParams(gp_walk_info &gwi, SCSEP &csep) { -#ifdef DEBUG_WALK_COND - cerr << "getSelectPlan()" << endl; -#endif - - // by pass the derived table resolve phase of mysql - if ( !isPushdownHand && - !(((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ) ) && gwi.thd->derived_tables_processing) - { - // MCOL-2178 isUnion member only assigned, never used - //MIGR::infinidb_vtable.isUnion = false; - return -1; - } - - // rollup is currently not supported - if (select_lex.olap == ROLLUP_TYPE) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - gwi.internalDecimalScale = (get_use_decimal_scale(gwi.thd) ? get_decimal_scale(gwi.thd) : -1); - - gwi.subSelectType = csep->subType(); - - JOIN* join = select_lex.join; - Item_cond* icp = 0; - - if (join != 0) - icp = reinterpret_cast(join->conds); - - // if icp is null, try to find the where clause other where - if (!join && gwi.thd->lex->derived_tables) - { - if (select_lex.prep_where) - icp = (Item_cond*)(select_lex.prep_where); - else if (select_lex.where) - icp = (Item_cond*)(select_lex.where); - } - else if (!join && ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || - ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || - ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ))) - { - icp = reinterpret_cast(select_lex.where); - } - - uint32_t sessionID = csep->sessionID(); - gwi.sessionid = sessionID; - boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); - csc->identity(CalpontSystemCatalog::FE); - csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - gwi.csc = csc; - // @bug 2123. Override large table estimate if infinidb_ordered hint was used. // @bug 2404. Always override if the infinidb_ordered_only variable is turned on. if (get_ordered_only(gwi.thd)) @@ -6148,13 +6089,30 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, csep->umMemLimit(numeric_limits::max()); else csep->umMemLimit(get_um_mem_limit(gwi.thd) * 1024ULL * 1024); +} +/*@brief Process FROM part of the query or sub-query */ +/*********************************************************** + * DESCRIPTION: + * This function processes elements of List in + * FROM part of the query. + * isUnion tells that CS processes FROM taken from UNION UNIT. + * The notion is described in MDB code. + * on_expr_list ON expressions used in OUTER JOINs. These are + * later used in processWhere() + * RETURNS + * error id as an int + ***********************************************************/ +int processFrom(bool &isUnion, + SELECT_LEX &select_lex, + gp_walk_info &gwi, + SCSEP &csep, + List &on_expr_list) +{ // populate table map and trigger syscolumn cache for all the tables (@bug 1637). // all tables on FROM list must have at least one col in colmap TABLE_LIST* table_ptr = select_lex.get_table_list(); - CalpontSelectExecutionPlan::SelectList derivedTbList; -// DEBUG #ifdef DEBUG_WALK_COND List_iterator sj_list_it(select_lex.sj_nests); TABLE_LIST* sj_nest; @@ -6163,12 +6121,8 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { cerr << sj_nest->db.str << "." << sj_nest->table_name.str << endl; } - #endif - // @bug 1796. Remember table order on the FROM list. - gwi.clauseType = FROM; - try { for (; table_ptr; table_ptr = table_ptr->next_local) @@ -6182,17 +6136,20 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); return ER_CHECK_NOT_IMPLEMENTED; } + + // Save on_expr to use it for WHERE processing + if (!table_ptr->outer_join && table_ptr->on_expr) + { + on_expr_list.push_back(table_ptr->on_expr); + } string viewName = getViewName(table_ptr); // @todo process from subquery if (table_ptr->derived) { - String str; - (table_ptr->derived->first_select())->print(gwi.thd, &str, QT_ORDINARY); - SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery fromSub(gwi, select_cursor, isPushdownHand); + FromSubQuery fromSub(gwi, select_cursor, true); string alias(table_ptr->alias.str); fromSub.alias(lower(alias)); @@ -6203,7 +6160,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, if (!plan) { setError(gwi.thd, ER_INTERNAL_ERROR, fromSub.gwip().parseErrorText, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); return ER_INTERNAL_ERROR; } @@ -6229,7 +6186,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // trigger system catalog cache if (columnStore) - csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str), true); + gwi.csc->columnRIDs(make_table(table_ptr->db.str, table_ptr->table_name.str), true); string table_name = table_ptr->table_name.str; @@ -6256,7 +6213,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, catch (IDBExcept& ie) { setError(gwi.thd, ER_INTERNAL_ERROR, ie.what(), gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); // @bug 3852. set error status for gwi. gwi.fatalParseError = true; gwi.parseErrorText = ie.what(); @@ -6269,14 +6226,14 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, gwi.fatalParseError = true; gwi.parseErrorText = emsg; setError(gwi.thd, ER_INTERNAL_ERROR, emsg, gwi); - CalpontSystemCatalog::removeCalpontSystemCatalog(sessionID); + CalpontSystemCatalog::removeCalpontSystemCatalog(gwi.sessionid); return ER_INTERNAL_ERROR; } csep->tableList(gwi.tbList); + // Send this recursively to getSelectPlan bool unionSel = false; - // UNION master unit check // Existed pushdown handlers won't get in this scope // except UNION pushdown that is to come. @@ -6299,25 +6256,12 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, plan->traceFlags(csep->traceFlags()); plan->data(csep->data()); - // @bug 3853. When one or more sides or union queries contain derived tables, - // sl->join->zero_result_cause is not trustable. Since we've already handled - // constant filter now (0/1), we can relax the following checking. - // @bug 2547. ignore union unit of zero result set case -// if (sl->join) -// { -// sl->join->optimize(); - // @bug 3067. not clear MySQL's behavior. when in subquery, this variable - // is not trustable. -// if (sl->join->zero_result_cause && !gwi.subQuery) -// continue; -// } - // gwi for the union unit gp_walk_info union_gwi; union_gwi.thd = gwi.thd; uint32_t err = 0; - if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel, isPushdownHand)) != 0) + if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel, true)) != 0) return err; unionVec.push_back(SCEP(plan)); @@ -6325,63 +6269,67 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // distinct union num if (sl == select_lex.master_unit()->union_distinct) distUnionNum = unionVec.size(); - - /*#ifdef DEBUG_WALK_COND - IDEBUG( cerr << ">>>> UNION DEBUG" << endl ); - JOIN* join = sl->join; - Item_cond* icp = 0; - if (join != 0) - icp = reinterpret_cast(join->conds); - if (icp) - icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - IDEBUG ( cerr << *plan << endl ); - IDEBUG ( cerr << "<<<unionVec(unionVec); csep->distinctUnionNum(distUnionNum); } - gwi.clauseType = WHERE; + + return 0; +} + +/*@brief Process WHERE part of the query or sub-query */ +/*********************************************************** + * DESCRIPTION: + * This function processes conditions from either JOIN->conds + * or SELECT_LEX->where|prep_where + * on_expr_list ON expressions used in OUTER JOINs. These are + * populated used in processFrom() + * RETURNS + * error id as an int + ***********************************************************/ +int processWhere(SELECT_LEX &select_lex, + gp_walk_info &gwi, + SCSEP &csep, + List &on_expr_list) +{ + JOIN* join = select_lex.join; + Item_cond* icp = 0; + + if (join != 0) + icp = reinterpret_cast(join->conds); + + // if icp is null, try to find the where clause other where + if (!join && gwi.thd->lex->derived_tables) + { + if (select_lex.prep_where) + icp = (Item_cond*)(select_lex.prep_where); + else if (select_lex.where) + icp = (Item_cond*)(select_lex.where); + } + else if (!join && ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE ) || + ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI ) || + ((gwi.thd->lex)->sql_command == SQLCOM_DELETE_MULTI ))) + { + icp = reinterpret_cast(select_lex.where); + } if (icp) { -// MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". -//#if MYSQL_VERSION_ID < 50172 + // MariaDB bug 624 - without the fix_fields call, delete with join may error with "No query step". //@bug 3039. fix fields for constants if (!icp->is_fixed()) { icp->fix_fields(gwi.thd, (Item**)&icp); } -//#endif gwi.fatalParseError = false; #ifdef DEBUG_WALK_COND - cerr << "------------------ WHERE -----------------------" << endl; + std::cerr << "------------------ WHERE -----------------------" << std::endl; icp->traverse_cond(debug_walk, &gwi, Item::POSTFIX); - if (join && join->cond_equal) - { - List_iterator li(join->cond_equal->current_level); - Item_equal *cur_item_eq; - while ((cur_item_eq= li++)) - { - // DRRTUY TODO replace the block with - //cur_item_eq->traverse_cond(debug_walk, gwip, Item::POSTFIX); - std::cerr << "item_equal("; - Item *item; - Item_equal_fields_iterator it(*cur_item_eq); - while ((item= it++)) - { - std::ostringstream ostream; - std::ostringstream& osr = ostream; - getColNameFromItem(osr, item); - std::cerr << osr.str() << ","; - } - std::cerr << ")" << std::endl; - } - } - cerr << "------------------------------------------------\n" << endl; + std::cerr << "------------------------------------------------\n" << std::endl; #endif icp->traverse_cond(gp_walk, &gwi, Item::POSTFIX); @@ -6410,6 +6358,26 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, (dynamic_cast(gwi.rcWorkStack.top()))->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); } +#ifdef DEBUG_WALK_COND + std::cerr << "------------------ ON_EXPR -----------------------" << endl; +#endif + // MCOL-3593 MDB now doesn't rewrite and/or consolidate ON and WHERE expressions + // and CS handles INNER ON expressions here. + if (!on_expr_list.is_empty()) + { + List_iterator on_expr_it(on_expr_list); + Item_cond *on_expr = NULL; + while((on_expr = reinterpret_cast(on_expr_it++))) + { + on_expr->traverse_cond(gp_walk, &gwi, Item::POSTFIX); +#ifdef DEBUG_WALK_COND + on_expr->traverse_cond(debug_walk, &gwi, Item::POSTFIX); +#endif + } + } +#ifdef DEBUG_WALK_COND + std::cerr << "-------------------------------------------------\n" << std::endl; +#endif // ZZ - the followinig debug shows the structure of nested outer join. should // use a recursive function. @@ -6500,7 +6468,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // ptWorkStack empty, the item is in rcWorkStack. // MySQL 5.6 (MariaDB?). when icp is null and zero_result_cause is set, a constant 0 // is pushed to rcWorkStack. - if (/*icp && */gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) + if (gwi.ptWorkStack.empty() && !gwi.rcWorkStack.empty()) { filters = new ParseTree(gwi.rcWorkStack.top()); gwi.rcWorkStack.pop(); @@ -6515,11 +6483,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, break; ptp = new ParseTree(new LogicOperator("and")); - //ptp->left(filters); ptp->right(filters); lhs = gwi.ptWorkStack.top(); gwi.ptWorkStack.pop(); - //ptp->right(rhs); ptp->left(lhs); gwi.ptWorkStack.push(ptp); } @@ -6532,6 +6498,68 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, filters->drawTree(aTmpDir); } + return 0; +} + +/*@brief Translates SELECT_LEX into CSEP */ +/*********************************************************** + * DESCRIPTION: + * This function takes SELECT_LEX and tries to produce + * a corresponding CSEP out of it. It is made of parts that + * process parts of the query, e.g. FROM, WHERE, SELECT, + * HAVING, GROUP BY, ORDER BY. FROM and WHERE are processed + * by processFrom(), processWhere(). CS calls getSelectPlan() + * recursively to process subqueries. + * ARGS + * isUnion if true CS processes UNION unit now + * isPushdownHand legacy to be removed + * RETURNS + * error id as an int + ***********************************************************/ +int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, + SCSEP& csep, + bool isUnion, + bool isPushdownHand) +{ +#ifdef DEBUG_WALK_COND + cerr << "getSelectPlan()" << endl; +#endif + int rc = 0; + // rollup is currently not supported + if (select_lex.olap == ROLLUP_TYPE) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ROLLUP_NOT_SUPPORT); + setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + setExecutionParams(gwi, csep); + + gwi.subSelectType = csep->subType(); + uint32_t sessionID = csep->sessionID(); + gwi.sessionid = sessionID; + boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); + csc->identity(CalpontSystemCatalog::FE); + csep->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + gwi.csc = csc; + + CalpontSelectExecutionPlan::SelectList derivedTbList; + // @bug 1796. Remember table order on the FROM list. + gwi.clauseType = FROM; + List on_expr_list; + if ((rc = processFrom(isUnion, select_lex, gwi, csep, on_expr_list))) + { + return rc; + } + bool unionSel = (!isUnion && select_lex.master_unit()->is_unit_op()) ? true : false; + + gwi.clauseType = WHERE; + if ((rc = processWhere(select_lex, gwi, csep, on_expr_list))) + { + return rc; + } + gwi.clauseType = SELECT; #ifdef DEBUG_WALK_COND { @@ -7084,7 +7112,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, for (uint32_t i = 0; i < funcFieldVec.size(); i++) { - //SimpleColumn *sc = new SimpleColumn(funcFieldVec[i]->db_name, bestTableName(funcFieldVec[i])/*funcFieldVec[i]->table_name*/, funcFieldVec[i]->field_name, sessionID); SimpleColumn* sc = buildSimpleColumn(funcFieldVec[i], gwi); if (!sc || gwi.fatalParseError) @@ -7110,7 +7137,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, String str; funcFieldVec[i]->print(&str, QT_ORDINARY); sc->alias(string(str.c_ptr())); - //sc->tableAlias(funcFieldVec[i]->table_name); sc->tableAlias(sc->tableAlias()); SRCP srcp(sc); uint32_t j = 0; @@ -7867,8 +7893,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { uint32_t limitOffset = 0; - if (join) + if (select_lex.join) { + JOIN* join = select_lex.join; #if MYSQL_VERSION_ID >= 50172 // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 57dbe8bdb..fef506c6f 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -413,6 +413,23 @@ int vbin2hex(const uint8_t* p, const unsigned l, char* o) return 0; } +// Table Map is used by both cond_push and table mode processing +// Entries made by cond_push don't have csep though. +// When +bool onlyOneTableinTM(cal_impl_if::cal_connection_info* ci) +{ + size_t counter = 0; + for (auto &tableMapEntry: ci->tableMap) + { + if (tableMapEntry.second.csep) + counter++; + if (counter >= 1) + return false; + } + + return true; +} + int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) { int rc = HA_ERR_END_OF_FILE; @@ -2441,7 +2458,9 @@ int ha_mcs_impl_rnd_init(TABLE* table) csep = ti.csep; // for ExeMgr logging sqltext. only log once for the query although multi plans may be sent - if (ci->tableMap.size() == 1) + // CS adds the ti into TM in the end of rnd_init thus we log the SQL + // only once when there is no ti with csep. + if (onlyOneTableinTM(ci)) { ti.csep->data(idb_mysql_query_str(thd)); } @@ -3251,7 +3270,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) aCmdLine = aCmdLine + table->s->db.str + " " + table->s->table_name.str ; - //cout << "aCmdLine = " << aCmdLine << endl; std::istringstream ss(aCmdLine); std::string arg; std::vector v2(20, ""); @@ -3278,19 +3296,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) saAttr.lpSecurityDescriptor = NULL; HANDLE handleList[2]; const char* pSectionMsg; - // Create a pipe for the child process's STDOUT. -#if 0 // We don't need stdout to come back right now. - pSectionMsg = "Create Stdout"; - bSuccess = CreatePipe(&ci->cpimport_stdout_Rd, &ci->cpimport_stdout_Wr, &saAttr, 0); - - // Ensure the read handle to the pipe for STDIN is not inherited. - if (bSuccess) - { - pSectionMsg = "SetHandleInformation(stdout)"; - bSuccess = SetHandleInformation(ci->cpimport_stdout_Rd, HANDLE_FLAG_INHERIT, 0); - } - -#endif bSuccess = true; // Create a pipe for the child process's STDIN. @@ -3340,10 +3345,8 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) pSectionMsg = "UpdateProcThreadAttribute"; bInitialized = true; handleList[0] = ci->cpimport_stdin_Rd; -// handleList[1] = ci->cpimport_stdout_Wr; bSuccess = UpdateProcThreadAttribute(lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, -// handleList, 2*sizeof(HANDLE), NULL, NULL); handleList, sizeof(HANDLE), NULL, NULL); } @@ -3365,8 +3368,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) siStartInfo.lpAttributeList = lpAttributeList; siStartInfo.StartupInfo.hStdError = NULL; siStartInfo.StartupInfo.hStdOutput = NULL; -// siStartInfo.StartupInfo.hStdError = ci->cpimport_stdout_Wr; -// siStartInfo.StartupInfo.hStdOutput = ci->cpimport_stdout_Wr; siStartInfo.StartupInfo.hStdInput = ci->cpimport_stdin_Rd; siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; // Create the child process. @@ -3485,14 +3486,11 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) { ci->filePtr = fdopen(ci->fdt[1], "w"); ci->cpimport_pid = aChPid; // This is the child PID - //cout << "Child PID is " << aChPid << endl; close(ci->fdt[0]); //close the READER of PARENT ci->fdt[0] = -1; // now we can send all the data thru FIFO[1], writer of PARENT } - //if(aChPid == 0) - //cout << "******** Child finished its work ********" << endl; #endif } else @@ -3500,7 +3498,6 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) if (!ci->dmlProc) { ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "start_bulk_insert starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; } } } @@ -3616,22 +3613,6 @@ int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table) // @bug 2515. Check command intead of vtable state if ( ( ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || (thd->lex)->sql_command == SQLCOM_INSERT_SELECT) && !ci->singleInsert ) { - - //@Bug 2438. Only load data infile calls last batch process - /* if ( ci->isLoaddataInfile && ((thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) || (ci->useCpimport == 0))) { - //@Bug 2829 Handle ctrl-C - if ( thd->killed > 0 ) - abort = true; - - if ( !ci->dmlProc ) - { - ci->dmlProc = new MessageQueueClient("DMLProc"); - //cout << "end_bulk_insert starts a client " << ci->dmlProc << " for session " << thd->thread_id << endl; - } - rc = ha_mcs_impl_write_last_batch(table, *ci, abort); - } - else if ((ci->useCpimport > 0) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) && (!ci->singleInsert) && ((ci->isLoaddataInfile) || - } */ if ((ci->useCpimport > 0) && (!(thd->variables.option_bits & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) && (!ci->singleInsert) && ((ci->isLoaddataInfile) || ((thd->lex)->sql_command == SQLCOM_INSERT) || ((thd->lex)->sql_command == SQLCOM_LOAD) || ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT)) ) @@ -4076,9 +4057,7 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) CalTableMap::iterator mapiter = ci->tableMap.find(table); // make sure this is a release lock (2nd) call called in // the table mode. - if (mapiter != ci->tableMap.end() - && (mapiter->second.condInfo && mapiter->second.csep) - && lock_type == 2) + if (mapiter != ci->tableMap.end() && mapiter->second.csep && lock_type == 2) { // table mode if (mapiter->second.conn_hndl) diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 8e2ec327a..e135c9c92 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -189,7 +189,6 @@ struct cal_table_info enum RowSources { FROM_ENGINE, FROM_FILE }; cal_table_info() : tpl_ctx(0), - //tpl_scan_ctx(0), c(0), msTablePtr(0), conn_hndl(0), diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp new file mode 100644 index 000000000..00c08d9e0 --- /dev/null +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -0,0 +1,245 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#include "ha_mcs_opt_rewrites.h" + +// Search simplify_joins() function in the server's code for detail +COND * +simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, + bool in_sj) +{ + TABLE_LIST *table; + NESTED_JOIN *nested_join; + TABLE_LIST *prev_table= 0; + List_iterator li(*join_list); + bool straight_join= MY_TEST(join->select_options & SELECT_STRAIGHT_JOIN); + DBUG_ENTER("simplify_joins"); + + /* + Try to simplify join operations from join_list. + The most outer join operation is checked for conversion first. + */ + while ((table= li++)) + { + table_map used_tables; + table_map not_null_tables= (table_map) 0; + + if ((nested_join= table->nested_join)) + { + /* + If the element of join_list is a nested join apply + the procedure to its nested join list first. + */ + if (table->on_expr) + { + Item *expr= table->on_expr; + /* + If an on expression E is attached to the table, + check all null rejected predicates in this expression. + If such a predicate over an attribute belonging to + an inner table of an embedded outer join is found, + the outer join is converted to an inner join and + the corresponding on expression is added to E. + */ + expr= simplify_joins_(join, &nested_join->join_list, + expr, FALSE, in_sj || table->sj_on_expr); + + if (!table->prep_on_expr || expr != table->on_expr) + { + DBUG_ASSERT(expr); + + table->on_expr= expr; + table->prep_on_expr= expr->copy_andor_structure(join->thd); + } + } + nested_join->used_tables= (table_map) 0; + nested_join->not_null_tables=(table_map) 0; + conds= simplify_joins_(join, &nested_join->join_list, conds, top, + in_sj || table->sj_on_expr); + used_tables= nested_join->used_tables; + not_null_tables= nested_join->not_null_tables; + /* The following two might become unequal after table elimination: */ + nested_join->n_tables= nested_join->join_list.elements; + } + else + { + if (!table->prep_on_expr) + table->prep_on_expr= table->on_expr; + used_tables= table->get_map(); + if (conds) + not_null_tables= conds->not_null_tables(); + } + + if (table->embedding) + { + table->embedding->nested_join->used_tables|= used_tables; + table->embedding->nested_join->not_null_tables|= not_null_tables; + } + + if (!(table->outer_join & (JOIN_TYPE_LEFT | JOIN_TYPE_RIGHT)) || + (used_tables & not_null_tables)) + { + /* + For some of the inner tables there are conjunctive predicates + that reject nulls => the outer join can be replaced by an inner join. + */ + if (table->outer_join && !table->embedding && table->table) + table->table->maybe_null= FALSE; + table->outer_join= 0; + if (!(straight_join || table->straight)) + { + table->dep_tables= 0; + TABLE_LIST *embedding= table->embedding; + while (embedding) + { + if (embedding->nested_join->join_list.head()->outer_join) + { + if (!embedding->sj_subq_pred) + table->dep_tables= embedding->dep_tables; + break; + } + embedding= embedding->embedding; + } + } + if (table->on_expr) + { + /* Add ON expression to the WHERE or upper-level ON condition. */ + if (conds) + { + conds= and_conds(join->thd, conds, table->on_expr); + conds->top_level_item(); + /* conds is always a new item as both cond and on_expr existed */ + DBUG_ASSERT(!conds->is_fixed()); + conds->fix_fields(join->thd, &conds); + } + else + conds= table->on_expr; + table->prep_on_expr= table->on_expr= 0; + } + } + + /* + Only inner tables of non-convertible outer joins + remain with on_expr. + */ + if (table->on_expr) + { + table_map table_on_expr_used_tables= table->on_expr->used_tables(); + table->dep_tables|= table_on_expr_used_tables; + if (table->embedding) + { + table->dep_tables&= ~table->embedding->nested_join->used_tables; + /* + Embedding table depends on tables used + in embedded on expressions. + */ + table->embedding->on_expr_dep_tables|= table_on_expr_used_tables; + } + else + table->dep_tables&= ~table->get_map(); + } + + if (prev_table) + { + /* The order of tables is reverse: prev_table follows table */ + if (prev_table->straight || straight_join) + prev_table->dep_tables|= used_tables; + if (prev_table->on_expr) + { + prev_table->dep_tables|= table->on_expr_dep_tables; + table_map prev_used_tables= prev_table->nested_join ? + prev_table->nested_join->used_tables : + prev_table->get_map(); + /* + If on expression contains only references to inner tables + we still make the inner tables dependent on the outer tables. + It would be enough to set dependency only on one outer table + for them. Yet this is really a rare case. + Note: + RAND_TABLE_BIT mask should not be counted as it + prevents update of inner table dependences. + For example it might happen if RAND() function + is used in JOIN ON clause. + */ + if (!((prev_table->on_expr->used_tables() & + ~(OUTER_REF_TABLE_BIT | RAND_TABLE_BIT)) & + ~prev_used_tables)) + prev_table->dep_tables|= used_tables; + } + } + prev_table= table; + } + + /* + Flatten nested joins that can be flattened. + no ON expression and not a semi-join => can be flattened. + */ + li.rewind(); + while ((table= li++)) + { + nested_join= table->nested_join; + if (table->sj_on_expr && !in_sj) + { + /* + If this is a semi-join that is not contained within another semi-join + leave it intact (otherwise it is flattened) + */ + /* + Make sure that any semi-join appear in + the join->select_lex->sj_nests list only once + */ + List_iterator_fast sj_it(join->select_lex->sj_nests); + TABLE_LIST *sj_nest; + while ((sj_nest= sj_it++)) + { + if (table == sj_nest) + break; + } + if (sj_nest) + continue; + join->select_lex->sj_nests.push_back(table, join->thd->mem_root); + + /* + Also, walk through semi-join children and mark those that are now + top-level + */ + TABLE_LIST *tbl; + List_iterator it(nested_join->join_list); + while ((tbl= it++)) + { + if (!tbl->on_expr && tbl->table) + tbl->table->maybe_null= FALSE; + } + } + else if (nested_join && !table->on_expr) + { + TABLE_LIST *tbl; + List_iterator it(nested_join->join_list); + List repl_list; + while ((tbl= it++)) + { + tbl->embedding= table->embedding; + if (!tbl->embedding && !tbl->on_expr && tbl->table) + tbl->table->maybe_null= FALSE; + tbl->join_list= table->join_list; + repl_list.push_back(tbl, join->thd->mem_root); + tbl->dep_tables|= table->dep_tables; + } + li.replace(repl_list); + } + } + DBUG_RETURN(conds); +} diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h new file mode 100644 index 000000000..d433c06a5 --- /dev/null +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -0,0 +1,26 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef HA_MCS_REWRITES +#define HA_MCS_REWRITES + +#include "idb_mysql.h" + +COND *simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); + +#endif + diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index d4adc4fea..bab93edd2 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -21,6 +21,27 @@ void check_walk(const Item* item, void* arg); + +void disable_indices_for_CEJ(THD *thd_) +{ + TABLE_LIST* global_list; + for (global_list = thd_->lex->query_tables; global_list; global_list = global_list->next_global) + { + // MCOL-652 - doing this with derived tables can cause bad things to happen + if (!global_list->derived) + { + global_list->index_hints= new (thd_->mem_root) List(); + + global_list->index_hints->push_front(new (thd_->mem_root) + Index_hint(INDEX_HINT_USE, + INDEX_HINT_MASK_JOIN, + NULL, + 0), thd_->mem_root); + + } + } +} + void mutate_optimizer_flags(THD *thd_) { // MCOL-2178 Disable all optimizer flags as it was in the fork. @@ -756,6 +777,7 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } + // Remove this in 1.4.3 // Save the original group_list as it can be mutated by the // optimizer which calls the remove_const() function Group_list_ptrs *group_list_ptrs = NULL; @@ -780,9 +802,24 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) // if execution fails. if (!unsupported_feature) { - // Most of optimizer_switch flags disabled in external_lock - join->optimization_state= JOIN::OPTIMIZATION_IN_PROGRESS; - join->optimize_inner(); + disable_indices_for_CEJ(thd); + + if (select_lex->handle_derived(thd->lex, DT_MERGE)) + { + // early quit b/c of the error in handle_derived + return handler; + } + + COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); + select_lex->optimize_unflattened_subqueries(false); + + if (conds) + { +#ifdef DEBUG_WALK_COND + conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); +#endif + join->conds = conds; + } // Impossible HAVING or WHERE // TODO replace with function call @@ -818,43 +855,18 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) } } - if (!unsupported_feature) { handler= new ha_columnstore_select_handler(thd, select_lex); - // This is an ugly hack to call simplify_joins() mcs_handler_info mhi= mcs_handler_info(reinterpret_cast(handler), SELECT); // this::table is the place for the result set int rc= ha_cs_impl_pushdown_init(&mhi, handler->table); - // Return SH if query execution is fine or fallback is disabled - if (!rc || !get_fallback_knob(thd)) - return handler; - - // Reset the DA and restore optimizer flags - // to allow query to fallback to other handlers - if (thd->get_stmt_da()->is_error()) - { - thd->get_stmt_da()->reset_diagnostics_area(); - restore_optimizer_flags(thd); + // Return SH even if init fails b/c CS changed SELECT_LEX structures + // with simplify_joins_() + if (rc) unsupported_feature = true; - } - } - - if (join->optimization_state != JOIN::NOT_OPTIMIZED) - { - if (!join->with_two_phase_optimization) - { - if (unsupported_feature && join->have_query_plan != JOIN::QEP_DELETED) - { - join->build_explain(); - } - join->optimization_state= JOIN::OPTIMIZATION_DONE; - } - else - { - join->optimization_state= JOIN::OPTIMIZATION_PHASE_1_DONE; - } + return handler; } return NULL; diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index 7e9d31696..268d6b5b6 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -23,6 +23,8 @@ #include "ha_mcs_sysvars.h" #define NEED_CALPONT_EXTERNS #include "ha_mcs_impl.h" +#include "ha_mcs_impl_if.h" +#include "ha_mcs_opt_rewrites.h" void mutate_optimizer_flags(THD *thd_); void restore_optimizer_flags(THD *thd_); diff --git a/dbcon/mysql/ha_mcs_sysvars.cpp b/dbcon/mysql/ha_mcs_sysvars.cpp index af4016139..597d50efa 100644 --- a/dbcon/mysql/ha_mcs_sysvars.cpp +++ b/dbcon/mysql/ha_mcs_sysvars.cpp @@ -98,15 +98,6 @@ static MYSQL_THDVAR_BOOL( 1 ); -static MYSQL_THDVAR_BOOL( - processing_handlers_fallback, - PLUGIN_VAR_NOCMDARG, - "Enable/Disable the unsupported features check in handlers.", - NULL, - NULL, - 0 -); - static MYSQL_THDVAR_UINT( orderby_threads, PLUGIN_VAR_RQCMDARG, @@ -304,7 +295,6 @@ st_mysql_sys_var* mcs_system_variables[] = MYSQL_SYSVAR(original_optimizer_flags), MYSQL_SYSVAR(select_handler), MYSQL_SYSVAR(derived_handler), - MYSQL_SYSVAR(processing_handlers_fallback), MYSQL_SYSVAR(group_by_handler), MYSQL_SYSVAR(orderby_threads), MYSQL_SYSVAR(decimal_scale), @@ -391,16 +381,7 @@ void set_group_by_handler(THD* thd, bool value) THDVAR(thd, group_by_handler) = value; } -bool get_fallback_knob(THD* thd) -{ - return ( thd == NULL ) ? false : THDVAR(thd, processing_handlers_fallback); -} -void set_fallback_knob(THD* thd, bool value) -{ - THDVAR(thd, processing_handlers_fallback) = value; -} - - void set_compression_type(THD* thd, ulong value) +void set_compression_type(THD* thd, ulong value) { THDVAR(thd, compression_type) = value; } diff --git a/dbcon/mysql/ha_mcs_sysvars.h b/dbcon/mysql/ha_mcs_sysvars.h index d9ec7f4d0..1efe29fcc 100644 --- a/dbcon/mysql/ha_mcs_sysvars.h +++ b/dbcon/mysql/ha_mcs_sysvars.h @@ -55,9 +55,6 @@ void set_derived_handler(THD* thd, bool value); bool get_group_by_handler(THD* thd); void set_group_by_handler(THD* thd, bool value); -bool get_fallback_knob(THD* thd); -void set_fallback_knob(THD* thd, bool value); - uint get_orderby_threads(THD* thd); void set_orderby_threads(THD* thd, uint value); diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index bd7f74d38..d14c95131 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -203,7 +203,6 @@ public: private: SELECT_LEX* fFromSub; std::string fAlias; - bool fPushdownHand; }; class SelectSubQuery : public SubQuery diff --git a/dbcon/mysql/sm.cpp b/dbcon/mysql/sm.cpp index a7e476f6f..9925c3c8b 100644 --- a/dbcon/mysql/sm.cpp +++ b/dbcon/mysql/sm.cpp @@ -400,7 +400,7 @@ tpl_close ( cpsm_tplh_t* ntplh, // MCOL-1601 Dispose of unused empty RowGroup if (clear_scan_ctx) { - std::cout << "tpl_close() clear_scan_ctx read" << std::endl; + SMDEBUGLOG << "tpl_close() clear_scan_ctx read" << std::endl; bs = hndl->exeMgr->read(); } From a5bfd484f4f8e66651f41af2a797f2f0012a74f6 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Tue, 26 Nov 2019 04:02:44 +0000 Subject: [PATCH 22/35] Enable unhex function and make minor implementation changes. --- utils/funcexp/func_unhex.cpp | 51 ++++++++++++++++++++---------------- utils/funcexp/funcexp.cpp | 2 +- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/utils/funcexp/func_unhex.cpp b/utils/funcexp/func_unhex.cpp index 1accdd5e3..2019e24ab 100644 --- a/utils/funcexp/func_unhex.cpp +++ b/utils/funcexp/func_unhex.cpp @@ -36,18 +36,25 @@ using namespace execplan; namespace { -int64_t hex_to_int(char c) +inline int hex_to_int(char c, bool& isNull) { if (c <= '9' && c >= '0') return c - '0'; - c = toupper(c); + c |= 32; - if (c <= 'F' && c >= 'A') - return c - 'A' + 10; + if (c <= 'f' && c >= 'a') + return c - 'a' + 10; + isNull = true; return -1; } + +inline string cleanup(char *to) +{ + delete[] to; + return ""; +} } namespace funcexp @@ -65,48 +72,46 @@ string Func_unhex::getStrVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& op_ct) { const string& from = parm[0]->data()->getStrVal(row, isNull); - char* to = new char[2 + from.length() / 2]; - if (!to) - { - isNull = true; + if (isNull) return ""; - } + + char* to = new char[2 + from.size() / 2]; uint64_t from_pos = 0, to_pos = 0; - int64_t hex_char = 0; + int hex_char = 0; - if (strlen(from.c_str()) % 2) + if (from.size() % 2) { - hex_char = hex_to_int(from[from_pos++]); - to[to_pos++] = hex_char; + hex_char = hex_to_int(from[from_pos++], isNull); if (hex_char == -1) - goto nullHandling; + return cleanup(to); + + to[to_pos++] = hex_char; } - for (; from_pos < strlen(from.c_str()); from_pos += 2) + for (; from_pos < from.size(); from_pos += 2) { - hex_char = hex_to_int(from[from_pos]) << 4; + hex_char = hex_to_int(from[from_pos], isNull) << 4; if (hex_char == -1) - goto nullHandling; + return cleanup(to); to[to_pos] = hex_char; - hex_char = hex_to_int(from[from_pos + 1]); + hex_char = hex_to_int(from[from_pos + 1], isNull); if (hex_char == -1) - goto nullHandling; + return cleanup(to); to[to_pos++] |= hex_char; } to[to_pos] = 0; - return string(to); + string tmp = string(to); + delete[] to; -nullHandling: - isNull = true; - return ""; + return tmp; } diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 1c01458d7..2444197d7 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -209,7 +209,7 @@ FuncExp::FuncExp() fFuncMap["trim"] = new Func_trim(); //dlh fFuncMap["truncate"] = new Func_truncate(); //dlh fFuncMap["ucase"] = new Func_ucase(); //dlh - //fFuncMap["unhex"] = new Func_unhex(); + fFuncMap["unhex"] = new Func_unhex(); fFuncMap["unix_timestamp"] = new Func_unix_timestamp(); fFuncMap["upper"] = new Func_ucase(); //dlh fFuncMap["week"] = new Func_week(); //dlh From c2aaf6cea2b664a966ebf001551aabf70b01b7a5 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 2 Dec 2019 15:18:42 -0500 Subject: [PATCH 23/35] Fixed compiler warning in resource manager. --- dbcon/joblist/resourcemanager.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbcon/joblist/resourcemanager.cpp b/dbcon/joblist/resourcemanager.cpp index cbf44109f..bea05b29e 100644 --- a/dbcon/joblist/resourcemanager.cpp +++ b/dbcon/joblist/resourcemanager.cpp @@ -80,6 +80,7 @@ ResourceManager::ResourceManager(bool runningInExeMgr) : fJlProcessorThreadsPerScan(defaultProcessorThreadsPerScan), fJlNumScanReceiveThreads(defaultScanReceiveThreads), fTwNumThreads(defaultNumThreads), + fJlMaxOutstandingRequests(defaultMaxOutstandingRequests), fHJUmMaxMemorySmallSideDistributor(fHashJoinStr, "UmMaxMemorySmallSide", getUintVal(fHashJoinStr, "TotalUmMaxMemorySmallSide", defaultTotalUmMemory), @@ -87,8 +88,7 @@ ResourceManager::ResourceManager(bool runningInExeMgr) : 0), fHJPmMaxMemorySmallSideSessionMap( getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide)), - isExeMgr(runningInExeMgr), - fJlMaxOutstandingRequests(defaultMaxOutstandingRequests) + isExeMgr(runningInExeMgr) { int temp; int configNumCores = -1; @@ -146,7 +146,7 @@ ResourceManager::ResourceManager(bool runningInExeMgr) : if (temp > defaultMaxOutstandingRequests) { fJlMaxOutstandingRequests = temp; - } + } } temp = getIntVal(fJobListStr, "NumScanReceiveThreads", -1); @@ -234,7 +234,7 @@ ResourceManager::ResourceManager(bool runningInExeMgr) : fAggNumBuckets = fAggNumThreads * 4; else fAggNumBuckets = fConfig->uFromText(nb); - + nr = fConfig->getConfig("RowAggregation", "RowAggrRowGroupsPerThread"); if (nr.empty()) @@ -418,4 +418,3 @@ bool ResourceManager::getMemory(int64_t amount, boost::shared_ptr sessi } //namespace - From 8812cceccf70bce5027b57196ee66a3fd5ecae52 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 3 Dec 2019 12:06:17 -0500 Subject: [PATCH 24/35] MCOL-3645 - Make ExeMgr destroy joblists in a different thread A couple lambdas and a little sync... --- exemgr/main.cpp | 70 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 288d2bada..67bd1a31a 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -40,9 +40,6 @@ * front-end. */ - - -#include #include #include #include @@ -75,6 +72,10 @@ #include "utils_utf8.h" #include "config.h" +#include +#include +#include + #if defined(SKIP_OAM_INIT) #include "dbrm.h" #endif @@ -531,6 +532,9 @@ public: csep.sessionID(0); joblist::SJLP jl; bool incSessionThreadCnt = true; + std::mutex jlMutex; + std::condition_variable jlCleanupDone; + int destructing = 0; bool selfJoin = false; bool tryTuples = false; @@ -545,7 +549,22 @@ public: tryTuples = false; usingTuples = false; - jl.reset(); + if (jl) + { + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + std::unique_lock scoped(jlMutex); + destructing++; + std::thread bgdtor([jl, &jlMutex, &jlCleanupDone, &destructing] { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this runs first + bgdtor.detach(); + } + bs = fIos.read(); if (bs.length() == 0) @@ -1038,7 +1057,7 @@ new_plan: //@Bug 1306. Added timing info for real time tracking. std::cout << ss << " at " << timeNow() << std::endl; - // log query status to debug log file + // log query stats to debug log file args.reset(); args.add((int)csep.statementID()); args.add(fStats.fMaxMemPct); @@ -1061,13 +1080,28 @@ new_plan: // here to make sure all syslogging from all the threads // are complete; and that our logDbProfEndStatement will // appear "last" in the syslog for this SQL statement. - jl.reset(); - args.reset(); - args.add((int)csep.statementID()); - msgLog.logMessage(logging::LOG_TYPE_DEBUG, - logDbProfEndStatement, - args, - li); + // puts the real destruction in another thread to avoid + // making the whole session wait. It can take several seconds. + int stmtID = csep.statementID(); + std::unique_lock scoped(jlMutex); + // C7's compiler complains about the msgLog capture here + // msgLog is global scope, and passed by copy, so, unclear + // what the warning is about. + destructing++; + std::thread bgdtor([jl, &jlMutex, &jlCleanupDone, stmtID, &li, msgLog, &destructing] { + std::unique_lock scoped(jlMutex); + const_cast(jl).reset(); // this happens second; does real destruction + logging::Message::Args args; + args.add(stmtID); + msgLog.logMessage(logging::LOG_TYPE_DEBUG, + logDbProfEndStatement, + args, + li); + if (--destructing == 0) + jlCleanupDone.notify_one(); + }); + jl.reset(); // this happens first + bgdtor.detach(); } else // delete sessionMemMap entry for this session's memory % use @@ -1127,14 +1161,12 @@ new_plan: qts.local_query = csep.localQuery(); fTeleClient.postQueryTele(qts); } - } - jl.reset(); // Release CSC object (for sessionID) that was added by makeJobList() - // Mask 0x80000000 is for associate user query and csc query + // Mask 0x80000000 is for associate user query and csc query. + // (actual joblist destruction happens at the top of this loop) decThreadCntPerSession( csep.sessionID() | 0x80000000 ); - } catch (std::exception& ex) { @@ -1158,6 +1190,12 @@ new_plan: msgLog.logMessage(logging::LOG_TYPE_CRITICAL, logExeMgrExcpt, args, li); fIos.close(); } + + // make sure we don't leave scope while joblists are being destroyed + std::unique_lock scoped(jlMutex); + while (destructing > 0) + jlCleanupDone.wait(scoped); + std::cout << "session thread exiting" << std::endl; } }; From 222f963ed7dc8768bebd05f1af21316ee291b56c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 3 Dec 2019 12:14:54 -0500 Subject: [PATCH 25/35] MCOL-3645 - Make ExeMgr destroy joblists in a different thread Removed a debugging printout. --- exemgr/main.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 67bd1a31a..5015dcc6f 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1195,7 +1195,6 @@ new_plan: std::unique_lock scoped(jlMutex); while (destructing > 0) jlCleanupDone.wait(scoped); - std::cout << "session thread exiting" << std::endl; } }; From ce39332e8b5a702d0af03b6a374d644e503e757e Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 3 Dec 2019 13:59:41 -0600 Subject: [PATCH 26/35] MCOL-3563: Add printouts directing user to check log files. --- storage-manager/src/unit_tests.cpp | 58 ++++++++++++------------------ 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/storage-manager/src/unit_tests.cpp b/storage-manager/src/unit_tests.cpp index 7a6c0baab..c77f37d11 100644 --- a/storage-manager/src/unit_tests.cpp +++ b/storage-manager/src/unit_tests.cpp @@ -996,7 +996,7 @@ bool listdirtask(bool connectionTest=false) return true; } -void pingtask(bool connectionTest=false) +void pingtask() { int err=0; uint8_t buf[1024]; @@ -1005,47 +1005,25 @@ void pingtask(bool connectionTest=false) size_t len = sizeof(*cmd); - // set payload to be shorter than actual message lengh - // and send a shortened message. - if (connectionTest) - len -= 2; - ssize_t result = ::write(sessionSock, cmd, sizeof(*cmd)); assert(result==(sizeof(*cmd))); - // set payload to be correct length again - if (connectionTest) - len += 2; // process task will look for the full length and // will wait on the rest of the message. + // don't test connection loss here since this message is only 1 byte ProcessTask pt(clientSock, len); boost::thread t(pt); - if (connectionTest) - { - // make sure the thread is waiting for the rest of the data - // then kill the connection. This will trigger the task thread - // to exit on an error handling path - sleep(1); - close(sessionSock); - close(clientSock); - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - assert(err == -1); - t.join(); - } - else - { - t.join(); - // read the response - err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); - sm_response *resp = (sm_response *) buf; - assert(err == sizeof(sm_response)); - assert(resp->header.type == SM_MSG_START); - assert(resp->header.payloadLen == sizeof(ssize_t)); - assert(resp->header.flags == 0); - assert(resp->returnCode == 0); - } + t.join(); + // read the response + err = ::recv(sessionSock, buf, 1024, MSG_DONTWAIT); + sm_response *resp = (sm_response *) buf; + assert(err == sizeof(sm_response)); + assert(resp->header.type == SM_MSG_START); + assert(resp->header.payloadLen == sizeof(ssize_t)); + assert(resp->header.flags == 0); + assert(resp->returnCode == 0); cout << "pingtask OK" << endl; } @@ -1994,21 +1972,29 @@ int main(int argc, char* argv[]) metadataJournalTestCleanup(); cout << " DONE" << endl; - cout << "Testing connection loss..." << endl; + cout << "Testing connection loss..." << endl << endl; + cout << "Check log files for lines:" << endl; + cout << "[NameTask read] caught an error: Bad file descriptor." << endl; + cout << "****** socket error!" << endl; + cout << "PosixTask::consumeMsg(): Discarding the tail end of a partial msg." << endl << endl; //opentask(); + cout << "OpenTask read2 connection test " << endl; opentask(true); makeConnection(); + cout << "UnlinkTask read connection test " << endl; unlinktask(true); makeConnection(); + cout << "StatTask read connection test " << endl; stattask(true); makeConnection(); + cout << "TruncateTask read connection test " << endl; truncatetask(true); makeConnection(); + cout << "ListDirextoryTask read connection test " << endl; listdirtask(true); makeConnection(); - pingtask(true); - makeConnection(); + cout << "CopyTask read connection test " << endl; copytask(true); (Cache::get())->shutdown(); From 9f89ab0559100703178ec41180f6782d93156fdc Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 25 Nov 2019 11:22:46 +0000 Subject: [PATCH 27/35] MCOL-3627 Fix library name The ColumnStore library is now called ha_columnstore.so to be inline with other storage engines. --- dbcon/mysql/CMakeLists.txt | 28 ++------ dbcon/mysql/columnstore.cnf | 2 +- dbcon/mysql/install_mcs_mysql.sh.in | 66 +++++++++--------- dbcon/mysql/libcalmysql.rc | 102 ---------------------------- dbcon/mysql/resource.h | 14 ---- 5 files changed, 39 insertions(+), 173 deletions(-) delete mode 100644 dbcon/mysql/libcalmysql.rc delete mode 100644 dbcon/mysql/resource.h diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index d99c75d9f..6183fdf68 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -29,11 +29,12 @@ add_definitions(-DMYSQL_DYNAMIC_PLUGIN) set_source_files_properties(ha_mcs.cpp PROPERTIES COMPILE_FLAGS "-fno-implicit-templates") -add_library(calmysql SHARED ${libcalmysql_SRCS}) +add_library(ha_columnstore SHARED ${libcalmysql_SRCS}) +SET_TARGET_PROPERTIES(ha_columnstore PROPERTIES PREFIX "") -target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) +target_link_libraries(ha_columnstore ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) -set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1) +set_target_properties(ha_columnstore PROPERTIES VERSION 1.0.0 SOVERSION 1) SET ( is_columnstore_tables_SRCS is_columnstore_tables.cpp @@ -84,7 +85,7 @@ set_target_properties(is_columnstore_files PROPERTIES PREFIX "") set_target_properties(is_columnstore_files PROPERTIES VERSION 1.0.0 SOVERSION 1) -install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents is_columnstore_files DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) +install(TARGETS ha_columnstore is_columnstore_tables is_columnstore_columns is_columnstore_extents is_columnstore_files DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) install(FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql @@ -97,22 +98,3 @@ install(PROGRAMS install_mcs_mysql.sh mysql-Columnstore install(FILES columnstore.cnf DESTINATION ${MARIADB_MYCNFDIR} COMPONENT storage-engine) - - -#AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags) -#AM_CFLAGS = $(idb_cflags) -#AM_CXXFLAGS = $(idb_cxxflags) -#AM_LDFLAGS = $(idb_ldflags) -#lib_LTLIBRARIES = libcalmysql.la -#libcalmysql_la_SOURCES = ha_mcs.cpp ha_mcs_impl.cpp ha_mcs_dml.cpp ha_mcs_ddl.cpp ha_mcs_execplan.cpp ha_scalar_sub.cpp ha_in_sub.cpp ha_exists_sub.cpp ha_from_sub.cpp ha_select_sub.cpp ha_view.cpp sm.cpp ha_window_function.cpp ha_mcs_partition.cpp ha_pseudocolumn.cpp -#libcalmysql_la_LDFLAGS = -version-info 1:0:0 $(idb_common_ldflags) $(idb_common_libs) $(idb_write_libs) $(AM_LDFLAGS) -#libcalmysql_la_CPPFLAGS = -I/usr/include/libxml2 -I../../../mysql/include -I../../../mysql/sql -I../../../mysql/regex -DMYSQL_DYNAMIC_PLUGIN $(AM_CPPFLAGS) -#include_HEADERS = idb_mysql.h -# -#dist_mysql_DATA = syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql calremoveuserpriority.sql calshowprocesslist.sql my.cnf -#dist_mysql_SCRIPTS = install_mcs_mysql.sh mysql-Columnstore dumpcat.pl -# -#libcalmysql_la-ha_mcs.lo: ha_mcs.cpp -# if $(LIBTOOL) --tag=CXX --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcalmysql_la_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -fno-rtti -fno-implicit-templates -MT libcalmysql_la-ha_mcs.lo -MD -MP -MF "$(DEPDIR)/libcalmysql_la-ha_mcs.Tpo" -c -o libcalmysql_la-ha_mcs.lo `test -f 'ha_mcs.cpp' || echo '$(srcdir)/'`ha_mcs.cpp; \ -# then mv -f "$(DEPDIR)/libcalmysql_la-ha_mcs.Tpo" "$(DEPDIR)/libcalmysql_la-ha_mcs.Plo"; else rm -f "$(DEPDIR)/libcalmysql_la-ha_mcs.Tpo"; exit 1; fi - diff --git a/dbcon/mysql/columnstore.cnf b/dbcon/mysql/columnstore.cnf index 7c551d355..174f6b7ed 100644 --- a/dbcon/mysql/columnstore.cnf +++ b/dbcon/mysql/columnstore.cnf @@ -39,7 +39,7 @@ server-id = 1 lower_case_table_names=1 -plugin-load-add=libcalmysql.so +plugin-load-add=ha_columnstore.so plugin-load-add=is_columnstore_tables.so plugin-load-add=is_columnstore_columns.so plugin-load-add=is_columnstore_extents.so diff --git a/dbcon/mysql/install_mcs_mysql.sh.in b/dbcon/mysql/install_mcs_mysql.sh.in index f677b92ae..6449279e1 100755 --- a/dbcon/mysql/install_mcs_mysql.sh.in +++ b/dbcon/mysql/install_mcs_mysql.sh.in @@ -17,31 +17,31 @@ for arg in "$@"; do done mysql --force --user=root mysql 2> ${tmpdir}/mysql_install.log < Date: Mon, 25 Nov 2019 11:27:34 +0000 Subject: [PATCH 28/35] MCOL-3626 Remove mcsmysql alias --- oam/install_scripts/columnstoreAlias | 1 - oamapps/postConfigure/installer.cpp | 4 ++-- oamapps/postConfigure/postConfigure.cpp | 4 ++-- utils/udfsdk/README.txt | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/oam/install_scripts/columnstoreAlias b/oam/install_scripts/columnstoreAlias index 800aefced..9d55397a4 100644 --- a/oam/install_scripts/columnstoreAlias +++ b/oam/install_scripts/columnstoreAlias @@ -1,6 +1,5 @@ # MariaDB Columnstore Alias Commands # -alias mcsmysql='mysql -u root' alias ma=mcsadmin alias core='cd /var/log/mariadb/columnstore/corefiles' alias tdebug='tail -f /var/log/mariadb/columnstore/debug.log' diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 616589522..906a62168 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -785,7 +785,7 @@ int main(int argc, char* argv[]) cout << ". " << ProfileFile << endl << endl; - cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; + cout << "Enter 'mariadb' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias.sh" << endl << endl; @@ -801,7 +801,7 @@ int main(int argc, char* argv[]) cout << ". " << ProfileFile << endl << endl; - cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; + cout << "Enter 'mariadb' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias" << endl << endl; diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 17fb4867d..10d602b17 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3875,7 +3875,7 @@ int main(int argc, char* argv[]) cout << ". " << ProfileFile << endl << endl; - cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; + cout << "Enter 'mariadb' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias.sh" << endl << endl; @@ -3892,7 +3892,7 @@ int main(int argc, char* argv[]) cout << ". " << ProfileFile << endl << endl; - cout << "Enter 'mcsmysql' to access the MariaDB ColumnStore SQL console" << endl; + cout << "Enter 'mariadb' to access the MariaDB ColumnStore SQL console" << endl; cout << "Enter 'mcsadmin' to access the MariaDB ColumnStore Admin console" << endl << endl; cout << "NOTE: The MariaDB ColumnStore Alias Commands are in /etc/profile.d/columnstoreAlias" << endl << endl; diff --git a/utils/udfsdk/README.txt b/utils/udfsdk/README.txt index 3210979b4..ffd3ad6d8 100644 --- a/utils/udfsdk/README.txt +++ b/utils/udfsdk/README.txt @@ -14,8 +14,8 @@ files to create your own function or just try that function as is. $ cp libudf_mysql.so.1.0.0 libudfsdk.so.1.0.0 /usr/local/mariadb/columnstore/lib/ - Restart ColumnStore $ mcsadmin restartsystem y -- Using the mcsmysql client add the user defined function, e.g, - $ mcsmysql +- Using the mariadb client add the user defined function, e.g, + $ mariadb > create function mcs_add returns integer soname 'libudf_mysql.so'; > create function mcs_isnull returns string soname 'libudf_mysql.so'; From efe829784d70faa4e9c1a3ccc73553f8cd490edb Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 25 Nov 2019 11:51:04 +0000 Subject: [PATCH 29/35] MCOL-3624 Move jemalloc to an LD_PRELOAD Linking will become an issue for the unified build and it is an issue for jemalloc 5.x. Instead we will LD_PRELOAD on the forked ColumnStore specific processes. --- CMakeLists.txt | 13 +------- cmake/FindJeMalloc.cmake | 44 -------------------------- cmake/FindSnappy.cmake | 4 +-- cmake/cpackEngineDEB.cmake | 6 ++-- cmake/cpackEngineRPM.cmake | 6 ++-- oam/install_scripts/columnstore_run.sh | 2 +- storage-manager/CMakeLists.txt | 2 +- 7 files changed, 11 insertions(+), 66 deletions(-) delete mode 100644 cmake/FindJeMalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e8e70be66..43bed845b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,17 +153,6 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() -# Jemalloc has issues with SLES 12, so disable for now -IF (EXISTS "/etc/SuSE-release") - SET(JEMALLOC_LIBRARIES "") -ELSE () - INCLUDE (FindJeMalloc) - if (NOT JEMALLOC_FOUND) - message(FATAL_ERROR "jemalloc not found!") - SET(JEMALLOC_LIBRARIES "") - endif() -ENDIF () - FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -199,7 +188,7 @@ ENDIF() SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${JEMALLOC_LIBRARIES}) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) diff --git a/cmake/FindJeMalloc.cmake b/cmake/FindJeMalloc.cmake deleted file mode 100644 index c97bb1b9f..000000000 --- a/cmake/FindJeMalloc.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# - Try to find jemalloc headers and libraries. -# -# Usage of this module as follows: -# -# find_package(JeMalloc) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# JEMALLOC_ROOT_DIR Set this variable to the root installation of -# jemalloc if the module has problems finding -# the proper installation path. -# -# Variables defined by this module: -# -# JEMALLOC_FOUND System has jemalloc libs/headers -# JEMALLOC_LIBRARIES The jemalloc library/libraries -# JEMALLOC_INCLUDE_DIR The location of jemalloc headers - -find_path(JEMALLOC_ROOT_DIR - NAMES include/jemalloc/jemalloc.h -) - -find_library(JEMALLOC_LIBRARIES - NAMES jemalloc - HINTS ${JEMALLOC_ROOT_DIR}/lib -) - -find_path(JEMALLOC_INCLUDE_DIR - NAMES jemalloc/jemalloc.h - HINTS ${JEMALLOC_ROOT_DIR}/include -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(JeMalloc DEFAULT_MSG - JEMALLOC_LIBRARIES - JEMALLOC_INCLUDE_DIR -) - -mark_as_advanced( - JEMALLOC_ROOT_DIR - JEMALLOC_LIBRARIES - JEMALLOC_INCLUDE_DIR -) diff --git a/cmake/FindSnappy.cmake b/cmake/FindSnappy.cmake index 7ac14a271..6aaf92229 100644 --- a/cmake/FindSnappy.cmake +++ b/cmake/FindSnappy.cmake @@ -8,7 +8,7 @@ # to be set before calling find_package: # # SNAPPY_ROOT_DIR Set this variable to the root installation of -# jemalloc if the module has problems finding +# snappy if the module has problems finding # the proper installation path. # # Variables defined by this module: @@ -28,7 +28,7 @@ find_library(SNAPPY_LIBRARIES find_path(SNAPPY_INCLUDE_DIR NAMES snappy.h - HINTS ${JEMALLOC_ROOT_DIR}/include + HINTS ${SNAPPY_ROOT_DIR}/include ) include(FindPackageHandleStandardArgs) diff --git a/cmake/cpackEngineDEB.cmake b/cmake/cpackEngineDEB.cmake index 4cae76f6a..090a1f8c0 100644 --- a/cmake/cpackEngineDEB.cmake +++ b/cmake/cpackEngineDEB.cmake @@ -65,11 +65,11 @@ if (EXISTS "/etc/debian_version") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") endif () if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1, libjemalloc1") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1, libjemalloc1 net-tools") elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5, libjemalloc1") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5, libjemalloc1 net-tools") else() - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1 net-tools") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/cmake/cpackEngineRPM.cmake b/cmake/cpackEngineRPM.cmake index 5f95cb8c6..3802b02bc 100644 --- a/cmake/cpackEngineRPM.cmake +++ b/cmake/cpackEngineRPM.cmake @@ -87,13 +87,13 @@ IF (EXISTS "/etc/SuSE-release") set(SUSE_VERSION_NUMBER "${CMAKE_MATCH_1}") ENDIF () if (${REDHAT_VERSION_NUMBER} EQUAL 6) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "mariadb-columnstore-libs" "mariadb-columnstore-shared" "snappy") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "mariadb-columnstore-libs" "mariadb-columnstore-shared" "snappy" "net-tools") # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1" "jemalloc") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1" "jemalloc" "net-tools") else () - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy" "jemalloc") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy" "jemalloc" "net-tools") endif() SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") diff --git a/oam/install_scripts/columnstore_run.sh b/oam/install_scripts/columnstore_run.sh index 59523337d..e77464543 100755 --- a/oam/install_scripts/columnstore_run.sh +++ b/oam/install_scripts/columnstore_run.sh @@ -49,7 +49,7 @@ if [ $vflg -gt 0 ]; then fi while [ $keep_going -ne 0 ]; do - $exename $args + LD_PRELOAD=libjemalloc.so $exename $args if [ -e ${lopt}/StopColumnstore ]; then exit 0 fi diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 36fcfc796..c944e8f0c 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -60,7 +60,7 @@ set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) add_library(storagemanager SHARED ${storagemanager_SRCS}) add_dependencies(storagemanager marias3) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(storagemanager boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS} ${JEMALLOC_LIBRARIES}) +target_link_libraries(storagemanager boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) set_property(TARGET storagemanager PROPERTY CXX_STANDARD 11) add_executable(StorageManager src/main.cpp) From e072bf9e9b79d8b6508bccb809f5947a30d90f22 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 25 Nov 2019 14:17:17 +0000 Subject: [PATCH 30/35] MCOL-3628 Move I_S tables into main handler lib The I_S tables are all now in ha_columnstore.so --- dbcon/mysql/CMakeLists.txt | 57 +++------------------ dbcon/mysql/ha_mcs.cpp | 69 ++++++++++++++++++++++++++ dbcon/mysql/is_columnstore.h | 23 +++++++++ dbcon/mysql/is_columnstore_columns.cpp | 26 +--------- dbcon/mysql/is_columnstore_extents.cpp | 28 +---------- dbcon/mysql/is_columnstore_files.cpp | 24 +-------- dbcon/mysql/is_columnstore_tables.cpp | 27 +--------- 7 files changed, 106 insertions(+), 148 deletions(-) create mode 100644 dbcon/mysql/is_columnstore.h diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 6183fdf68..ca2bb9381 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -23,7 +23,11 @@ SET ( libcalmysql_SRCS ha_view.cpp sm.cpp ha_window_function.cpp ha_mcs_partition.cpp - ha_pseudocolumn.cpp) + ha_pseudocolumn.cpp + is_columnstore_tables.cpp + is_columnstore_columns.cpp + is_columnstore_files.cpp + is_columnstore_extents.cpp) add_definitions(-DMYSQL_DYNAMIC_PLUGIN) @@ -36,56 +40,7 @@ target_link_libraries(ha_columnstore ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NE set_target_properties(ha_columnstore PROPERTIES VERSION 1.0.0 SOVERSION 1) -SET ( is_columnstore_tables_SRCS - is_columnstore_tables.cpp - sm.cpp - ) -add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS}) - -target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) - -# Don't prepend .so file with 'lib' -set_target_properties(is_columnstore_tables PROPERTIES PREFIX "") -set_target_properties(is_columnstore_tables PROPERTIES VERSION 1.0.0 SOVERSION 1) - -SET ( is_columnstore_columns_SRCS - is_columnstore_columns.cpp - sm.cpp - ) -add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS}) - -target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) - -# Don't prepend .so file with 'lib' -set_target_properties(is_columnstore_columns PROPERTIES PREFIX "") -set_target_properties(is_columnstore_columns PROPERTIES VERSION 1.0.0 SOVERSION 1) - -SET ( is_columnstore_extents_SRCS - is_columnstore_extents.cpp - sm.cpp - ) -add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS}) - -target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) - -# Don't prepend .so file with 'lib' -set_target_properties(is_columnstore_extents PROPERTIES PREFIX "") -set_target_properties(is_columnstore_extents PROPERTIES VERSION 1.0.0 SOVERSION 1) - -SET ( is_columnstore_files_SRCS - is_columnstore_files.cpp - sm.cpp - ) -add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS}) - -target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) - -# Don't prepend .so file with 'lib' -set_target_properties(is_columnstore_files PROPERTIES PREFIX "") -set_target_properties(is_columnstore_files PROPERTIES VERSION 1.0.0 SOVERSION 1) - - -install(TARGETS ha_columnstore is_columnstore_tables is_columnstore_columns is_columnstore_extents is_columnstore_files DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) +install(TARGETS ha_columnstore DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) install(FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index d501140f5..76932b545 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -23,6 +23,7 @@ #include "ha_mcs_pushdown.h" #define NEED_CALPONT_EXTERNS #include "ha_mcs_impl.h" +#include "is_columnstore.h" static handler* calpont_create_handler(handlerton* hton, TABLE_SHARE* table, @@ -911,6 +912,10 @@ const COND* ha_mcs::cond_push(const COND* cond) struct st_mysql_storage_engine columnstore_storage_engine = { MYSQL_HANDLERTON_INTERFACE_VERSION }; +static struct st_mysql_information_schema is_columnstore_plugin_version = +{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; + + mysql_declare_plugin(columnstore) { MYSQL_STORAGE_ENGINE_PLUGIN, @@ -943,6 +948,70 @@ maria_declare_plugin(columnstore) mcs_system_variables, /* system variables */ "1.0", /* string version */ MariaDB_PLUGIN_MATURITY_STABLE /* maturity */ +}, +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_plugin_version, + "COLUMNSTORE_COLUMNS", + "MariaDB Corporation", + "An information schema plugin to list ColumnStore columns", + PLUGIN_LICENSE_GPL, + is_columnstore_columns_plugin_init, + //is_columnstore_tables_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +}, +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_plugin_version, + "COLUMNSTORE_TABLES", + "MariaDB Corporation", + "An information schema plugin to list ColumnStore tables", + PLUGIN_LICENSE_GPL, + is_columnstore_tables_plugin_init, + //is_columnstore_tables_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +}, +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_plugin_version, + "COLUMNSTORE_FILES", + "MariaDB Corporation", + "An information schema plugin to list ColumnStore filess", + PLUGIN_LICENSE_GPL, + is_columnstore_files_plugin_init, + //is_columnstore_files_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE +}, +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &is_columnstore_plugin_version, + "COLUMNSTORE_EXTENTS", + "MariaDB Corporation", + "An information schema plugin to list ColumnStore extents", + PLUGIN_LICENSE_GPL, + is_columnstore_extents_plugin_init, + //is_columnstore_extents_plugin_deinit, + NULL, + 0x0100, + NULL, + NULL, + "1.0", + MariaDB_PLUGIN_MATURITY_STABLE } maria_declare_plugin_end; diff --git a/dbcon/mysql/is_columnstore.h b/dbcon/mysql/is_columnstore.h new file mode 100644 index 000000000..27f899007 --- /dev/null +++ b/dbcon/mysql/is_columnstore.h @@ -0,0 +1,23 @@ +/* Copyright (C) 2019 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +int is_columnstore_extents_plugin_init(void* p); +int is_columnstore_files_plugin_init(void* p); +int is_columnstore_tables_plugin_init(void* p); +int is_columnstore_columns_plugin_init(void* p); diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp index 66d97564c..b664cfc7c 100644 --- a/dbcon/mysql/is_columnstore_columns.cpp +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -28,6 +28,7 @@ #include "calpontsystemcatalog.h" #include "dataconvert.h" #include "exceptclasses.h" +#include "is_columnstore.h" using namespace logging; @@ -260,7 +261,7 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond) return 0; } -static int is_columnstore_columns_plugin_init(void* p) +int is_columnstore_columns_plugin_init(void* p) { ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; schema->fields_info = is_columnstore_columns_fields; @@ -268,26 +269,3 @@ static int is_columnstore_columns_plugin_init(void* p) return 0; } -static struct st_mysql_information_schema is_columnstore_columns_plugin_version = -{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -maria_declare_plugin(is_columnstore_columns_plugin) -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_columns_plugin_version, - "COLUMNSTORE_COLUMNS", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore columns", - PLUGIN_LICENSE_GPL, - is_columnstore_columns_plugin_init, - //is_columnstore_tables_plugin_deinit, - NULL, - 0x0100, - NULL, - NULL, - "1.0", - MariaDB_PLUGIN_MATURITY_STABLE -} -maria_declare_plugin_end; - - diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 99aa43eba..cb8044806 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -26,7 +26,7 @@ #include "dbrm.h" #include "objectidmanager.h" - +#include "is_columnstore.h" // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); @@ -278,34 +278,10 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond) return 0; } -static int is_columnstore_extents_plugin_init(void* p) +int is_columnstore_extents_plugin_init(void* p) { ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; schema->fields_info = is_columnstore_extents_fields; schema->fill_table = is_columnstore_extents_fill; return 0; } - -static struct st_mysql_information_schema is_columnstore_extents_plugin_version = -{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -maria_declare_plugin(is_columnstore_extents_plugin) -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_extents_plugin_version, - "COLUMNSTORE_EXTENTS", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore extents", - PLUGIN_LICENSE_GPL, - is_columnstore_extents_plugin_init, - //is_columnstore_extents_plugin_deinit, - NULL, - 0x0100, - NULL, - NULL, - "1.0", - MariaDB_PLUGIN_MATURITY_STABLE -} -maria_declare_plugin_end; - - diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 378985303..429b5c650 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -35,6 +35,7 @@ #include "messagequeue.h" #include "messagequeuepool.h" #include "we_messages.h" +#include "is_columnstore.h" // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); @@ -289,7 +290,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond) return 0; } -static int is_columnstore_files_plugin_init(void* p) +int is_columnstore_files_plugin_init(void* p) { ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; schema->fields_info = is_columnstore_files_fields; @@ -297,24 +298,3 @@ static int is_columnstore_files_plugin_init(void* p) return 0; } -static struct st_mysql_information_schema is_columnstore_files_plugin_version = -{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -maria_declare_plugin(is_columnstore_files_plugin) -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_files_plugin_version, - "COLUMNSTORE_FILES", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore filess", - PLUGIN_LICENSE_GPL, - is_columnstore_files_plugin_init, - //is_columnstore_files_plugin_deinit, - NULL, - 0x0100, - NULL, - NULL, - "1.0", - MariaDB_PLUGIN_MATURITY_STABLE -} -maria_declare_plugin_end; diff --git a/dbcon/mysql/is_columnstore_tables.cpp b/dbcon/mysql/is_columnstore_tables.cpp index 1f66308f7..711d80da5 100644 --- a/dbcon/mysql/is_columnstore_tables.cpp +++ b/dbcon/mysql/is_columnstore_tables.cpp @@ -26,7 +26,7 @@ #include #include "calpontsystemcatalog.h" #include "dataconvert.h" - +#include "is_columnstore.h" // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); @@ -158,7 +158,7 @@ static int is_columnstore_tables_fill(THD* thd, TABLE_LIST* tables, COND* cond) return 0; } -static int is_columnstore_tables_plugin_init(void* p) +int is_columnstore_tables_plugin_init(void* p) { ST_SCHEMA_TABLE* schema = (ST_SCHEMA_TABLE*) p; schema->fields_info = is_columnstore_tables_fields; @@ -166,26 +166,3 @@ static int is_columnstore_tables_plugin_init(void* p) return 0; } -static struct st_mysql_information_schema is_columnstore_tables_plugin_version = -{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION }; - -maria_declare_plugin(is_columnstore_tables_plugin) -{ - MYSQL_INFORMATION_SCHEMA_PLUGIN, - &is_columnstore_tables_plugin_version, - "COLUMNSTORE_TABLES", - "MariaDB Corporation", - "An information schema plugin to list ColumnStore tables", - PLUGIN_LICENSE_GPL, - is_columnstore_tables_plugin_init, - //is_columnstore_tables_plugin_deinit, - NULL, - 0x0100, - NULL, - NULL, - "1.0", - MariaDB_PLUGIN_MATURITY_STABLE -} -maria_declare_plugin_end; - - From 27ec629acee8e76653bdacf7ce27135cdf23af65 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 25 Nov 2019 14:19:46 +0000 Subject: [PATCH 31/35] Remove I_S plugins from my.cnf --- dbcon/mysql/columnstore.cnf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbcon/mysql/columnstore.cnf b/dbcon/mysql/columnstore.cnf index 174f6b7ed..24e43cef3 100644 --- a/dbcon/mysql/columnstore.cnf +++ b/dbcon/mysql/columnstore.cnf @@ -40,7 +40,3 @@ server-id = 1 lower_case_table_names=1 plugin-load-add=ha_columnstore.so -plugin-load-add=is_columnstore_tables.so -plugin-load-add=is_columnstore_columns.so -plugin-load-add=is_columnstore_extents.so -plugin-load-add=is_columnstore_files.so From a8cd34f86d011fec3c31ad8455a83fea1f0ba476 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 25 Nov 2019 16:47:45 +0000 Subject: [PATCH 32/35] Add support for building from server --- CMakeLists.txt | 105 ++++++++---- build/CMakeLists.txt | 155 ------------------ dbcon/mysql/CMakeLists.txt | 16 +- exemgr/main.cpp | 2 +- config.h.in => mcsconfig.h.in | 6 +- oam/oamcpp/CMakeLists.txt | 1 - oam/oamcpp/liboamcpp.cpp | 2 +- oam/oamcpp/oamcache.cpp | 2 +- .../columnstoreSupport/columnstoreSupport.cpp | 2 +- oamapps/mcsadmin/mcsadmin.cpp | 2 +- oamapps/postConfigure/helpers.cpp | 2 +- oamapps/postConfigure/installer.cpp | 2 +- oamapps/postConfigure/mycnfUpgrade.cpp | 2 +- oamapps/postConfigure/postConfigure.cpp | 2 +- primitives/blockcache/iomanager.cpp | 2 +- procmgr/processmanager.cpp | 2 +- procmon/processmonitor.cpp | 2 +- storage-manager/CMakeLists.txt | 2 +- storage-manager/src/Config.cpp | 2 +- tools/clearShm/main.cpp | 2 +- tools/dbbuilder/dbbuilder.cpp | 2 +- tools/qfe/returnedrows.cpp | 2 +- tools/qfe/server.cpp | 2 +- tools/qfe/socketio.cpp | 2 +- utils/cloudio/CMakeLists.txt | 2 +- utils/configcpp/configcpp.cpp | 2 +- utils/configcpp/configstream.cpp | 2 +- utils/configcpp/xmlparser.cpp | 2 +- utils/funcexp/CMakeLists.txt | 1 - utils/loggingcpp/idberrorinfo.cpp | 2 +- utils/loggingcpp/message.cpp | 2 +- utils/messageqcpp/compressed_iss.cpp | 2 +- utils/messageqcpp/inetstreamsocket.cpp | 2 +- utils/multicast/socklib.h | 2 +- versioning/BRM/brmtypes.cpp | 2 +- .../we_redistributecontrolthread.cpp | 2 +- writeengine/server/we_dataloader.cpp | 2 +- writeengine/shared/we_convertor.cpp | 2 +- writeengine/shared/we_fileop.cpp | 2 +- writeengine/splitter/we_sdhandler.cpp | 2 +- 40 files changed, 125 insertions(+), 227 deletions(-) delete mode 100644 build/CMakeLists.txt rename config.h.in => mcsconfig.h.in (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43bed845b..be77d6c99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,35 +77,68 @@ SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) SET (ENGINE_SYSCONFDIR "/etc") SET (ENGINE_DATADIR "/var/lib/columnstore") -# TODO: This will all be replaced by variables from server install_layout.cmake once we are a submodule IF (RPM) - SET(MARIADB_PLUGINDIR "/usr/lib64/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/my.cnf.d") - SET (ENGINE_LIBDIR "/usr/lib64") - SET (ENGINE_BINDIR "/usr/bin") - SET (ENGINE_INCDIR "/usr/include") - SET (ENGINE_MANDIR "/usr/share/man") - SET (ENGINE_SBINDIR "/usr/sbin") - SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") + # We are building from MariaDB server submodule if this is set + if (INSTALL_LAYOUT) + SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR_RPM}") + SET(MARIADB_MYCNFDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SYSCONF2DIR_RPM}") + SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR_RPM}") + SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR_RPM}") + SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") + SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR_RPM}") + SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR_RPM}") + SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") + else () + SET(MARIADB_PLUGINDIR "/usr/lib64/mysql/plugin") + SET(MARIADB_MYCNFDIR "/etc/my.cnf.d") + SET (ENGINE_LIBDIR "/usr/lib64") + SET (ENGINE_BINDIR "/usr/bin") + SET (ENGINE_INCDIR "/usr/include") + SET (ENGINE_MANDIR "/usr/share/man") + SET (ENGINE_SBINDIR "/usr/sbin") + SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") + endif () ELSEIF (DEB) - SET(MARIADB_PLUGINDIR "/usr/lib/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/mysql/conf.d") - SET (ENGINE_LIBDIR "/usr/lib") - SET (ENGINE_BINDIR "/usr/bin") - SET (ENGINE_INCDIR "/usr/include") - SET (ENGINE_MANDIR "/usr/share/man") - SET (ENGINE_SBINDIR "/usr/sbin") - SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") + if (INSTALL_LAYOUT) + SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR_DEB}") + SET(MARIADB_MYCNFDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SYSCONF2DIR_DEB}") + SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR_DEB}") + SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR_DEB}") + SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") + SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR_DEB}") + SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR_DEB}") + SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") + else () + SET(MARIADB_PLUGINDIR "/usr/lib/mysql/plugin") + SET(MARIADB_MYCNFDIR "/etc/mysql/conf.d") + SET (ENGINE_LIBDIR "/usr/lib") + SET (ENGINE_BINDIR "/usr/bin") + SET (ENGINE_INCDIR "/usr/include") + SET (ENGINE_MANDIR "/usr/share/man") + SET (ENGINE_SBINDIR "/usr/sbin") + SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") + endif () ELSE() - # TODO: prefix should probably apply here - SET(MARIADB_PLUGINDIR "/usr/local/lib/mysql/plugin") - SET(MARIADB_MYCNFDIR "/etc/mysql") - SET (ENGINE_LIBDIR "/usr/local/lib") - SET (ENGINE_BINDIR "/usr/local/bin") - SET (ENGINE_INCDIR "/usr/local/include") - SET (ENGINE_MANDIR "/usr/local/man") - SET (ENGINE_SBINDIR "/usr/local/sbin") - SET (ENGINE_SUPPORTDIR "/usr/local/share/columnstore") + if (INSTALL_LAYOUT) + SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR_STANDALONE}") + SET(MARIADB_MYCNFDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SYSCONF2DIR_STANDALONE}") + SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR_STANDALONE}") + SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR_STANDALONE}") + SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") + SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR_STANDALONE}") + SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR_STANDALONE}") + SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") + else () + # TODO: prefix should probably apply here + SET(MARIADB_PLUGINDIR "/usr/local/lib/mysql/plugin") + SET(MARIADB_MYCNFDIR "/etc/mysql") + SET (ENGINE_LIBDIR "/usr/local/lib") + SET (ENGINE_BINDIR "/usr/local/bin") + SET (ENGINE_INCDIR "/usr/local/include") + SET (ENGINE_MANDIR "/usr/local/man") + SET (ENGINE_SBINDIR "/usr/local/sbin") + SET (ENGINE_SUPPORTDIR "/usr/local/share/columnstore") + endif () ENDIF() SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) @@ -200,6 +233,8 @@ IF (SERVER_BUILD_INCLUDE_DIR) IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR}) SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) ENDIF() +ELSEIF (INSTALL_LAYOUT) + SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/include) ELSE() SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) ENDIF() @@ -208,6 +243,8 @@ IF (SERVER_SOURCE_ROOT_DIR) IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR}) SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) ENDIF() +ELSEIF (INSTALL_LAYOUT) + SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_SOURCE_DIR}) ELSE() SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) ENDIF() @@ -219,7 +256,11 @@ ENDIF() MESSAGE("SERVER_BUILD_INCLUDE_DIR = ${SERVER_BUILD_INCLUDE_DIR}") MESSAGE("SERVER_SOURCE_ROOT_DIR = ${SERVER_SOURCE_ROOT_DIR}") -SET (MARIADB_CLIENT_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so) +IF (INSTALL_LAYOUT) + SET (MARIADB_CLIENT_LIBS libmariadb) +ELSE() + SET (MARIADB_CLIENT_LIBS -L${SERVER_BUILD_INCLUDE_DIR}/../libmariadb/libmariadb/ libmariadb.so) +ENDIF() #SET (ENGINE_UTILS_BOOSTIDB_INCLUDE "{CMAKE_CURRENT_SOURCE_DIR}/utils/boost_idb") SET (ENGINE_UTILS_MESSAGEQCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/messageqcpp") @@ -322,10 +363,18 @@ exec_program("git" CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) -CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h) +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform) +# Do this or when MariaDB builds us we don't have GenError which is required for these +IF (INSTALL_LAYOUT) + ADD_DEPENDENCIES(udf_mysql GenError) + ADD_DEPENDENCIES(funcexp GenError) + ADD_DEPENDENCIES(oamcpp GenError) + ADD_DEPENDENCIES(regr_mysql GenError) +ENDIF () + INCLUDE(cpackEngineRPM) INCLUDE(cpackEngineDEB) diff --git a/build/CMakeLists.txt b/build/CMakeLists.txt deleted file mode 100644 index b28fab8b8..000000000 --- a/build/CMakeLists.txt +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (C) 2006 MySQL AB -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -INCLUDE("${PROJECT_SOURCE_DIR}/win/mysql_manifest.cmake") - -SET(CMAKE_CXX_FLAGS_DEBUG - "${CMAKE_CXX_FLAGS_DEBUG} -DSAFEMALLOC -DSAFE_MUTEX -DUSE_SYMDIR /Zi") -SET(CMAKE_C_FLAGS_DEBUG - "${CMAKE_C_FLAGS_DEBUG} -DSAFEMALLOC -DSAFE_MUTEX -DUSE_SYMDIR /Zi") -SET(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /MAP /MAPINFO:EXPORTS") - -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/extra/yassl/include - ${CMAKE_SOURCE_DIR}/sql - ${CMAKE_SOURCE_DIR}/regex - ${CMAKE_SOURCE_DIR}/zlib -) - -SET_SOURCE_FILES_PROPERTIES(${CMAKE_SOURCE_DIR}/sql/sql_yacc.h - ${CMAKE_SOURCE_DIR}/sql/sql_yacc.cc - ${CMAKE_SOURCE_DIR}/include/mysql_version.h - ${CMAKE_SOURCE_DIR}/sql/sql_builtin.cc - ${CMAKE_SOURCE_DIR}/sql/lex_hash.h - ${PROJECT_SOURCE_DIR}/include/mysqld_error.h - ${PROJECT_SOURCE_DIR}/include/mysqld_ername.h - ${PROJECT_SOURCE_DIR}/include/sql_state.h - PROPERTIES GENERATED 1) - -ADD_DEFINITIONS(-DMYSQL_SERVER -D_CONSOLE -DHAVE_DLOPEN -DHAVE_EVENT_SCHEDULER) - - -SET (SQL_SOURCE - ../sql-common/client.c derror.cc des_key_file.cc - discover.cc ../libmysql/errmsg.c field.cc field_conv.cc - filesort.cc gstream.cc - ha_partition.cc - handler.cc hash_filo.cc hash_filo.h - hostname.cc init.cc item.cc item_buff.cc item_cmpfunc.cc - item_create.cc item_func.cc item_geofunc.cc item_row.cc - item_strfunc.cc item_subselect.cc item_sum.cc item_timefunc.cc - item_window_function.cc item_create_window_function.cc - key.cc log.cc lock.cc message.rc - log_event.cc rpl_record.cc rpl_reporting.cc - log_event_old.cc rpl_record_old.cc - message.h mf_iocache.cc my_decimal.cc ../sql-common/my_time.c - mysqld.cc net_serv.cc - nt_servc.cc nt_servc.h opt_range.cc opt_range.h opt_sum.cc - ../sql-common/pack.c parse_file.cc password.c procedure.cc - protocol.cc records.cc repl_failsafe.cc rpl_filter.cc set_var.cc - slave.cc sp.cc sp_cache.cc sp_head.cc sp_pcontext.cc - sp_rcontext.cc spatial.cc sql_acl.cc sql_analyse.cc sql_base.cc - sql_cache.cc sql_class.cc sql_client.cc sql_crypt.cc sql_crypt.h - sql_cursor.cc sql_db.cc sql_delete.cc sql_derived.cc sql_do.cc - sql_error.cc sql_handler.cc sql_help.cc sql_insert.cc sql_lex.cc - sql_list.cc sql_load.cc sql_manager.cc sql_map.cc sql_parse.cc - sql_partition.cc sql_plugin.cc sql_prepare.cc sql_rename.cc - sql_repl.cc sql_select.cc sql_show.cc sql_state.c sql_string.cc - sql_table.cc sql_test.cc sql_trigger.cc sql_udf.cc sql_union.cc - sql_update.cc sql_view.cc strfunc.cc table.cc thr_malloc.cc - time.cc tztime.cc uniques.cc unireg.cc item_xmlfunc.cc - rpl_tblmap.cc sql_binlog.cc event_scheduler.cc event_data_objects.cc - event_queue.cc event_db_repository.cc - sql_tablespace.cc events.cc ../sql-common/my_user.c - partition_info.cc rpl_utility.cc rpl_injector.cc sql_locale.cc - rpl_rli.cc rpl_mi.cc sql_servers.cc - sql_connect.cc scheduler.cc - sql_profile.cc event_parse_data.cc - ${PROJECT_SOURCE_DIR}/sql/sql_yacc.cc - ${PROJECT_SOURCE_DIR}/sql/sql_yacc.h - ${PROJECT_SOURCE_DIR}/include/mysqld_error.h - ${PROJECT_SOURCE_DIR}/include/mysqld_ername.h - ${PROJECT_SOURCE_DIR}/include/sql_state.h - ${PROJECT_SOURCE_DIR}/include/mysql_version.h - ${PROJECT_SOURCE_DIR}/sql/sql_builtin.cc - ${PROJECT_SOURCE_DIR}/sql/lex_hash.h) -ADD_LIBRARY(sql ${SQL_SOURCE}) - -IF (NOT EXISTS cmake_dummy.cc) - FILE (WRITE cmake_dummy.cc "") -ENDIF (NOT EXISTS cmake_dummy.cc) -ADD_EXECUTABLE(mysqld cmake_dummy.cc) - -SET_TARGET_PROPERTIES(mysqld PROPERTIES OUTPUT_NAME mysqld${MYSQLD_EXE_SUFFIX}) -SET_TARGET_PROPERTIES(mysqld PROPERTIES ENABLE_EXPORTS TRUE) - -SET (MYSQLD_CORE_LIBS mysys zlib dbug strings yassl taocrypt vio regex sql) -TARGET_LINK_LIBRARIES(mysqld ${MYSQLD_CORE_LIBS} ${MYSQLD_STATIC_ENGINE_LIBS}) -TARGET_LINK_LIBRARIES(mysqld ws2_32.lib) - - -IF(MSVC AND NOT WITHOUT_DYNAMIC_PLUGINS) - # Set module definition file. Also use non-incremental linker, - # incremental appears to crash from time to time,if used with /DEF option - SET_TARGET_PROPERTIES(mysqld PROPERTIES LINK_FLAGS "/DEF:mysqld.def /INCREMENTAL:NO") - - FOREACH (CORELIB ${MYSQLD_CORE_LIBS}) - GET_TARGET_PROPERTY(LOC ${CORELIB} LOCATION) - FILE(TO_NATIVE_PATH ${LOC} LOC) - SET (LIB_LOCATIONS ${LIB_LOCATIONS} ${LOC}) - ENDFOREACH (CORELIB ${MYSQLD_CORE_LIBS}) - - ADD_CUSTOM_COMMAND(TARGET mysqld PRE_LINK - COMMAND cscript ARGS //nologo ${PROJECT_SOURCE_DIR}/win/create_def_file.js - ${PLATFORM} ${LIB_LOCATIONS} > mysqld.def - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/sql) -ENDIF(MSVC AND NOT WITHOUT_DYNAMIC_PLUGINS) - -ADD_DEPENDENCIES(sql GenError) - -# Sql Parser custom command -ADD_CUSTOM_COMMAND( - OUTPUT ${PROJECT_SOURCE_DIR}/sql/sql_yacc.h - ${PROJECT_SOURCE_DIR}/sql/sql_yacc.cc - COMMAND bison ARGS -y -p MYSQL --defines=sql_yacc.h - --output=sql_yacc.cc sql_yacc.yy - DEPENDS ${PROJECT_SOURCE_DIR}/sql/sql_yacc.yy) - - -# Gen_lex_hash -ADD_EXECUTABLE(gen_lex_hash gen_lex_hash.cc) -TARGET_LINK_LIBRARIES(gen_lex_hash debug dbug mysqlclient wsock32) -GET_TARGET_PROPERTY(GEN_LEX_HASH_EXE gen_lex_hash LOCATION) -ADD_CUSTOM_COMMAND( - OUTPUT ${PROJECT_SOURCE_DIR}/sql/lex_hash.h - COMMAND ${GEN_LEX_HASH_EXE} ARGS > lex_hash.h - DEPENDS ${GEN_LEX_HASH_EXE}) - -ADD_CUSTOM_TARGET( - GenServerSource ALL - DEPENDS ${PROJECT_SOURCE_DIR}/sql/sql_yacc.h - ${PROJECT_SOURCE_DIR}/sql/sql_yacc.cc - ${PROJECT_SOURCE_DIR}/sql/message.h - ${PROJECT_SOURCE_DIR}/sql/message.rc - ${PROJECT_SOURCE_DIR}/sql/lex_hash.h) - -ADD_DEPENDENCIES(mysqld GenServerSource) - -# Remove the auto-generated files as part of 'Clean Solution' -SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES - "lex_hash.h;sql_yacc.h;sql_yacc.cc;mysqld.def") - -ADD_LIBRARY(udf_example MODULE udf_example.c udf_example.def) -ADD_DEPENDENCIES(udf_example strings GenError) -TARGET_LINK_LIBRARIES(udf_example strings wsock32) diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index ca2bb9381..ec1f84c47 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -33,14 +33,20 @@ add_definitions(-DMYSQL_DYNAMIC_PLUGIN) set_source_files_properties(ha_mcs.cpp PROPERTIES COMPILE_FLAGS "-fno-implicit-templates") -add_library(ha_columnstore SHARED ${libcalmysql_SRCS}) -SET_TARGET_PROPERTIES(ha_columnstore PROPERTIES PREFIX "") +if (COMMAND mysql_add_plugin) + mysql_add_plugin(columnstore ${libcalmysql_SRCS} STORAGE_ENGINE MODULE_ONLY DEFAULT + LINK_LIBRARIES ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool + COMPONENT columnstore-engine) +else () + add_library(ha_columnstore SHARED ${libcalmysql_SRCS}) + SET_TARGET_PROPERTIES(ha_columnstore PROPERTIES PREFIX "") -target_link_libraries(ha_columnstore ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) + target_link_libraries(ha_columnstore ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) -set_target_properties(ha_columnstore PROPERTIES VERSION 1.0.0 SOVERSION 1) + set_target_properties(ha_columnstore PROPERTIES VERSION 1.0.0 SOVERSION 1) -install(TARGETS ha_columnstore DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) + install(TARGETS ha_columnstore DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) +endif () install(FILES syscatalog_mysql.sql dumpcat_mysql.sql calsetuserpriority.sql diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 288d2bada..79b26fd6d 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -73,7 +73,7 @@ #include "liboamcpp.h" #include "crashtrace.h" #include "utils_utf8.h" -#include "config.h" +#include "mcsconfig.h" #if defined(SKIP_OAM_INIT) #include "dbrm.h" diff --git a/config.h.in b/mcsconfig.h.in similarity index 99% rename from config.h.in rename to mcsconfig.h.in index e2b9b6c1e..4ee8e9ee6 100644 --- a/config.h.in +++ b/mcsconfig.h.in @@ -1,6 +1,6 @@ -/* config.h.cmake */ -#ifndef TEST_CONFIG_H -#define TEST_CONFIG_H +/* mcsconfig.h.cmake */ +#ifndef TEST_MCSCONFIG_H +#define TEST_MCSCONFIG_H /* Define to 1 to let the system come up without using OAM */ #cmakedefine SKIP_OAM_INIT 1 diff --git a/oam/oamcpp/CMakeLists.txt b/oam/oamcpp/CMakeLists.txt index 72d6cdae2..413673504 100644 --- a/oam/oamcpp/CMakeLists.txt +++ b/oam/oamcpp/CMakeLists.txt @@ -16,4 +16,3 @@ set_target_properties(oamcpp PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS oamcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) - diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 16ba7aec1..ca5132718 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -56,7 +56,7 @@ #ifdef _MSC_VER #include "idbregistry.h" #endif -#include "config.h" +#include "mcsconfig.h" #include "installdir.h" #include "dbrm.h" #include "sessionmanager.h" diff --git a/oam/oamcpp/oamcache.cpp b/oam/oamcpp/oamcache.cpp index 8e6c39e11..bf15b000b 100644 --- a/oam/oamcpp/oamcache.cpp +++ b/oam/oamcpp/oamcache.cpp @@ -34,7 +34,7 @@ using namespace boost; #include "exceptclasses.h" #include "configcpp.h" #include "installdir.h" -#include "config.h" +#include "mcsconfig.h" namespace { diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index ad94cabc8..93cb23ada 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -29,7 +29,7 @@ #include #include -#include "config.h" +#include "mcsconfig.h" #include "liboamcpp.h" #include "configcpp.h" #include "installdir.h" diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index ea1c221cd..cdaaa9b8a 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -33,7 +33,7 @@ extern int h_errno; #include #include -#include "config.h" +#include "mcsconfig.h" #include "sessionmanager.h" #include "dbrm.h" #include "messagequeue.h" diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index f8406d174..2df779195 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -25,7 +25,7 @@ #include -#include "config.h" +#include "mcsconfig.h" #include "configcpp.h" using namespace config; diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 906a62168..3266bac8d 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -47,7 +47,7 @@ #include #include -#include "config.h" +#include "mcsconfig.h" #include "liboamcpp.h" #include "configcpp.h" diff --git a/oamapps/postConfigure/mycnfUpgrade.cpp b/oamapps/postConfigure/mycnfUpgrade.cpp index 0297c715f..33f2f2f91 100644 --- a/oamapps/postConfigure/mycnfUpgrade.cpp +++ b/oamapps/postConfigure/mycnfUpgrade.cpp @@ -51,7 +51,7 @@ #include #include "liboamcpp.h" #include "installdir.h" -#include "config.h" +#include "mcsconfig.h" using namespace std; using namespace oam; diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 10d602b17..bfaeee084 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -68,7 +68,7 @@ #include #include -#include "config.h" +#include "mcsconfig.h" #include "columnstoreversion.h" #include "liboamcpp.h" #include "configcpp.h" diff --git a/primitives/blockcache/iomanager.cpp b/primitives/blockcache/iomanager.cpp index 716b0b69d..097f3cad6 100644 --- a/primitives/blockcache/iomanager.cpp +++ b/primitives/blockcache/iomanager.cpp @@ -28,7 +28,7 @@ // // -#include "config.h" +#include "mcsconfig.h" #define _FILE_OFFSET_BITS 64 #define _LARGEFILE64_SOURCE diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index aa7877148..faffb62fd 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -25,7 +25,7 @@ #include #include "columnstoreversion.h" -#include "config.h" +#include "mcsconfig.h" #include "processmanager.h" #include "installdir.h" #include "dbrm.h" diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 121318758..4e88bd752 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -28,7 +28,7 @@ #include #include "columnstoreversion.h" -#include "config.h" +#include "mcsconfig.h" #include "IDBDataFile.h" #include "IDBPolicy.h" #include "processmonitor.h" diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index c944e8f0c..3109efd73 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -95,7 +95,7 @@ add_custom_command( # The includes and lib linkages required to link against cloudio ... # pretty crazy. When lib dependencies are eventually config'd right, # change this to only include and link against cloudio. -include_directories(${CMAKE_SOURCE_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES}) +include_directories(${ENGINE_SRC_DIR}/utils/cloudio ${ENGINE_COMMON_INCLUDES}) add_executable(smcat src/smcat.cpp) target_link_libraries(smcat storagemanager cloudio ${ENGINE_LDFLAGS} diff --git a/storage-manager/src/Config.cpp b/storage-manager/src/Config.cpp index 48fa663f7..2ba96e3e7 100644 --- a/storage-manager/src/Config.cpp +++ b/storage-manager/src/Config.cpp @@ -19,7 +19,7 @@ #include "Config.h" // This one is the build system config -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/tools/clearShm/main.cpp b/tools/clearShm/main.cpp index cf3a7e92b..f1a33ef78 100644 --- a/tools/clearShm/main.cpp +++ b/tools/clearShm/main.cpp @@ -18,7 +18,7 @@ // $Id: main.cpp 2101 2013-01-21 14:12:52Z rdempsey $ -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/tools/dbbuilder/dbbuilder.cpp b/tools/dbbuilder/dbbuilder.cpp index ed9153690..334e05333 100644 --- a/tools/dbbuilder/dbbuilder.cpp +++ b/tools/dbbuilder/dbbuilder.cpp @@ -28,7 +28,7 @@ using namespace std; #include -#include "config.h" +#include "mcsconfig.h" #include "dbbuilder.h" #include "systemcatalog.h" #include "liboamcpp.h" diff --git a/tools/qfe/returnedrows.cpp b/tools/qfe/returnedrows.cpp index da356b211..fc631993a 100644 --- a/tools/qfe/returnedrows.cpp +++ b/tools/qfe/returnedrows.cpp @@ -9,7 +9,7 @@ using namespace std; using namespace boost; #ifndef _MSC_VER -#include "config.h" +#include "mcsconfig.h" #endif #include "socktype.h" diff --git a/tools/qfe/server.cpp b/tools/qfe/server.cpp index 2bf4e187e..e38a86ab5 100644 --- a/tools/qfe/server.cpp +++ b/tools/qfe/server.cpp @@ -55,7 +55,7 @@ using namespace std; using namespace boost; #ifndef _MSC_VER -#include "config.h" +#include "mcsconfig.h" #endif #include "socktype.h" diff --git a/tools/qfe/socketio.cpp b/tools/qfe/socketio.cpp index 0bd6981d4..c6ccf367c 100644 --- a/tools/qfe/socketio.cpp +++ b/tools/qfe/socketio.cpp @@ -22,7 +22,7 @@ using namespace std; using namespace boost; #ifndef _MSC_VER -#include "config.h" +#include "mcsconfig.h" #endif #include "exceptclasses.h" diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index 7ef9ed0ab..91f1d936f 100755 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -1,4 +1,4 @@ -include_directories(${ENGINE_COMMON_INCLUDES} ${CMAKE_SOURCE_DIR}/storage-manager/include) +include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_SRC_DIR}/storage-manager/include) set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp cloud_plugin.cpp) diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 2e5c02795..1da1b22c8 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -19,7 +19,7 @@ * $Id: configcpp.cpp 3899 2013-06-17 20:54:10Z rdempsey $ * ******************************************************************************************/ -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/utils/configcpp/configstream.cpp b/utils/configcpp/configstream.cpp index 1bf85343f..2ad5aea96 100644 --- a/utils/configcpp/configstream.cpp +++ b/utils/configcpp/configstream.cpp @@ -19,7 +19,7 @@ * $Id$ * ******************************************************************************************/ -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/utils/configcpp/xmlparser.cpp b/utils/configcpp/xmlparser.cpp index 9ae0690e1..7b16af9d5 100644 --- a/utils/configcpp/xmlparser.cpp +++ b/utils/configcpp/xmlparser.cpp @@ -19,7 +19,7 @@ * $Id$ * ******************************************************************************************/ -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index 70e4b54e3..c83895a7b 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -114,4 +114,3 @@ set_target_properties(funcexp PROPERTIES VERSION 1.0.0 SOVERSION 1) install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) - diff --git a/utils/loggingcpp/idberrorinfo.cpp b/utils/loggingcpp/idberrorinfo.cpp index a2e9056d2..fd562ae1d 100644 --- a/utils/loggingcpp/idberrorinfo.cpp +++ b/utils/loggingcpp/idberrorinfo.cpp @@ -34,7 +34,7 @@ using namespace std; #include using namespace boost; -#include "config.h" +#include "mcsconfig.h" #include "configcpp.h" using namespace config; #include "loggingid.h" diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp index 7f1468594..5b7ec6c4f 100644 --- a/utils/loggingcpp/message.cpp +++ b/utils/loggingcpp/message.cpp @@ -34,7 +34,7 @@ using namespace std; #include using namespace boost; -#include "config.h" +#include "mcsconfig.h" #include "configcpp.h" using namespace config; #include "messageobj.h" diff --git a/utils/messageqcpp/compressed_iss.cpp b/utils/messageqcpp/compressed_iss.cpp index ae28577ac..26af982e3 100644 --- a/utils/messageqcpp/compressed_iss.cpp +++ b/utils/messageqcpp/compressed_iss.cpp @@ -20,7 +20,7 @@ * * ***********************************************************************/ -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp index d94a3eb94..ef6782e68 100644 --- a/utils/messageqcpp/inetstreamsocket.cpp +++ b/utils/messageqcpp/inetstreamsocket.cpp @@ -44,7 +44,7 @@ of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. */ -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/utils/multicast/socklib.h b/utils/multicast/socklib.h index 0c75af389..28d2b8f4d 100644 --- a/utils/multicast/socklib.h +++ b/utils/multicast/socklib.h @@ -20,7 +20,7 @@ #ifndef UDPCAST_CONFIG_H # define UDPCAST_CONFIG_H -# include "config.h" +# include "mcsconfig.h" #endif #include diff --git a/versioning/BRM/brmtypes.cpp b/versioning/BRM/brmtypes.cpp index b614bba03..848cecdb1 100644 --- a/versioning/BRM/brmtypes.cpp +++ b/versioning/BRM/brmtypes.cpp @@ -24,7 +24,7 @@ * Definitions of the functions declared in brmtypes.h */ -#include "config.h" +#include "mcsconfig.h" #include #include diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 90f4fbaf9..07af0112c 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -37,7 +37,7 @@ using namespace std; #include "boost/filesystem/operations.hpp" using namespace boost; -#include "config.h" +#include "mcsconfig.h" #include "installdir.h" #include "configcpp.h" diff --git a/writeengine/server/we_dataloader.cpp b/writeengine/server/we_dataloader.cpp index f2239458b..10a0d0c76 100644 --- a/writeengine/server/we_dataloader.cpp +++ b/writeengine/server/we_dataloader.cpp @@ -25,7 +25,7 @@ * Author: Boby Paul: bpaul@calpont.com */ -#include "config.h" // Used to pickup STRERROR_R_CHAR_P definition +#include "mcsconfig.h" // Used to pickup STRERROR_R_CHAR_P definition #include #include diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index dadec77da..c550c1050 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -27,7 +27,7 @@ #ifdef _MSC_VER #include #endif -#include "config.h" +#include "mcsconfig.h" #include "we_convertor.h" diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 1b181ac22..076882268 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -17,7 +17,7 @@ // $Id: we_fileop.cpp 4737 2013-08-14 20:45:46Z bwilkinson $ -#include "config.h" +#include "mcsconfig.h" #include #include #include diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 14b055a33..0ef4064fe 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -43,7 +43,7 @@ using namespace std; #include using namespace boost; -#include "config.h" +#include "mcsconfig.h" #include "configcpp.h" using namespace config; From 7489d0bfd02a53c0435ff564767a20f0eebfca4b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 27 Nov 2019 08:21:58 +0000 Subject: [PATCH 33/35] MCOL-3625 Rename packages Rename packages to MariaDB-columnstore-engine, MariaDB-columnstore-libs and MariaDB-columnstore-platform. Also add the "columnstore-" prefix the the components so that MariaDB's packaging system understands then and add a line to include them in MariaDB's packaging. In addition * Fix S3 building for dist source build * Fix Debian 10 dependency issue * Fix git handling for dist builds * Add support for MariaDB's RPM building * Use MariaDB's PCRE and readline * Removes a few dead files * Fix Boost noncopyable includes --- CMakeLists.txt | 208 ++++++++++-------- cmake/columnstore_version.cmake | 36 +-- cmake/cpackEngineDEB.cmake | 18 +- cmake/cpackEngineRPM.cmake | 18 +- columnstoreversion.h.in | 2 +- dbcon/ddlpackage/CMakeLists.txt | 4 +- dbcon/ddlpackageproc/CMakeLists.txt | 4 +- dbcon/dmlpackage/CMakeLists.txt | 4 +- dbcon/dmlpackageproc/CMakeLists.txt | 4 +- dbcon/execplan/CMakeLists.txt | 4 +- dbcon/joblist/CMakeLists.txt | 6 +- dbcon/joblist/expressionstep.cpp | 2 +- dbcon/joblist/groupconcat.cpp | 2 +- dbcon/joblist/jlf_tuplejoblist.cpp | 18 +- dbcon/joblist/joblistfactory.cpp | 10 +- dbcon/joblist/windowfunctionstep.cpp | 4 +- dbcon/mysql/CMakeLists.txt | 12 +- ddlproc/CMakeLists.txt | 2 +- dmlproc/CMakeLists.txt | 2 +- exemgr/CMakeLists.txt | 2 +- oam/cloud/CMakeLists.txt | 2 +- oam/etc/CMakeLists.txt | 2 +- oam/install_scripts/CMakeLists.txt | 6 +- oam/oamcpp/CMakeLists.txt | 4 +- oam/post/CMakeLists.txt | 2 +- oamapps/alarmmanager/CMakeLists.txt | 4 +- oamapps/columnstoreDB/CMakeLists.txt | 2 +- oamapps/columnstoreSupport/CMakeLists.txt | 6 +- .../columnstoreSupport/columnstoreSupport.cpp | 2 +- oamapps/mcsadmin/CMakeLists.txt | 4 +- oamapps/mcsadmin/mcsadmin.h | 4 +- oamapps/postConfigure/CMakeLists.txt | 22 +- oamapps/postConfigure/helpers.cpp | 2 +- oamapps/postConfigure/postConfigure.cpp | 4 +- oamapps/serverMonitor/CMakeLists.txt | 2 +- primitives/primproc/CMakeLists.txt | 2 +- procmgr/CMakeLists.txt | 2 +- procmon/CMakeLists.txt | 2 +- storage-manager/CMakeLists.txt | 8 +- storage-manager/src/Ownership.h | 1 + storage-manager/src/ThreadPool.h | 1 + tools/clearShm/CMakeLists.txt | 2 +- tools/cleartablelock/CMakeLists.txt | 2 +- tools/configMgt/CMakeLists.txt | 2 +- tools/configMgt/configure.cpp | 4 +- tools/cplogger/CMakeLists.txt | 2 +- tools/dbbuilder/CMakeLists.txt | 2 +- tools/dbloadxml/CMakeLists.txt | 2 +- tools/ddlcleanup/CMakeLists.txt | 2 +- tools/editem/CMakeLists.txt | 2 +- tools/getConfig/CMakeLists.txt | 2 +- tools/idbmeminfo/CMakeLists.txt | 2 +- tools/setConfig/CMakeLists.txt | 4 +- tools/viewtablelock/CMakeLists.txt | 2 +- utils/batchloader/CMakeLists.txt | 4 +- utils/cacheutils/CMakeLists.txt | 4 +- utils/cloudio/CMakeLists.txt | 4 +- utils/clusterTester/CMakeLists.txt | 4 +- utils/common/CMakeLists.txt | 5 +- utils/compress/CMakeLists.txt | 4 +- utils/configcpp/CMakeLists.txt | 4 +- utils/dataconvert/CMakeLists.txt | 4 +- utils/ddlcleanup/CMakeLists.txt | 4 +- utils/funcexp/CMakeLists.txt | 4 +- utils/idbdatafile/CMakeLists.txt | 4 +- utils/joiner/CMakeLists.txt | 4 +- utils/libmarias3/CMakeLists.txt | 8 +- utils/libmysql_client/CMakeLists.txt | 4 +- utils/loggingcpp/CMakeLists.txt | 6 +- utils/messageqcpp/CMakeLists.txt | 4 +- utils/querystats/CMakeLists.txt | 4 +- utils/querytele/CMakeLists.txt | 4 +- utils/regr/CMakeLists.txt | 8 +- utils/rowgroup/CMakeLists.txt | 4 +- utils/rwlock/CMakeLists.txt | 4 +- utils/threadpool/CMakeLists.txt | 4 +- utils/thrift/CMakeLists.txt | 4 +- utils/udfsdk/CMakeLists.txt | 8 +- utils/windowfunction/CMakeLists.txt | 6 +- versioning/BRM/CMakeLists.txt | 18 +- writeengine/bulk/CMakeLists.txt | 2 +- writeengine/client/CMakeLists.txt | 4 +- writeengine/redistribute/CMakeLists.txt | 4 +- writeengine/server/CMakeLists.txt | 2 +- writeengine/shared/CMakeLists.txt | 2 +- writeengine/splitter/CMakeLists.txt | 2 +- writeengine/wrapper/CMakeLists.txt | 4 +- 87 files changed, 294 insertions(+), 337 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be77d6c99..3211cd5fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,17 +35,14 @@ else() set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) endif(CCACHE_FOUND) endif() -# Distinguish between community and non-community builds, with the -# default being a community build. This does not impact the feature -# set that will be compiled in; it's merely provided as a hint to -# custom packaging steps. -OPTION(COMMUNITY_BUILD "Set to true if this is a community build" ON) -IF(NOT CMAKE_BUILD_TYPE) - SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING - "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) -ENDIF(NOT CMAKE_BUILD_TYPE) -SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") +IF(NOT INSTALL_LAYOUT) + IF(NOT CMAKE_BUILD_TYPE) + SET(CMAKE_BUILD_TYPE RELWITHDEBINFO CACHE STRING + "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) + ENDIF(NOT CMAKE_BUILD_TYPE) + SET_PROPERTY(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") +ENDIF() if(SERVER_BUILD_DIR) if (NOT IS_ABSOLUTE ${SERVER_BUILD_DIR}) @@ -77,18 +74,18 @@ SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib) SET (ENGINE_SYSCONFDIR "/etc") SET (ENGINE_DATADIR "/var/lib/columnstore") -IF (RPM) +IF (INSTALL_LAYOUT) # We are building from MariaDB server submodule if this is set - if (INSTALL_LAYOUT) - SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR_RPM}") - SET(MARIADB_MYCNFDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SYSCONF2DIR_RPM}") - SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR_RPM}") - SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR_RPM}") - SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") - SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR_RPM}") - SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR_RPM}") - SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") - else () + SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR}") + SET(MARIADB_MYCNFDIR "${INSTALL_SYSCONF2DIR}") + SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR}") + SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR}") + SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") + SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR}") + SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR}") + SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") +ELSE () + IF (RPM) SET(MARIADB_PLUGINDIR "/usr/lib64/mysql/plugin") SET(MARIADB_MYCNFDIR "/etc/my.cnf.d") SET (ENGINE_LIBDIR "/usr/lib64") @@ -97,18 +94,7 @@ IF (RPM) SET (ENGINE_MANDIR "/usr/share/man") SET (ENGINE_SBINDIR "/usr/sbin") SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") - endif () -ELSEIF (DEB) - if (INSTALL_LAYOUT) - SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR_DEB}") - SET(MARIADB_MYCNFDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SYSCONF2DIR_DEB}") - SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR_DEB}") - SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR_DEB}") - SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") - SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR_DEB}") - SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR_DEB}") - SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") - else () + ELSEIF (DEB) SET(MARIADB_PLUGINDIR "/usr/lib/mysql/plugin") SET(MARIADB_MYCNFDIR "/etc/mysql/conf.d") SET (ENGINE_LIBDIR "/usr/lib") @@ -117,18 +103,7 @@ ELSEIF (DEB) SET (ENGINE_MANDIR "/usr/share/man") SET (ENGINE_SBINDIR "/usr/sbin") SET (ENGINE_SUPPORTDIR "/usr/share/columnstore") - endif () -ELSE() - if (INSTALL_LAYOUT) - SET(MARIADB_PLUGINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_PLUGINDIR_STANDALONE}") - SET(MARIADB_MYCNFDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SYSCONF2DIR_STANDALONE}") - SET(ENGINE_LIBDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_LIBDIR_STANDALONE}") - SET(ENGINE_BINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_BINDIR_STANDALONE}") - SET(ENGINE_INCDIR "${CMAKE_INSTALL_PREFIX}/include") - SET(ENGINE_MANDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_MANDIR_STANDALONE}") - SET(ENGINE_SBINDIR "${CMAKE_INSTALL_PREFIX}/${INSTALL_SBINDIR_STANDALONE}") - SET(ENGINE_SUPPORTDIR "${CMAKE_INSTALL_PREFIX}/share/columnstore") - else () + ELSE () # TODO: prefix should probably apply here SET(MARIADB_PLUGINDIR "/usr/local/lib/mysql/plugin") SET(MARIADB_MYCNFDIR "/etc/mysql") @@ -138,18 +113,20 @@ ELSE() SET (ENGINE_MANDIR "/usr/local/man") SET (ENGINE_SBINDIR "/usr/local/sbin") SET (ENGINE_SUPPORTDIR "/usr/local/share/columnstore") - endif () -ENDIF() + ENDIF () +ENDIF () SET_PROPERTY(DIRECTORY PROPERTY EP_BASE ${CMAKE_CURRENT_BINARY_DIR}/external) LIST(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) -FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) +FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time chrono) FIND_PACKAGE(BISON REQUIRED) check_cxx_source_compiles("#include \n void main(){}" HAS_STD_FILESYSTEM) check_cxx_source_compiles("#include \n void main(){}" HAS_STD_EXPERIMENTAL_FILESYSTEM) +SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + INCLUDE(columnstore_version) SET (PACKAGE columnstore) @@ -160,9 +137,6 @@ SET (PACKAGE_URL "") SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION}) - -SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - INCLUDE (configureEngine) @@ -191,36 +165,40 @@ if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") endif() -INCLUDE(check_compiler_flag) +IF (NOT INSTALL_LAYOUT) + INCLUDE(check_compiler_flag) -MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -std=c++11 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) -MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -std=c++11 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) + MY_CHECK_AND_SET_COMPILER_FLAG("-g -O3 -std=c++11 -fno-omit-frame-pointer -fno-strict-aliasing -Wall -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DDBUG_OFF -DHAVE_CONFIG_H" RELEASE RELWITHDEBINFO MINSIZEREL) + MY_CHECK_AND_SET_COMPILER_FLAG("-ggdb3 -std=c++11 -fno-omit-frame-pointer -fno-tree-vectorize -D_GLIBCXX_ASSERTIONS -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H" DEBUG) -# enable security hardening features, like most distributions do -# in our benchmarks that costs about ~1% of performance, depending on the load -IF(CMAKE_C_COMPILER_VERSION VERSION_LESS "4.6") - SET(security_default OFF) -ELSE() - SET(security_default ON) + # enable security hardening features, like most distributions do + # in our benchmarks that costs about ~1% of performance, depending on the load + IF(CMAKE_C_COMPILER_VERSION VERSION_LESS "4.6") + SET(security_default OFF) + ELSE() + SET(security_default ON) + ENDIF() + OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default}) + OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF) + IF(SECURITY_HARDENED) + # security-enhancing flags + MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC") + MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now") + MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4") + MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO) + MY_CHECK_AND_SET_COMPILER_FLAG("-fexceptions") + IF(SECURITY_HARDENED_NEW) + MY_CHECK_AND_SET_COMPILER_FLAG("-mcet -fcf-protection") + MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector-strong") + MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-clash-protection") + ENDIF() + ENDIF() +ELSE () + # Remove visibility flag for now as it breaks Ubuntu 18.05 and we need to + # fix our libraries anyway + STRING(REPLACE "-fvisibility=hidden" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) ENDIF() -OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protector, relro, etc)" ${security_default}) -OPTION(SECURITY_HARDENED_NEW "Use new security-enhancing compilier features" OFF) -IF(SECURITY_HARDENED) - # security-enhancing flags - MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4") - MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO) - MY_CHECK_AND_SET_COMPILER_FLAG("-fexceptions") - IF(SECURITY_HARDENED_NEW) - MY_CHECK_AND_SET_COMPILER_FLAG("-mcet -fcf-protection") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector-strong") - MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-clash-protection") - ENDIF() -ENDIF() - SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") - SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) @@ -301,12 +279,24 @@ SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysq SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") -SET (ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre") +IF (PCRE_INCLUDES) + SET (ENGINE_SERVER_PCRE_INCLUDE "${PCRE_INCLUDES}") +ELSE () + SET (ENGINE_SERVER_PCRE_INCLUDE "${SERVER_BUILD_INCLUDE_DIR}/../pcre") +ENDIF () SET (ENGINE_SERVER_WSREP_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/include") SET (ENGINE_SERVER_WSREP_API_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/wsrep-lib/wsrep-API/v26/") SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk") -SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) +SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) + +IF (MY_READLINE_INCLUDE_DIR) + SET (ENGINE_DEFAULT_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${MY_READLINE_INCLUDE_DIR}) + SET (ENGINE_READLINE_LIBRARY ${MY_READLINE_LIBRARY}) +ELSE () + SET (ENGINE_DEFAULT_INCLUDES ${ENGINE_DEFAULT_INCLUDES} "/usr/include/readline/") + SET (ENGINE_READLINE_LIBRARY "readline") +ENDIF () SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE}) @@ -354,27 +344,67 @@ IF( WITH_SHARED_COMP_TESTS ) ADD_SUBDIRECTORY(writeengine/shared) ENDIF( WITH_SHARED_COMP_TESTS ) -exec_program("git" - ${CMAKE_CURRENT_SOURCE_DIR} - ARGS "describe --match=NeVeRmAtCh --always --dirty" - OUTPUT_VARIABLE GIT_VERSION) +find_package(Git QUIET) + +IF (GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git) + exec_program("git" + ${CMAKE_CURRENT_SOURCE_DIR} + ARGS "describe --match=NeVeRmAtCh --always --dirty" + OUTPUT_VARIABLE GIT_VERSION) +ELSE () + SET(GIT_VERSION "source") +ENDIF () # releasenum is used by external scripts for various tasks. Leave it alone. CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-platform) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mcsconfig.h.in ${CMAKE_CURRENT_BINARY_DIR}/mcsconfig.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-platform) -# Do this or when MariaDB builds us we don't have GenError which is required for these IF (INSTALL_LAYOUT) + # Do this or when MariaDB builds us we don't have GenError which is required for these ADD_DEPENDENCIES(udf_mysql GenError) ADD_DEPENDENCIES(funcexp GenError) ADD_DEPENDENCIES(oamcpp GenError) ADD_DEPENDENCIES(regr_mysql GenError) -ENDIF () + # Component columnstore-engine automatically added by mysql_add_plugin() + SET(CPACK_COMPONENTS_ALL ${CPACK_COMPONENTS_ALL} columnstore-platform columnstore-libs) + # Don't know why, but this doesn't work if you make the above line PARENT_SCOPE + SET(CPACK_COMPONENTS_ALL ${CPACK_COMPONENTS_ALL} PARENT_SCOPE) + IF (RPM) + SET(ignored + "%ignore ${CMAKE_INSTALL_PREFIX}" + "%ignore ${CMAKE_INSTALL_PREFIX}/local" + "%ignore ${CMAKE_INSTALL_PREFIX}/bin" + "%ignore ${CMAKE_INSTALL_PREFIX}/lib" + "%ignore ${CMAKE_INSTALL_PREFIX}/sbin" + "%ignore ${CMAKE_INSTALL_PREFIX}/lib64/mysql" + "%ignore ${CMAKE_INSTALL_PREFIX}/lib64/mysql/plugin" + "%ignore /var/lib" + "%ignore /var" + ) + SET(CPACK_RPM_columnstore-platform_USER_FILELIST ${ignored} PARENT_SCOPE) + SET(CPACK_RPM_columnstore-libs_USER_FILELIST ${ignored} PARENT_SCOPE) + SET(CPACK_RPM_columnstore-engine_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSCONF2DIR}/*" PARENT_SCOPE) -INCLUDE(cpackEngineRPM) -INCLUDE(cpackEngineDEB) + SET(SUSE_VERSION_NUMBER OFF) + IF (EXISTS "/etc/SuSE-release") + file (READ "/etc/SuSE-release" SUSE_VERSION) + string(REGEX MATCH "VERSION = ([0-9]+)" SUSE "${SUSE_VERSION}") + set(SUSE_VERSION_NUMBER "${CMAKE_MATCH_1}") + ENDIF () + + if (${SUSE_VERSION_NUMBER} EQUAL 12) + SETA(CPACK_RPM_columnstore-platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "MariaDB-columnstore-libs" "snappy" "jemalloc" "net-tools" PARENT_SCOPE) + else () + SETA(CPACK_RPM_columnstore-platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "MariaDB-columnstore-libs" "snappy" "jemalloc" "net-tools" PARENT_SCOPE) + endif() + ENDIF () +ELSE () + # MariaDB has its own packaging routines + INCLUDE(cpackEngineRPM) + INCLUDE(cpackEngineDEB) +ENDIF () diff --git a/cmake/columnstore_version.cmake b/cmake/columnstore_version.cmake index e8d0c7c2d..571fba591 100644 --- a/cmake/columnstore_version.cmake +++ b/cmake/columnstore_version.cmake @@ -2,7 +2,7 @@ # Generate "something" to trigger cmake rerun when VERSION changes CONFIGURE_FILE( - ${CMAKE_SOURCE_DIR}/VERSION + ${ENGINE_SRC_DIR}/VERSION ${CMAKE_BINARY_DIR}/VERSION.dep ) @@ -10,7 +10,7 @@ CONFIGURE_FILE( MACRO(COLUMNSTORE_GET_CONFIG_VALUE keyword var) IF(NOT ${var}) - FILE (STRINGS ${CMAKE_SOURCE_DIR}/VERSION str REGEX "^[ ]*${keyword}=") + FILE (STRINGS ${ENGINE_SRC_DIR}/VERSION str REGEX "^[ ]*${keyword}=") IF(str) STRING(REPLACE "${keyword}=" "" str ${str}) STRING(REGEX REPLACE "[ ].*" "" str "${str}") @@ -20,27 +20,29 @@ MACRO(COLUMNSTORE_GET_CONFIG_VALUE keyword var) ENDMACRO() MACRO(GET_COLUMNSTORE_VERSION) - COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_MAJOR" MAJOR_VERSION) - COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_MINOR" MINOR_VERSION) - COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_PATCH" PATCH_VERSION) - COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_EXTRA" EXTRA_VERSION) - COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_RELEASE" RELEASE_VERSION) + COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_MAJOR" CS_MAJOR_VERSION) + COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_MINOR" CS_MINOR_VERSION) + COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_PATCH" CS_PATCH_VERSION) + COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_EXTRA" CS_EXTRA_VERSION) + COLUMNSTORE_GET_CONFIG_VALUE("COLUMNSTORE_VERSION_RELEASE" CS_RELEASE_VERSION) -IF(NOT "${MAJOR_VERSION}" MATCHES "[0-9]+" OR - NOT "${MINOR_VERSION}" MATCHES "[0-9]+" OR - NOT "${PATCH_VERSION}" MATCHES "[0-9]+") +IF(NOT "${CS_MAJOR_VERSION}" MATCHES "[0-9]+" OR + NOT "${CS_MINOR_VERSION}" MATCHES "[0-9]+" OR + NOT "${CS_PATCH_VERSION}" MATCHES "[0-9]+") MESSAGE(FATAL_ERROR "VERSION file cannot be parsed.") ENDIF() - SET(VERSION "${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}${EXTRA_VERSION}") + SET(VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}") MESSAGE(STATUS "MariaDB-Columnstore ${VERSION}") - SET(CPACK_PACKAGE_VERSION_MAJOR ${MAJOR_VERSION}) - SET(CPACK_PACKAGE_VERSION_MINOR ${MINOR_VERSION}) - SET(CPACK_PACKAGE_VERSION_PATCH ${PATCH_VERSION}${EXTRA_VERSION}) - SET(PACKAGE_VERSION "${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}${EXTRA_VERSION}") - SET(PACKAGE_RELEASE "${RELEASE_VERSION}") + IF (NOT INSTALL_LAYOUT) + SET(CPACK_PACKAGE_VERSION_MAJOR ${CS_MAJOR_VERSION}) + SET(CPACK_PACKAGE_VERSION_MINOR ${CS_MINOR_VERSION}) + SET(CPACK_PACKAGE_VERSION_PATCH ${CS_PATCH_VERSION}${CS_EXTRA_VERSION}) + ENDIF () + SET(PACKAGE_VERSION "${CS_MAJOR_VERSION}.${CS_MINOR_VERSION}.${CS_PATCH_VERSION}${CS_EXTRA_VERSION}") + SET(PACKAGE_RELEASE "${CS_RELEASE_VERSION}") ENDMACRO() # Get columnstore version -GET_COLUMNSTORE_VERSION() \ No newline at end of file +GET_COLUMNSTORE_VERSION() diff --git a/cmake/cpackEngineDEB.cmake b/cmake/cpackEngineDEB.cmake index 090a1f8c0..9b59b4b4b 100644 --- a/cmake/cpackEngineDEB.cmake +++ b/cmake/cpackEngineDEB.cmake @@ -11,9 +11,9 @@ SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) # Note that this variable is DEB not DEBIAN! http://public.kitware.com/pipermail/cmake/2014-July/058030.html SET(CPACK_DEB_COMPONENT_INSTALL ON) -SET(CPACK_COMPONENTS_ALL platform libs storage-engine) +SET(CPACK_COMPONENTS_ALL columnstore-platform columnstore-libs columnstore-engine) -SET(CPACK_PACKAGE_NAME "mariadb-columnstore") +SET(CPACK_PACKAGE_NAME "MariaDB") SET(ENGINE_ARCH "amd64") IF (NOT CPACK_DEBIAN_PACKAGE_VERSION) @@ -54,9 +54,9 @@ SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DESCRIPTION "MariaDB Columnstore connect SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_SUMMARY "MariaDB ColumnStore: A Scale out Columnar storage engine for MariaDB") -SET(CPACK_DEBIAN_LIBS_PACKAGE_PROVIDES "mariadb-columnstore-libs") -SET(CPACK_DEBIAN_PLATFORM_PACKAGE_PROVIDES "mariadb-columnstore-platform") -SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_PROVIDES "mariadb-columnstore-storage-engine") +SET(CPACK_DEBIAN_LIBS_PACKAGE_PROVIDES "MariaDB-columnstore-libs") +SET(CPACK_DEBIAN_PLATFORM_PACKAGE_PROVIDES "MariaDB-columnstore-platform") +SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_PROVIDES "MariaDB-columnstore-engine") set(DEBIAN_VERSION_NUMBER OFF) if (EXISTS "/etc/debian_version") @@ -65,14 +65,14 @@ if (EXISTS "/etc/debian_version") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") endif () if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1, libjemalloc1 net-tools") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, MariaDB-columnstore-libs, libsnappy1, libjemalloc1") elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5, libjemalloc1 net-tools") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, MariaDB-columnstore-libs, libsnappy1v5, libreadline5, libjemalloc1") else() - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1 net-tools") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1") endif () -SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") +SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "MariaDB-columnstore-libs") set( CPACK_DEBIAN_LIBS_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/build/debian/libs/postinst;${CMAKE_CURRENT_SOURCE_DIR}/build/debian/libs/prerm;" ) diff --git a/cmake/cpackEngineRPM.cmake b/cmake/cpackEngineRPM.cmake index 3802b02bc..ac1a88569 100644 --- a/cmake/cpackEngineRPM.cmake +++ b/cmake/cpackEngineRPM.cmake @@ -8,9 +8,9 @@ SET(CPACK_PACKAGING_INSTALL_PREFIX ${INSTALL_ENGINE}) SET(CPACK_RPM_COMPONENT_INSTALL ON) -SET(CPACK_COMPONENTS_ALL platform libs storage-engine) +SET(CPACK_COMPONENTS_ALL columnstore-platform columnstore-libs columnstore-engine) -SET(CPACK_PACKAGE_NAME "mariadb-columnstore") +SET(CPACK_PACKAGE_NAME "MariaDB") SET(ENGINE_ARCH "x86_64") IF (NOT CPACK_RPM_PACKAGE_VERSION) @@ -68,9 +68,9 @@ MACRO(SETA var) ENDFOREACH() ENDMACRO(SETA) -SETA(CPACK_RPM_libs_PACKAGE_PROVIDES "mariadb-columnstore-libs") -SETA(CPACK_RPM_platform_PACKAGE_PROVIDES "mariadb-columnstore-platform") -SETA(CPACK_RPM_storage-engine_PACKAGE_PROVIDES "mariadb-columnstore-storage-engine") +SETA(CPACK_RPM_libs_PACKAGE_PROVIDES "MariaDB-columnstore-libs") +SETA(CPACK_RPM_platform_PACKAGE_PROVIDES "MariaDB-columnstore-platform") +SETA(CPACK_RPM_storage-engine_PACKAGE_PROVIDES "MariaDB-columnstore-engine") #boost is a source build in CentOS 6 so don't require it as a package @@ -87,16 +87,16 @@ IF (EXISTS "/etc/SuSE-release") set(SUSE_VERSION_NUMBER "${CMAKE_MATCH_1}") ENDIF () if (${REDHAT_VERSION_NUMBER} EQUAL 6) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "mariadb-columnstore-libs" "mariadb-columnstore-shared" "snappy" "net-tools") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "MariaDB-columnstore-libs" "MariaDB-columnstore-shared" "snappy" "net-tools") # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1" "jemalloc" "net-tools") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "MariaDB-columnstore-libs" "libsnappy1" "jemalloc" "net-tools") else () - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy" "jemalloc" "net-tools") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "MariaDB-columnstore-libs" "snappy" "jemalloc" "net-tools") endif() -SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") +SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "MariaDB-columnstore-libs") SET(CPACK_RPM_platform_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/postInstall_platform.sh) SET(CPACK_RPM_libs_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/postInstall_libs.sh) diff --git a/columnstoreversion.h.in b/columnstoreversion.h.in index 2746bd60c..c0b557ff2 100644 --- a/columnstoreversion.h.in +++ b/columnstoreversion.h.in @@ -1,7 +1,7 @@ #ifndef VERSIONNUMBER_H_ #define VERSIONNUMBER_H_ #include -const std::string columnstore_version("${VERSION}"); +const std::string columnstore_version("${PACKAGE_VERSION}"); const std::string columnstore_release("${PACKAGE_RELEASE}"); const std::string columnstore_commit_hash("${GIT_VERSION}"); #endif diff --git a/dbcon/ddlpackage/CMakeLists.txt b/dbcon/ddlpackage/CMakeLists.txt index 8bdc68b1e..4e95ff073 100644 --- a/dbcon/ddlpackage/CMakeLists.txt +++ b/dbcon/ddlpackage/CMakeLists.txt @@ -35,7 +35,5 @@ ADD_LIBRARY(ddlpackage SHARED ${CMAKE_CURRENT_SOURCE_DIR}/ddl-scan.cpp ) -SET_TARGET_PROPERTIES(ddlpackage PROPERTIES VERSION 1.0.0 SOVERSION 1) - -INSTALL(TARGETS ddlpackage DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +INSTALL(TARGETS ddlpackage DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/dbcon/ddlpackageproc/CMakeLists.txt b/dbcon/ddlpackageproc/CMakeLists.txt index 31e6af0a8..e23091615 100644 --- a/dbcon/ddlpackageproc/CMakeLists.txt +++ b/dbcon/ddlpackageproc/CMakeLists.txt @@ -16,7 +16,5 @@ add_library(ddlpackageproc SHARED ${ddlpackageproc_LIB_SRCS}) target_link_libraries(ddlpackageproc ${NETSNMP_LIBRARIES}) -set_target_properties(ddlpackageproc PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS ddlpackageproc DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS ddlpackageproc DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/dbcon/dmlpackage/CMakeLists.txt b/dbcon/dmlpackage/CMakeLists.txt index 4e711c967..b54bbc822 100644 --- a/dbcon/dmlpackage/CMakeLists.txt +++ b/dbcon/dmlpackage/CMakeLists.txt @@ -37,7 +37,5 @@ ADD_LIBRARY(dmlpackage SHARED ) -SET_TARGET_PROPERTIES(dmlpackage PROPERTIES VERSION 1.0.0 SOVERSION 1) - -INSTALL(TARGETS dmlpackage DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +INSTALL(TARGETS dmlpackage DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/dbcon/dmlpackageproc/CMakeLists.txt b/dbcon/dmlpackageproc/CMakeLists.txt index 3bede4ee9..f63db5522 100644 --- a/dbcon/dmlpackageproc/CMakeLists.txt +++ b/dbcon/dmlpackageproc/CMakeLists.txt @@ -17,8 +17,6 @@ add_library(dmlpackageproc SHARED ${dmlpackageproc_LIB_SRCS}) target_link_libraries(dmlpackageproc ${NETSNMP_LIBRARIES}) -set_target_properties(dmlpackageproc PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS dmlpackageproc DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS dmlpackageproc DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/dbcon/execplan/CMakeLists.txt b/dbcon/execplan/CMakeLists.txt index eb354691b..a038384cc 100755 --- a/dbcon/execplan/CMakeLists.txt +++ b/dbcon/execplan/CMakeLists.txt @@ -48,7 +48,5 @@ add_library(execplan SHARED ${execplan_LIB_SRCS}) target_link_libraries(execplan ${NETSNMP_LIBRARIES}) -set_target_properties(execplan PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS execplan DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS execplan DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/dbcon/joblist/CMakeLists.txt b/dbcon/joblist/CMakeLists.txt index c450b61af..072194795 100644 --- a/dbcon/joblist/CMakeLists.txt +++ b/dbcon/joblist/CMakeLists.txt @@ -59,13 +59,11 @@ set(joblist_LIB_SRCS add_library(joblist SHARED ${joblist_LIB_SRCS}) -set_target_properties(joblist PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS joblist DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS joblist DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) if (WITH_ORDERBY_UT) add_executable(job_orderby_tests orderby-tests.cpp) target_link_libraries(job_orderby_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) - install(TARGETS job_orderby_tests DESTINATION ${ENGINE_BINDIR} COMPONENT platform) + install(TARGETS job_orderby_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) endif() diff --git a/dbcon/joblist/expressionstep.cpp b/dbcon/joblist/expressionstep.cpp index f8020a6c9..e5d656936 100644 --- a/dbcon/joblist/expressionstep.cpp +++ b/dbcon/joblist/expressionstep.cpp @@ -509,7 +509,7 @@ void ExpressionStep::updateInputIndex(map& indexMap, const J CalpontSystemCatalog::OID oid = sc->oid(); CalpontSystemCatalog::OID dictOid = 0; CalpontSystemCatalog::ColType ct; - uint32_t key = fColumnKeys[distance(fColumns.begin(), it)]; + uint32_t key = fColumnKeys[std::distance(fColumns.begin(), it)]; if (sc->schemaName().empty()) { diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 283a73090..7f832946e 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -263,7 +263,7 @@ void GroupConcatInfo::mapColumns(const RowGroup& projRG) } else { - idx = distance(keys.begin(), i3); + idx = std::distance(keys.begin(), i3); } (*k)->fOrderCond.push_back(make_pair(idx, i2->second)); diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 713703ae8..dbcba47ea 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -1007,12 +1007,12 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) unsigned itInc = 1; // iterator increase number unsigned numOfStepsAddToBps = 0; // # steps to be added into TBPS - if ((distance(it, end) > 2 && + if ((std::distance(it, end) > 2 && dynamic_cast(it->get()) != NULL && (dynamic_cast((it + 1)->get()) != NULL || dynamic_cast((it + 1)->get()) != NULL) && dynamic_cast((it + 2)->get()) != NULL) || - (distance(it, end) > 1 && + (std::distance(it, end) > 1 && dynamic_cast(it->get()) != NULL && dynamic_cast((it + 1)->get()) != NULL)) { @@ -1053,7 +1053,7 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) itInc = 1; numOfStepsAddToBps = 0; } - else if (distance(begin, it) > 1 && + else if (std::distance(begin, it) > 1 && (dynamic_cast((it - 1)->get()) != NULL || dynamic_cast((it - 2)->get()) != NULL) && dynamic_cast(it->get()) != NULL) @@ -1062,14 +1062,14 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) itInc = 1; numOfStepsAddToBps = 0; } - else if (distance(it, end) > 2 && + else if (std::distance(it, end) > 2 && dynamic_cast((it + 1)->get()) != NULL && dynamic_cast((it + 2)->get()) != NULL) { itInc = 3; numOfStepsAddToBps = 3; } - else if (distance(it, end) > 3 && + else if (std::distance(it, end) > 3 && dynamic_cast((it + 1)->get()) != NULL && dynamic_cast((it + 2)->get()) != NULL && dynamic_cast((it + 3)->get()) != NULL) @@ -1077,7 +1077,7 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) itInc = 4; numOfStepsAddToBps = 4; } - else if (distance(it, end) > 3 && + else if (std::distance(it, end) > 3 && dynamic_cast((it + 1)->get()) != NULL && dynamic_cast((it + 2)->get()) != NULL && dynamic_cast((it + 3)->get()) != NULL) @@ -1085,7 +1085,7 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) itInc = 4; numOfStepsAddToBps = 4; } - else if (distance(it, end) > 4 && + else if (std::distance(it, end) > 4 && dynamic_cast((it + 1)->get()) != NULL && dynamic_cast((it + 2)->get()) != NULL && dynamic_cast((it + 3)->get()) != NULL && @@ -1094,7 +1094,7 @@ bool combineJobStepsByTable(TableInfoMap::iterator& mit, JobInfo& jobInfo) itInc = 5; numOfStepsAddToBps = 5; } - else if (distance(it, end) > 1 && + else if (std::distance(it, end) > 1 && (dynamic_cast(it->get()) != NULL || dynamic_cast(it->get()) != NULL) && dynamic_cast((it + 1)->get()) != NULL) @@ -2060,7 +2060,7 @@ uint32_t getKeyIndex(uint32_t key, const RowGroup& rg) if (i == rg.getKeys().end()) throw runtime_error("No key found."); - return distance(rg.getKeys().begin(), i); + return std::distance(rg.getKeys().begin(), i); } diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index b087c54ac..392387d4b 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -1110,11 +1110,11 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo else it = pcv.insert(pcv.end(), srcp); - projectKeys.insert(projectKeys.begin() + distance(pcv.begin(), it), tupleKey); + projectKeys.insert(projectKeys.begin() + std::distance(pcv.begin(), it), tupleKey); } else if (doDistinct) // @bug4250, move forward distinct column if necessary. { - uint32_t pos = distance(projectKeys.begin(), keyIt); + uint32_t pos = std::distance(projectKeys.begin(), keyIt); if (pos >= lastGroupByPos) { @@ -1263,11 +1263,11 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo else it = pcv.insert(pcv.end(), srcp); - projectKeys.insert(projectKeys.begin() + distance(pcv.begin(), it), tupleKey); + projectKeys.insert(projectKeys.begin() + std::distance(pcv.begin(), it), tupleKey); } else if (doDistinct) // @bug4250, move forward distinct column if necessary. { - uint32_t pos = distance(projectKeys.begin(), keyIt); + uint32_t pos = std::distance(projectKeys.begin(), keyIt); if (pos >= lastGroupByPos) { @@ -1392,7 +1392,7 @@ void changePcolStepToPcolScan(JobStepVector::iterator& it, JobStepVector::iterat { //If we have a pDictionaryScan-pColStep duo, then change the pColStep if (typeid(*(it->get())) == typeid(pDictionaryScan) && - distance(it, end) > 1 && + std::distance(it, end) > 1 && typeid(*((it + 1)->get())) == typeid(pColStep)) { ++it; diff --git a/dbcon/joblist/windowfunctionstep.cpp b/dbcon/joblist/windowfunctionstep.cpp index c09224899..ab1acd1fe 100755 --- a/dbcon/joblist/windowfunctionstep.cpp +++ b/dbcon/joblist/windowfunctionstep.cpp @@ -1647,8 +1647,8 @@ void WindowFunctionStep::sort(std::vector::iterator v, uint64_t n) } } - sort(v, distance(v, h) + 1); - sort(l, distance(l, v) + n); + sort(v, std::distance(v, h) + 1); + sort(l, std::distance(l, v) + n); } diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index ec1f84c47..6a06ad703 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -35,7 +35,7 @@ set_source_files_properties(ha_mcs.cpp PROPERTIES COMPILE_FLAGS "-fno-implicit-t if (COMMAND mysql_add_plugin) mysql_add_plugin(columnstore ${libcalmysql_SRCS} STORAGE_ENGINE MODULE_ONLY DEFAULT - LINK_LIBRARIES ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool + LINK_LIBRARIES ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool COMPONENT columnstore-engine) else () add_library(ha_columnstore SHARED ${libcalmysql_SRCS}) @@ -43,9 +43,7 @@ else () target_link_libraries(ha_columnstore ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} ${SERVER_BUILD_DIR}/libservices/libmysqlservices.a threadpool) - set_target_properties(ha_columnstore PROPERTIES VERSION 1.0.0 SOVERSION 1) - - install(TARGETS ha_columnstore DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) + install(TARGETS ha_columnstore DESTINATION ${MARIADB_PLUGINDIR} COMPONENT columnstore-engine) endif () install(FILES syscatalog_mysql.sql dumpcat_mysql.sql @@ -53,9 +51,9 @@ install(FILES syscatalog_mysql.sql calremoveuserpriority.sql calshowprocesslist.sql columnstore_info.sql - DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT storage-engine) + DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) install(PROGRAMS install_mcs_mysql.sh mysql-Columnstore - DESTINATION ${ENGINE_SBINDIR} COMPONENT storage-engine) + DESTINATION ${ENGINE_SBINDIR} COMPONENT columnstore-engine) install(FILES columnstore.cnf - DESTINATION ${MARIADB_MYCNFDIR} COMPONENT storage-engine) + DESTINATION ${MARIADB_MYCNFDIR} COMPONENT columnstore-engine) diff --git a/ddlproc/CMakeLists.txt b/ddlproc/CMakeLists.txt index 222313a69..80703b77b 100644 --- a/ddlproc/CMakeLists.txt +++ b/ddlproc/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(DDLProc ${DDLProc_SRCS}) target_link_libraries(DDLProc ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool) -install(TARGETS DDLProc DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS DDLProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/dmlproc/CMakeLists.txt b/dmlproc/CMakeLists.txt index fece5b614..267b9aee7 100644 --- a/dmlproc/CMakeLists.txt +++ b/dmlproc/CMakeLists.txt @@ -15,7 +15,7 @@ add_executable(DMLProc ${DMLProc_SRCS}) target_link_libraries(DMLProc ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${NETSNMP_LIBRARIES} threadpool ddlcleanuputil batchloader) -install(TARGETS DMLProc DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS DMLProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/exemgr/CMakeLists.txt b/exemgr/CMakeLists.txt index 4c8e7b65c..ae4ca54b4 100644 --- a/exemgr/CMakeLists.txt +++ b/exemgr/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries(ExeMgr ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIB target_include_directories(ExeMgr PRIVATE ${Boost_INCLUDE_DIRS}) -install(TARGETS ExeMgr DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS ExeMgr DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### install files ############### diff --git a/oam/cloud/CMakeLists.txt b/oam/cloud/CMakeLists.txt index 8d16f4a29..b5e7130ab 100644 --- a/oam/cloud/CMakeLists.txt +++ b/oam/cloud/CMakeLists.txt @@ -1,4 +1,4 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/MCSVolumeCmds.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/MCSVolumeCmds.sh" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/MCSInstanceCmds.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/MCSInstanceCmds.sh" @ONLY) -install(PROGRAMS MCSInstanceCmds.sh MCSVolumeCmds.sh MCSgetCredentials.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(PROGRAMS MCSInstanceCmds.sh MCSVolumeCmds.sh MCSgetCredentials.sh DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/oam/etc/CMakeLists.txt b/oam/etc/CMakeLists.txt index 735555108..3223de1d4 100644 --- a/oam/etc/CMakeLists.txt +++ b/oam/etc/CMakeLists.txt @@ -4,4 +4,4 @@ install(FILES AlarmConfig.xml Columnstore.xml ProcessConfig.xml ConsoleCmds.xml - DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT platform) + DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-platform) diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index 632f85c3a..05387c5d0 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -34,7 +34,7 @@ install(PROGRAMS columnstore-post-install disable-rep-columnstore.sh mariadb-command-line.sh mcs_module_installer.sh - DESTINATION ${ENGINE_BINDIR} COMPONENT platform) + DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) install(FILES columnstore.service columnstoreAlias @@ -44,7 +44,7 @@ install(FILES columnstore.service columnstoreLogRotate myCnf-include-args.text myCnf-exclude-args.text - DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform) + DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-platform) - install(FILES module DESTINATION ${ENGINE_DATADIR}/local COMPONENT platform) +install(FILES module DESTINATION ${ENGINE_DATADIR}/local COMPONENT columnstore-platform) diff --git a/oam/oamcpp/CMakeLists.txt b/oam/oamcpp/CMakeLists.txt index 413673504..a1fd0292e 100644 --- a/oam/oamcpp/CMakeLists.txt +++ b/oam/oamcpp/CMakeLists.txt @@ -12,7 +12,5 @@ target_link_libraries(oamcpp ) target_compile_options(oamcpp PRIVATE -Wno-unused-result) -set_target_properties(oamcpp PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS oamcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS oamcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/oam/post/CMakeLists.txt b/oam/post/CMakeLists.txt index 43de4807f..7ac1d273c 100644 --- a/oam/post/CMakeLists.txt +++ b/oam/post/CMakeLists.txt @@ -3,5 +3,5 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcstest-002.sh.in" "${CMAKE_CURRENT_ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcstest-003.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcstest-003.sh" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcstest-004.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcstest-004.sh" @ONLY) -install(PROGRAMS columnstore_functions mcstest-001.sh mcstest-002.sh mcstest-003.sh mcstest-004.sh DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT platform) +install(PROGRAMS columnstore_functions mcstest-001.sh mcstest-002.sh mcstest-003.sh mcstest-004.sh DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-platform) diff --git a/oamapps/alarmmanager/CMakeLists.txt b/oamapps/alarmmanager/CMakeLists.txt index a6b12ab75..b1e028717 100644 --- a/oamapps/alarmmanager/CMakeLists.txt +++ b/oamapps/alarmmanager/CMakeLists.txt @@ -10,8 +10,6 @@ add_library(alarmmanager SHARED ${alarmmanager_LIB_SRCS}) target_compile_options(alarmmanager PRIVATE -Wno-unused-result) -set_target_properties(alarmmanager PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS alarmmanager DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS alarmmanager DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/oamapps/columnstoreDB/CMakeLists.txt b/oamapps/columnstoreDB/CMakeLists.txt index df17ff3e2..b9e483ad3 100644 --- a/oamapps/columnstoreDB/CMakeLists.txt +++ b/oamapps/columnstoreDB/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(columnstoreDBWrite ${columnstoreDBWrite_SRCS}) target_link_libraries(columnstoreDBWrite ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS columnstoreDBWrite DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS columnstoreDBWrite DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/oamapps/columnstoreSupport/CMakeLists.txt b/oamapps/columnstoreSupport/CMakeLists.txt index 98573e2b0..5d19b0784 100644 --- a/oamapps/columnstoreSupport/CMakeLists.txt +++ b/oamapps/columnstoreSupport/CMakeLists.txt @@ -10,11 +10,11 @@ add_executable(columnstoreSupport ${columnstoreSupport_SRCS}) target_compile_options(columnstoreSupport PRIVATE -Wno-unused-result) -target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +target_link_libraries(columnstoreSupport ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS columnstoreSupport DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS columnstoreSupport DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) install(PROGRAMS alarmReport.sh bulklogReport.sh configReport.sh hardwareReport.sh logReport.sh resourceReport.sh softwareReport.sh - DESTINATION ${ENGINE_BINDIR} COMPONENT platform) + DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/oamapps/columnstoreSupport/columnstoreSupport.cpp b/oamapps/columnstoreSupport/columnstoreSupport.cpp index 93cb23ada..3c6f487a4 100644 --- a/oamapps/columnstoreSupport/columnstoreSupport.cpp +++ b/oamapps/columnstoreSupport/columnstoreSupport.cpp @@ -27,7 +27,7 @@ #include "stdio.h" #include "ctype.h" #include -#include +#include #include "mcsconfig.h" #include "liboamcpp.h" diff --git a/oamapps/mcsadmin/CMakeLists.txt b/oamapps/mcsadmin/CMakeLists.txt index 1052d77f8..63ef583a0 100644 --- a/oamapps/mcsadmin/CMakeLists.txt +++ b/oamapps/mcsadmin/CMakeLists.txt @@ -10,7 +10,7 @@ add_executable(mcsadmin ${mcsadmin_SRCS}) target_compile_options(mcsadmin PRIVATE -Wno-unused-result) -target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS} ${ENGINE_WRITE_LIBS}) +target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS} ${ENGINE_WRITE_LIBS}) -install(TARGETS mcsadmin DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS mcsadmin DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/oamapps/mcsadmin/mcsadmin.h b/oamapps/mcsadmin/mcsadmin.h index 66fda95c8..59fd366a2 100644 --- a/oamapps/mcsadmin/mcsadmin.h +++ b/oamapps/mcsadmin/mcsadmin.h @@ -39,8 +39,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/oamapps/postConfigure/CMakeLists.txt b/oamapps/postConfigure/CMakeLists.txt index 55831d259..8f573d0a8 100644 --- a/oamapps/postConfigure/CMakeLists.txt +++ b/oamapps/postConfigure/CMakeLists.txt @@ -10,9 +10,9 @@ add_executable(postConfigure ${postConfigure_SRCS}) target_compile_options(postConfigure PRIVATE -Wno-unused-result) -target_link_libraries(postConfigure ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +target_link_libraries(postConfigure ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS postConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS postConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -23,9 +23,9 @@ add_executable(columnstore_installer ${installer_SRCS}) target_compile_options(columnstore_installer PRIVATE -Wno-unused-result) -target_link_libraries(columnstore_installer ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +target_link_libraries(columnstore_installer ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS columnstore_installer DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS columnstore_installer DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -34,9 +34,9 @@ set(getMySQLpw_SRCS getMySQLpw.cpp) add_executable(getMySQLpw ${getMySQLpw_SRCS}) -target_link_libraries(getMySQLpw ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +target_link_libraries(getMySQLpw ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS getMySQLpw DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS getMySQLpw DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -45,9 +45,9 @@ install(TARGETS getMySQLpw DESTINATION ${ENGINE_BINDIR} COMPONENT platform) #add_executable(amazonInstaller ${amazonInstaller_SRCS}) -#target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} readline ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +#target_link_libraries(amazonInstaller ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${SNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -#install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +#install(TARGETS amazonInstaller DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -58,12 +58,12 @@ add_executable(mycnfUpgrade ${mycnfUpgrade_SRCS}) target_compile_options(mycnfUpgrade PRIVATE -Wno-unused-result) -target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} readline ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) +target_link_libraries(mycnfUpgrade ${ENGINE_LDFLAGS} ${ENGINE_READLINE_LIBRARY} ncurses ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS mycnfUpgrade DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### install(PROGRAMS quick_installer_single_server.sh quick_installer_multi_server.sh quick_installer_amazon.sh - DESTINATION ${ENGINE_BINDIR} COMPONENT platform) + DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index 2df779195..a61d7a1f0 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -23,7 +23,7 @@ #include #include -#include +#include #include "mcsconfig.h" #include "configcpp.h" diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bfaeee084..953f7bcc4 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -62,8 +62,8 @@ #include #include -#include -#include +#include +#include #include #include #include diff --git a/oamapps/serverMonitor/CMakeLists.txt b/oamapps/serverMonitor/CMakeLists.txt index 59fbad662..dbd4dcd53 100644 --- a/oamapps/serverMonitor/CMakeLists.txt +++ b/oamapps/serverMonitor/CMakeLists.txt @@ -22,5 +22,5 @@ target_compile_options(ServerMonitor PRIVATE -Wno-unused-result) target_link_libraries(ServerMonitor ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS ServerMonitor DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS ServerMonitor DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index 477792b0d..f96f8eecb 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -27,6 +27,6 @@ add_executable(PrimProc ${PrimProc_SRCS}) target_link_libraries(PrimProc ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} threadpool cacheutils dbbc processor) -install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS PrimProc DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/procmgr/CMakeLists.txt b/procmgr/CMakeLists.txt index 99320901d..f1bc5fa09 100644 --- a/procmgr/CMakeLists.txt +++ b/procmgr/CMakeLists.txt @@ -12,5 +12,5 @@ target_compile_options(ProcMgr PRIVATE -Wno-unused-result) target_link_libraries(ProcMgr ${ENGINE_LDFLAGS} cacheutils ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS ProcMgr DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS ProcMgr DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/procmon/CMakeLists.txt b/procmon/CMakeLists.txt index bd6efcceb..a0a8045a1 100644 --- a/procmon/CMakeLists.txt +++ b/procmon/CMakeLists.txt @@ -12,5 +12,5 @@ target_compile_options(ProcMon PRIVATE -Wno-unused-result) target_link_libraries(ProcMon ${ENGINE_LDFLAGS} cacheutils ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS ProcMon DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS ProcMon DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 3109efd73..d63fe8296 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -60,7 +60,7 @@ set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) add_library(storagemanager SHARED ${storagemanager_SRCS}) add_dependencies(storagemanager marias3) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(storagemanager boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) +target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) set_property(TARGET storagemanager PROPERTY CXX_STANDARD 11) add_executable(StorageManager src/main.cpp) @@ -126,16 +126,16 @@ target_link_libraries(smrm storagemanager cloudio install(TARGETS storagemanager LIBRARY DESTINATION ${ENGINE_LIBDIR} - COMPONENT platform + COMPONENT columnstore-platform ) install(TARGETS StorageManager smcat smput smls smrm RUNTIME DESTINATION ${ENGINE_BINDIR} - COMPONENT platform + COMPONENT columnstore-platform ) install(FILES storagemanager.cnf DESTINATION ${ENGINE_SYSCONFDIR}/columnstore - COMPONENT platform) + COMPONENT columnstore-platform) diff --git a/storage-manager/src/Ownership.h b/storage-manager/src/Ownership.h index fc53769c4..5327c940e 100644 --- a/storage-manager/src/Ownership.h +++ b/storage-manager/src/Ownership.h @@ -19,6 +19,7 @@ #define OWNERSHIP_H_ #include +#include #include #include #include "SMLogging.h" diff --git a/storage-manager/src/ThreadPool.h b/storage-manager/src/ThreadPool.h index e72780eff..3bf811a55 100644 --- a/storage-manager/src/ThreadPool.h +++ b/storage-manager/src/ThreadPool.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "SMLogging.h" namespace storagemanager diff --git a/tools/clearShm/CMakeLists.txt b/tools/clearShm/CMakeLists.txt index 301a14494..14a68ce09 100644 --- a/tools/clearShm/CMakeLists.txt +++ b/tools/clearShm/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(clearShm ${clearShm_SRCS}) target_link_libraries(clearShm ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS clearShm DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS clearShm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/cleartablelock/CMakeLists.txt b/tools/cleartablelock/CMakeLists.txt index b87712fd4..d80ca05d7 100644 --- a/tools/cleartablelock/CMakeLists.txt +++ b/tools/cleartablelock/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(cleartablelock ${cleartablelock_SRCS}) target_link_libraries(cleartablelock ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) -install(TARGETS cleartablelock DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS cleartablelock DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/configMgt/CMakeLists.txt b/tools/configMgt/CMakeLists.txt index 12eb03cb1..ac6aaa236 100644 --- a/tools/configMgt/CMakeLists.txt +++ b/tools/configMgt/CMakeLists.txt @@ -10,4 +10,4 @@ add_executable(autoConfigure ${autoConfigure_SRCS}) target_link_libraries(autoConfigure ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS autoConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS autoConfigure DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/configMgt/configure.cpp b/tools/configMgt/configure.cpp index 38a686d49..be4461cde 100644 --- a/tools/configMgt/configure.cpp +++ b/tools/configMgt/configure.cpp @@ -46,8 +46,8 @@ #include "ctype.h" #include -#include -#include +#include +#include #include "liboamcpp.h" #include "configcpp.h" diff --git a/tools/cplogger/CMakeLists.txt b/tools/cplogger/CMakeLists.txt index be05afe50..4a9af844f 100644 --- a/tools/cplogger/CMakeLists.txt +++ b/tools/cplogger/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(cplogger ${cplogger_SRCS}) target_link_libraries(cplogger ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS cplogger DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS cplogger DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/dbbuilder/CMakeLists.txt b/tools/dbbuilder/CMakeLists.txt index 1d09cd6aa..c9b5ab816 100644 --- a/tools/dbbuilder/CMakeLists.txt +++ b/tools/dbbuilder/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(dbbuilder ${dbbuilder_SRCS}) target_link_libraries(dbbuilder ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) -install(TARGETS dbbuilder DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS dbbuilder DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/dbloadxml/CMakeLists.txt b/tools/dbloadxml/CMakeLists.txt index 52532ca40..d7629c96b 100644 --- a/tools/dbloadxml/CMakeLists.txt +++ b/tools/dbloadxml/CMakeLists.txt @@ -15,5 +15,5 @@ add_executable(colxml ${colxml_SRCS}) target_link_libraries(colxml ${ENGINE_LDFLAGS} dbload ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS}) -install(TARGETS colxml DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS colxml DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/ddlcleanup/CMakeLists.txt b/tools/ddlcleanup/CMakeLists.txt index 8094aeaaa..c8cac521c 100644 --- a/tools/ddlcleanup/CMakeLists.txt +++ b/tools/ddlcleanup/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(ddlcleanup ${ddlcleanup_SRCS}) target_link_libraries(ddlcleanup ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ddlcleanuputil) -install(TARGETS ddlcleanup DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS ddlcleanup DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/editem/CMakeLists.txt b/tools/editem/CMakeLists.txt index 15d2dfe31..f7e38728d 100644 --- a/tools/editem/CMakeLists.txt +++ b/tools/editem/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(editem ${editem_SRCS}) target_link_libraries(editem ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS editem DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS editem DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/getConfig/CMakeLists.txt b/tools/getConfig/CMakeLists.txt index 8266b163b..a79409370 100644 --- a/tools/getConfig/CMakeLists.txt +++ b/tools/getConfig/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(mcsGetConfig ${getConfig_SRCS}) target_link_libraries(mcsGetConfig ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS mcsGetConfig DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS mcsGetConfig DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/idbmeminfo/CMakeLists.txt b/tools/idbmeminfo/CMakeLists.txt index 4c9908b23..905c63c95 100644 --- a/tools/idbmeminfo/CMakeLists.txt +++ b/tools/idbmeminfo/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(idbmeminfo ${idbmeminfo_SRCS}) target_link_libraries(idbmeminfo ${ENGINE_LDFLAGS}) -install(TARGETS idbmeminfo DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS idbmeminfo DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/setConfig/CMakeLists.txt b/tools/setConfig/CMakeLists.txt index 4cecd0f98..a1a0cad2c 100644 --- a/tools/setConfig/CMakeLists.txt +++ b/tools/setConfig/CMakeLists.txt @@ -11,7 +11,7 @@ add_executable(mcsSetConfig ${setConfig_SRCS}) target_link_libraries(mcsSetConfig ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS mcsSetConfig DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS mcsSetConfig DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) -install(PROGRAMS configxml.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(PROGRAMS configxml.sh DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/tools/viewtablelock/CMakeLists.txt b/tools/viewtablelock/CMakeLists.txt index 44d45bafa..24f97293a 100644 --- a/tools/viewtablelock/CMakeLists.txt +++ b/tools/viewtablelock/CMakeLists.txt @@ -10,5 +10,5 @@ add_executable(viewtablelock ${viewtablelock_SRCS}) target_link_libraries(viewtablelock ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS viewtablelock DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS viewtablelock DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/utils/batchloader/CMakeLists.txt b/utils/batchloader/CMakeLists.txt index 07ecc04e5..474133f95 100644 --- a/utils/batchloader/CMakeLists.txt +++ b/utils/batchloader/CMakeLists.txt @@ -10,8 +10,6 @@ add_library(batchloader SHARED ${batchloader_LIB_SRCS}) target_link_libraries(batchloader ${NETSNMP_LIBRARIES}) -set_target_properties(batchloader PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS batchloader DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS batchloader DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/cacheutils/CMakeLists.txt b/utils/cacheutils/CMakeLists.txt index bb8c766a6..20b85b71b 100644 --- a/utils/cacheutils/CMakeLists.txt +++ b/utils/cacheutils/CMakeLists.txt @@ -8,7 +8,5 @@ set(cacheutils_LIB_SRCS cacheutils.cpp) add_library(cacheutils SHARED ${cacheutils_LIB_SRCS}) -set_target_properties(cacheutils PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS cacheutils DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS cacheutils DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index 91f1d936f..5105f835e 100755 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -9,9 +9,7 @@ add_library(cloudio SHARED ${cloudio_LIB_SRCS}) # we should be able to reverse the dependency like so: target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp) -set_target_properties(cloudio PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS cloudio DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS cloudio DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) add_executable(cloudio_component_test component_test.cpp) diff --git a/utils/clusterTester/CMakeLists.txt b/utils/clusterTester/CMakeLists.txt index ef3830c3e..8d8a930b7 100644 --- a/utils/clusterTester/CMakeLists.txt +++ b/utils/clusterTester/CMakeLists.txt @@ -1,5 +1,5 @@ install(PROGRAMS columnstoreClusterTester.sh os_detect.sh - DESTINATION ${ENGINE_BINDIR} COMPONENT platform) - + DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index eff8be12f..17dac6385 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -15,8 +15,5 @@ add_library(common SHARED ${common_LIB_SRCS}) add_dependencies(common loggingcpp) - -set_target_properties(common PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS common DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS common DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/compress/CMakeLists.txt b/utils/compress/CMakeLists.txt index adf173c48..b2849b233 100644 --- a/utils/compress/CMakeLists.txt +++ b/utils/compress/CMakeLists.txt @@ -12,7 +12,5 @@ add_library(compress SHARED ${compress_LIB_SRCS}) target_link_libraries(compress ${SNAPPY_LIBRARIES}) -set_target_properties(compress PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS compress DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS compress DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/configcpp/CMakeLists.txt b/utils/configcpp/CMakeLists.txt index f52f2a84e..13f7b80aa 100644 --- a/utils/configcpp/CMakeLists.txt +++ b/utils/configcpp/CMakeLists.txt @@ -7,9 +7,7 @@ set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp configstream.cpp) add_library(configcpp SHARED ${configcpp_LIB_SRCS}) -set_target_properties(configcpp PROPERTIES VERSION 1.0.0 SOVERSION 1) - target_compile_definitions(configcpp PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -install(TARGETS configcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS configcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/dataconvert/CMakeLists.txt b/utils/dataconvert/CMakeLists.txt index e9a11648d..8888b441b 100644 --- a/utils/dataconvert/CMakeLists.txt +++ b/utils/dataconvert/CMakeLists.txt @@ -8,6 +8,4 @@ set(dataconvert_LIB_SRCS dataconvert.cpp) add_library(dataconvert SHARED ${dataconvert_LIB_SRCS}) -set_target_properties(dataconvert PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/ddlcleanup/CMakeLists.txt b/utils/ddlcleanup/CMakeLists.txt index d15ba02b1..7569800f1 100644 --- a/utils/ddlcleanup/CMakeLists.txt +++ b/utils/ddlcleanup/CMakeLists.txt @@ -10,7 +10,5 @@ add_library(ddlcleanuputil SHARED ${ddlcleanuputil_LIB_SRCS}) target_link_libraries(ddlcleanuputil ${NETSNMP_LIBRARIES}) -set_target_properties(ddlcleanuputil PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS ddlcleanuputil DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS ddlcleanuputil DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index c83895a7b..acee44e8d 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -110,7 +110,5 @@ add_library(funcexp SHARED ${funcexp_LIB_SRCS}) target_link_libraries(funcexp ${NETSNMP_LIBRARIES}) -set_target_properties(funcexp PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index c8f63ad37..c61547ee1 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -19,6 +19,4 @@ target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS}) target_compile_definitions(idbdatafile PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -set_target_properties(idbdatafile PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS idbdatafile DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS idbdatafile DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/joiner/CMakeLists.txt b/utils/joiner/CMakeLists.txt index cc67192cf..dd2003a1d 100644 --- a/utils/joiner/CMakeLists.txt +++ b/utils/joiner/CMakeLists.txt @@ -8,8 +8,6 @@ set(joiner_LIB_SRCS joiner.cpp tuplejoiner.cpp joinpartition.cpp) add_library(joiner SHARED ${joiner_LIB_SRCS}) -set_target_properties(joiner PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS joiner DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS joiner DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/libmarias3/CMakeLists.txt b/utils/libmarias3/CMakeLists.txt index 4cbc50a95..8695dcefd 100644 --- a/utils/libmarias3/CMakeLists.txt +++ b/utils/libmarias3/CMakeLists.txt @@ -2,13 +2,13 @@ set(S3API_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libmarias3 CACHE INTERNAL "S3API_DIR") find_package(Git QUIET) -if(GIT_FOUND) +if(GIT_FOUND AND EXISTS ${ENGINE_SRC_DIR}/.git) # Update submodules as needed option(GIT_SUBMODULE "Check submodules during build" ON) if(GIT_SUBMODULE) message(STATUS "Submodule update") execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + WORKING_DIRECTORY ${ENGINE_SRC_DIR} RESULT_VARIABLE GIT_SUBMOD_RESULT) if(NOT GIT_SUBMOD_RESULT EQUAL "0") message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") @@ -18,7 +18,7 @@ endif() include(ExternalProject) ExternalProject_Add(ms3 - DOWNLOAD_COMMAND cd ${CMAKE_SOURCE_DIR} && git submodule update --init + SOURCE_DIR ${S3API_DIR} CONFIGURE_COMMAND autoreconf -fi ${S3API_DIR} && ${S3API_DIR}/configure --enable-shared --disable-static --prefix=${CMAKE_BINARY_DIR} ${S3_CONFIGURE_OPT} BUILD_COMMAND make BUILD_IN_SOURCE 0 @@ -32,5 +32,5 @@ install(PROGRAMS ${CMAKE_BINARY_DIR}/lib/libmarias3.so.3 ${CMAKE_BINARY_DIR}/lib/libmarias3.so DESTINATION ${ENGINE_LIBDIR} - COMPONENT platform + COMPONENT columnstore-platform ) diff --git a/utils/libmysql_client/CMakeLists.txt b/utils/libmysql_client/CMakeLists.txt index 67da6461a..78324d632 100644 --- a/utils/libmysql_client/CMakeLists.txt +++ b/utils/libmysql_client/CMakeLists.txt @@ -7,7 +7,5 @@ set(libmysql_client_LIB_SRCS libmysql_client.cpp) add_library(libmysql_client SHARED ${libmysql_client_LIB_SRCS}) -set_target_properties(libmysql_client PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS libmysql_client DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS libmysql_client DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/loggingcpp/CMakeLists.txt b/utils/loggingcpp/CMakeLists.txt index b44cd8b21..491346529 100644 --- a/utils/loggingcpp/CMakeLists.txt +++ b/utils/loggingcpp/CMakeLists.txt @@ -26,9 +26,7 @@ add_library(loggingcpp SHARED -set_target_properties(loggingcpp PROPERTIES VERSION 1.0.0 SOVERSION 1) +install(TARGETS loggingcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) -install(TARGETS loggingcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) - -install(FILES MessageFile.txt ErrorMessage.txt DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT platform) +install(FILES MessageFile.txt ErrorMessage.txt DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT columnstore-platform) diff --git a/utils/messageqcpp/CMakeLists.txt b/utils/messageqcpp/CMakeLists.txt index 4f7c0e24c..149eb3847 100644 --- a/utils/messageqcpp/CMakeLists.txt +++ b/utils/messageqcpp/CMakeLists.txt @@ -16,6 +16,4 @@ set(messageqcpp_LIB_SRCS add_library(messageqcpp SHARED ${messageqcpp_LIB_SRCS}) -set_target_properties(messageqcpp PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS messageqcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS messageqcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/querystats/CMakeLists.txt b/utils/querystats/CMakeLists.txt index 0a8b6f5df..8057468b5 100644 --- a/utils/querystats/CMakeLists.txt +++ b/utils/querystats/CMakeLists.txt @@ -7,7 +7,5 @@ set(querystats_LIB_SRCS querystats.cpp) add_library(querystats SHARED ${querystats_LIB_SRCS}) -set_target_properties(querystats PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS querystats DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS querystats DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/querytele/CMakeLists.txt b/utils/querytele/CMakeLists.txt index 404c3fc73..d64a95163 100644 --- a/utils/querytele/CMakeLists.txt +++ b/utils/querytele/CMakeLists.txt @@ -14,7 +14,5 @@ set(querytele_LIB_SRCS add_library(querytele SHARED ${querytele_LIB_SRCS}) -set_target_properties(querytele PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS querytele DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS querytele DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/regr/CMakeLists.txt b/utils/regr/CMakeLists.txt index 0b858a917..7dca571f9 100755 --- a/utils/regr/CMakeLists.txt +++ b/utils/regr/CMakeLists.txt @@ -10,9 +10,7 @@ add_definitions(-DMYSQL_DYNAMIC_PLUGIN) add_library(regr SHARED ${regr_LIB_SRCS} ) -set_target_properties(regr PROPERTIES VERSION 1.1.0 SOVERSION 1) - -install(TARGETS regr DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS regr DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) @@ -20,7 +18,5 @@ set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp) add_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS}) -set_target_properties(regr_mysql PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS regr_mysql DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) +install(TARGETS regr_mysql DESTINATION ${MARIADB_PLUGINDIR} COMPONENT columnstore-engine) diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 028fa2a67..f79e757aa 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -12,7 +12,5 @@ add_library(rowgroup SHARED ${rowgroup_LIB_SRCS}) target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp) -set_target_properties(rowgroup PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/rwlock/CMakeLists.txt b/utils/rwlock/CMakeLists.txt index a6201d40f..fcde11218 100644 --- a/utils/rwlock/CMakeLists.txt +++ b/utils/rwlock/CMakeLists.txt @@ -9,7 +9,5 @@ set(rwlock_LIB_SRCS rwlock.cpp rwlock_local.cpp) add_library(rwlock SHARED ${rwlock_LIB_SRCS}) -set_target_properties(rwlock PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS rwlock DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS rwlock DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/threadpool/CMakeLists.txt b/utils/threadpool/CMakeLists.txt index 47a09d88b..6939047b9 100644 --- a/utils/threadpool/CMakeLists.txt +++ b/utils/threadpool/CMakeLists.txt @@ -8,6 +8,4 @@ set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool add_library(threadpool SHARED ${threadpool_LIB_SRCS}) -set_target_properties(threadpool PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS threadpool DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS threadpool DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/thrift/CMakeLists.txt b/utils/thrift/CMakeLists.txt index 1521cb617..d5c9a29af 100644 --- a/utils/thrift/CMakeLists.txt +++ b/utils/thrift/CMakeLists.txt @@ -16,7 +16,5 @@ add_definitions(-DTHRIFT_SQUELCH_CONSOLE_OUTPUT) add_library(thrift SHARED ${thrift_LIB_SRCS}) -set_target_properties(thrift PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS thrift DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS thrift DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/utils/udfsdk/CMakeLists.txt b/utils/udfsdk/CMakeLists.txt index 053725698..016e7d892 100755 --- a/utils/udfsdk/CMakeLists.txt +++ b/utils/udfsdk/CMakeLists.txt @@ -10,9 +10,7 @@ add_definitions(-DMYSQL_DYNAMIC_PLUGIN) add_library(udfsdk SHARED ${udfsdk_LIB_SRCS}) -set_target_properties(udfsdk PROPERTIES VERSION 1.1.0 SOVERSION 1) - -install(TARGETS udfsdk DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS udfsdk DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) @@ -20,7 +18,5 @@ set(udf_mysql_LIB_SRCS udfmysql.cpp) add_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS}) -set_target_properties(udf_mysql PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS udf_mysql DESTINATION ${MARIADB_PLUGINDIR} COMPONENT storage-engine) +install(TARGETS udf_mysql DESTINATION ${MARIADB_PLUGINDIR} COMPONENT columnstore-engine) diff --git a/utils/windowfunction/CMakeLists.txt b/utils/windowfunction/CMakeLists.txt index 77e4e86d3..9298cd1f3 100755 --- a/utils/windowfunction/CMakeLists.txt +++ b/utils/windowfunction/CMakeLists.txt @@ -26,12 +26,10 @@ set(windowfunction_LIB_SRCS add_library(windowfunction SHARED ${windowfunction_LIB_SRCS}) -set_target_properties(windowfunction PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS windowfunction DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS windowfunction DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) if (WITH_SORTING_COMPARATORS_UT) add_executable(comparators_tests comparators-tests.cpp) target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) - install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT platform) + install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) endif() diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 8182ffc33..feda0c245 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -32,9 +32,7 @@ set(brm_LIB_SRCS add_library(brm SHARED ${brm_LIB_SRCS}) -set_target_properties(brm PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS brm DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) ########### next target ############### @@ -45,7 +43,7 @@ add_executable(controllernode ${controllernode_SRCS}) target_link_libraries(controllernode ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ) -install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS controllernode DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -56,7 +54,7 @@ add_executable(workernode ${workernode_SRCS}) target_link_libraries(workernode ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS workernode DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS workernode DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -67,7 +65,7 @@ add_executable(dbrmctl ${dbrmctl_SRCS}) target_link_libraries(dbrmctl ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS}) -install(TARGETS dbrmctl DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS dbrmctl DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -78,7 +76,7 @@ add_executable(reset_locks ${reset_locks_SRCS}) target_link_libraries(reset_locks ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) -install(TARGETS reset_locks DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS reset_locks DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -89,7 +87,7 @@ add_executable(rollback ${rollback_SRCS}) target_link_libraries(rollback ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) -install(TARGETS rollback DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS rollback DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -100,7 +98,7 @@ add_executable(save_brm ${save_brm_SRCS}) target_link_libraries(save_brm ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) -install(TARGETS save_brm DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS save_brm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### next target ############### @@ -111,5 +109,5 @@ add_executable(load_brm ${load_brm_SRCS}) target_link_libraries(load_brm ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_OAM_LIBS} ${ENGINE_EXEC_LIBS} ${NETSNMP_LIBRARIES}) -install(TARGETS load_brm DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS load_brm DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/writeengine/bulk/CMakeLists.txt b/writeengine/bulk/CMakeLists.txt index bd1d83f29..dfc0e6fba 100644 --- a/writeengine/bulk/CMakeLists.txt +++ b/writeengine/bulk/CMakeLists.txt @@ -40,5 +40,5 @@ add_executable(cpimport.bin ${cpimport.bin_SRCS}) add_dependencies(cpimport.bin marias3) target_link_libraries(cpimport.bin ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${S3API_DEPS} we_bulk we_xml) -install(TARGETS cpimport.bin DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS cpimport.bin DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/writeengine/client/CMakeLists.txt b/writeengine/client/CMakeLists.txt index 2133e1632..431dd5583 100644 --- a/writeengine/client/CMakeLists.txt +++ b/writeengine/client/CMakeLists.txt @@ -10,8 +10,6 @@ add_library(writeengineclient SHARED ${writeengineclient_LIB_SRCS}) target_link_libraries(writeengineclient ${NETSNMP_LIBRARIES}) -set_target_properties(writeengineclient PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS writeengineclient DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS writeengineclient DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/writeengine/redistribute/CMakeLists.txt b/writeengine/redistribute/CMakeLists.txt index 22308ced0..f53add736 100644 --- a/writeengine/redistribute/CMakeLists.txt +++ b/writeengine/redistribute/CMakeLists.txt @@ -13,10 +13,8 @@ add_library(writeengineredistribute SHARED ${writeengineredistribute_LIB_SRCS}) target_link_libraries(writeengineredistribute ${NETSNMP_LIBRARIES}) -set_target_properties(writeengineredistribute PROPERTIES VERSION 1.0.0 SOVERSION 1) - target_compile_definitions(writeengineredistribute PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -install(TARGETS writeengineredistribute DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS writeengineredistribute DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index cb11ff375..7629eaf64 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -21,5 +21,5 @@ add_executable(WriteEngineServer ${WriteEngineServer_SRCS}) target_link_libraries(WriteEngineServer ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${NETSNMP_LIBRARIES} ${ENGINE_WRITE_LIBS} threadpool writeengineredistribute) -install(TARGETS WriteEngineServer DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS WriteEngineServer DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/writeengine/shared/CMakeLists.txt b/writeengine/shared/CMakeLists.txt index 898da28a0..0e62b7bef 100644 --- a/writeengine/shared/CMakeLists.txt +++ b/writeengine/shared/CMakeLists.txt @@ -4,7 +4,7 @@ include_directories(${ENGINE_COMMON_INCLUDES} ../dictionary) add_executable(we_shared_components_tests ./shared_components_tests.cpp) target_link_libraries(we_shared_components_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES}) -install(TARGETS we_shared_components_tests DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS we_shared_components_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) ########### install files ############### diff --git a/writeengine/splitter/CMakeLists.txt b/writeengine/splitter/CMakeLists.txt index 09aca8542..c02645534 100644 --- a/writeengine/splitter/CMakeLists.txt +++ b/writeengine/splitter/CMakeLists.txt @@ -19,5 +19,5 @@ add_executable(cpimport ${cpimport_SRCS}) target_link_libraries(cpimport ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} batchloader threadpool) -install(TARGETS cpimport DESTINATION ${ENGINE_BINDIR} COMPONENT platform) +install(TARGETS cpimport DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/writeengine/wrapper/CMakeLists.txt b/writeengine/wrapper/CMakeLists.txt index 205e3cd77..8d9266c90 100644 --- a/writeengine/wrapper/CMakeLists.txt +++ b/writeengine/wrapper/CMakeLists.txt @@ -41,6 +41,4 @@ add_library(writeengine SHARED ${writeengine_LIB_SRCS}) target_link_libraries(writeengine ${NETSNMP_LIBRARIES}) -set_target_properties(writeengine PROPERTIES VERSION 1.0.0 SOVERSION 1) - -install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) +install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) From 766d84ada8855875b3beae3737883ac216bfe1f3 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 26 Nov 2019 11:59:54 -0600 Subject: [PATCH 34/35] MCOL-3585 Enabled MCOL-498 by default for all dbroots. --- CMakeLists.txt | 5 ----- utils/idbdatafile/IDBPolicy.cpp | 7 +++---- utils/idbdatafile/IDBPolicy.h | 8 ++++---- writeengine/shared/shared_components_tests.cpp | 16 ++++++++-------- writeengine/shared/we_fileop.cpp | 4 ++-- 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e8e70be66..ef0bb09ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -316,11 +316,6 @@ ADD_SUBDIRECTORY(storage-manager) # WriteEngine component tests IF( WITH_SHARED_COMP_TESTS ) - # search for cppunit - INCLUDE (findcppunit.cmake) - if (NOT CPPUNIT_FOUND) - MESSAGE(FATAL_ERROR "CPPUnit not found please install cppunit-devel for CentOS/RedHat or libcppunit-dev for Ubuntu/Debian") - endif() ADD_SUBDIRECTORY(writeengine/shared) ENDIF( WITH_SHARED_COMP_TESTS ) diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 5bdeb2271..b88c918a5 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -296,10 +296,9 @@ void IDBPolicy::configIDBPolicy() if ( setting.length() != 0 ) { - if ( setting.size() == 3 + if ( setting.size() == 2 && ( setting[0] == 'O' || setting[0] == 'o' ) - && ( setting[1] == 'F' || setting[1] == 'f' ) - && ( setting[2] == 'F' || setting[2] == 'f' ) + && ( setting[1] == 'N' || setting[1] == 'n' ) ) { // int into uint16_t implicit conversion @@ -310,7 +309,7 @@ void IDBPolicy::configIDBPolicy() } } -void IDBPolicy::setPreallocSpace(uint16_t dbRoot) +void IDBPolicy::enablePreallocSpace(uint16_t dbRoot) { s_PreallocSpace.push_back(dbRoot); } diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index d05415117..11a534da2 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -89,7 +89,7 @@ public: /** * Checks for disk space preallocation feature status for a dbroot */ - static bool PreallocSpace(uint16_t dbRoot); + static bool PreallocSpaceDisabled(uint16_t dbRoot); /** * Accessor method that returns whether to use HDFS memory buffers @@ -137,7 +137,7 @@ public: /** * This is used in WE shared components Unit Tests */ - static void setPreallocSpace(uint16_t dbRoot); + static void enablePreallocSpace(uint16_t dbRoot); private: /** @@ -178,11 +178,11 @@ bool IDBPolicy::useCloud() // MCOL-498 Looking for dbRoot in the List set in configIDBPolicy. inline -bool IDBPolicy::PreallocSpace(uint16_t dbRoot) +bool IDBPolicy::PreallocSpaceDisabled(uint16_t dbRoot) { std::vector::iterator dbRootIter = find(s_PreallocSpace.begin(), s_PreallocSpace.end(), dbRoot); - return dbRootIter != s_PreallocSpace.end(); + return dbRootIter == s_PreallocSpace.end(); } inline diff --git a/writeengine/shared/shared_components_tests.cpp b/writeengine/shared/shared_components_tests.cpp index e569ecee6..682bf1251 100644 --- a/writeengine/shared/shared_components_tests.cpp +++ b/writeengine/shared/shared_components_tests.cpp @@ -1017,7 +1017,7 @@ public: CPPUNIT_ASSERT(rc == NO_ERROR); - CPPUNIT_ASSERT(bFile->size() == 67108864); + CPPUNIT_ASSERT(bFile->size() == 2105344); fileOp.closeFile(pFile); // file has been extended delete the file before // the second run @@ -1046,8 +1046,8 @@ public: bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0); pFile = dynamic_cast(bFile); - // disable disk space preallocation and extend - idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + // enable disk space preallocation and extend + idbdatafile::IDBPolicy::enablePreallocSpace(dbRoot); rc = fileOp.initColumnExtent(pFile, dbRoot, BYTE_PER_BLOCK, // number of blocks @@ -1060,7 +1060,7 @@ public: CPPUNIT_ASSERT(rc == NO_ERROR); - CPPUNIT_ASSERT(bFile->size() == 2105344); + CPPUNIT_ASSERT(bFile->size() == 67108864); fileOp.closeFile(pFile); // file has been extended @@ -1114,7 +1114,7 @@ public: false, false ); //enable preallocation // Check the file size and remove the file - CPPUNIT_ASSERT(bFile->size() == 67379200); + CPPUNIT_ASSERT(bFile->size() == 483328); CPPUNIT_ASSERT(rc == NO_ERROR); fileOp.deleteFile( fileName ); CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); @@ -1129,8 +1129,8 @@ public: bFile = (idbdatafile::BufferedFile*)m_dFile; CPPUNIT_ASSERT(m_dFile != NULL); - // disable preallocation and create a Dictionary - idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + // enable preallocation and create a Dictionary + idbdatafile::IDBPolicy::enablePreallocSpace(dbRoot); m_Dctnry.compressionType(1); rc = m_Dctnry.initDctnryExtent( m_dFile, dbRoot, @@ -1141,7 +1141,7 @@ public: true ); //skip preallocation // Check the size and remove the file. - CPPUNIT_ASSERT(bFile->size() == 483328); + CPPUNIT_ASSERT(bFile->size() == 67379200); CPPUNIT_ASSERT(rc == NO_ERROR); fileOp.deleteFile(fileName); CPPUNIT_ASSERT(fileOp.exists( fileName ) == false); diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 1b181ac22..c2426ca2c 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -1081,7 +1081,7 @@ int FileOp::initColumnExtent( // IMO it is better to check bool then to call a function. if ( bOptExtension ) { - bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot)) + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpaceDisabled(dbRoot)) ? bOptExtension : false; } // Reduce number of blocks allocated for abbreviated extents thus @@ -1849,7 +1849,7 @@ int FileOp::initDctnryExtent( // CS doesn't optimize non-compressed dict creation. if ( bOptExtension ) { - bOptExtension = (idbdatafile::IDBPolicy::PreallocSpace(dbRoot) + bOptExtension = (idbdatafile::IDBPolicy::PreallocSpaceDisabled(dbRoot) && m_compressionType) ? bOptExtension : false; } // Reduce number of blocks allocated for abbreviated extents thus From 7b67636c385ac777994c5fbb679d1c9fb057898c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 4 Dec 2019 13:42:06 -0500 Subject: [PATCH 35/35] MCOL-3645 - Make ExeMgr destroy joblists in a different thread Fixed a compiler warning. --- exemgr/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 5015dcc6f..7dbe7f4d4 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -1088,7 +1088,7 @@ new_plan: // msgLog is global scope, and passed by copy, so, unclear // what the warning is about. destructing++; - std::thread bgdtor([jl, &jlMutex, &jlCleanupDone, stmtID, &li, msgLog, &destructing] { + std::thread bgdtor([jl, &jlMutex, &jlCleanupDone, stmtID, &li, &destructing] { std::unique_lock scoped(jlMutex); const_cast(jl).reset(); // this happens second; does real destruction logging::Message::Args args;