From d01df951449f30aa4f9a0a26acee76180c0013e5 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 15 Jan 2020 16:31:07 -0500 Subject: [PATCH 01/84] Made the SKIP_OAM_INIT checks happen at run time vs build time. --- cmake/configureEngine.cmake | 6 +-- exemgr/main.cpp | 14 +++--- mcsconfig.h.in | 2 +- oam/oamcpp/liboamcpp.cpp | 15 +++--- oam/oamcpp/oamcache.cpp | 12 ++--- oamapps/mcsadmin/mcsadmin.cpp | 32 ++++++------- oamapps/postConfigure/postConfigure.cpp | 5 +- .../we_redistributecontrolthread.cpp | 47 +++++++++---------- writeengine/splitter/we_sdhandler.cpp | 5 +- 9 files changed, 65 insertions(+), 73 deletions(-) diff --git a/cmake/configureEngine.cmake b/cmake/configureEngine.cmake index 0814ca454..98d202345 100644 --- a/cmake/configureEngine.cmake +++ b/cmake/configureEngine.cmake @@ -716,9 +716,9 @@ IF (NOT INLINE) SET (inline "") ENDIF() -IF($ENV{SKIP_OAM_INIT}) - set(SKIP_OAM_INIT 1 CACHE BOOL "Skip OAM initialization" FORCE) -ENDIF() +#IF($ENV{SKIP_OAM_INIT}) +# set(SKIP_OAM_INIT 1 CACHE BOOL "Skip OAM initialization" FORCE) +#ENDIF() EXECUTE_PROCESS( COMMAND rm -f conftest.data conftest.file conftest.sym diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 94b9aabb7..3fa3a1c79 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -76,10 +76,7 @@ #include #include -#if defined(SKIP_OAM_INIT) #include "dbrm.h" -#endif - namespace { @@ -1646,11 +1643,12 @@ int main(int argc, char* argv[]) { } } -#if defined(SKIP_OAM_INIT) - BRM::DBRM *dbrm = new BRM::DBRM(); - dbrm->setSystemQueryReady(true); - delete dbrm; -#endif + if (getenv("SKIP_OAM_INIT")) + { + BRM::DBRM *dbrm = new BRM::DBRM(); + dbrm->setSystemQueryReady(true); + delete dbrm; + } threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); exeMgrThreadPool.setName("ExeMgrServer"); diff --git a/mcsconfig.h.in b/mcsconfig.h.in index 4ee8e9ee6..dc392d947 100644 --- a/mcsconfig.h.in +++ b/mcsconfig.h.in @@ -3,7 +3,7 @@ #define TEST_MCSCONFIG_H /* Define to 1 to let the system come up without using OAM */ -#cmakedefine SKIP_OAM_INIT 1 +// #cmakedefine SKIP_OAM_INIT 1 /* Define to 1 if you have the `alarm' function. */ #cmakedefine HAVE_ALARM 1 diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index ca5132718..5fb72f387 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1255,7 +1255,7 @@ void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string passw bool storeHostnames) { // build and send msg - int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, + int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, password, mysqlpw); if (returnStatus != API_SUCCESS) @@ -1953,9 +1953,9 @@ void Oam::getProcessConfig(SystemProcessConfig& systemprocessconfig) Config* proConfig = Config::makeConfig(ProcessConfigFile.c_str()); Config *csConfig = Config::makeConfig(); string strStorageManagerEnabled = csConfig->getConfig("StorageManager", "Enabled"); - bool storageManagerEnabled = !strStorageManagerEnabled.empty() && (strStorageManagerEnabled[0] == 'Y' || + bool storageManagerEnabled = !strStorageManagerEnabled.empty() && (strStorageManagerEnabled[0] == 'Y' || strStorageManagerEnabled[0] == 'y' || strStorageManagerEnabled[0] == 'T' || strStorageManagerEnabled[0] == 't'); - + for (int processID = 1; processID < MAX_PROCESS + 1; processID++) { ProcessConfig processconfig; @@ -1969,7 +1969,7 @@ void Oam::getProcessConfig(SystemProcessConfig& systemprocessconfig) processconfig ); // hide StorageManager from everything else if it is disabled - if (processconfig.ProcessName.empty() || + if (processconfig.ProcessName.empty() || (!storageManagerEnabled && processconfig.ProcessName == "StorageManager")) continue; @@ -2370,7 +2370,8 @@ void Oam::setProcessStatus(const std::string process, const std::string module, void Oam::processInitComplete(std::string processName, int state) { //This method takes too long on Windows and doesn't do anything there anyway... -#if !defined(_MSC_VER) && !defined(SKIP_OAM_INIT) + if (getenv("SKIP_OAM_INIT") != NULL) + return; // get current Module name string moduleName; oamModuleInfo_t st; @@ -2414,7 +2415,6 @@ void Oam::processInitComplete(std::string processName, int state) writeLog("processInitComplete: Status update failed", LOG_TYPE_ERROR ); exceptionControl("processInitComplete", API_FAILURE); -#endif } /******************************************************************** @@ -10716,7 +10716,7 @@ void Oam::mountDBRoot(dbrootList dbrootConfigList, bool mount) // nothing to do here if (DBRootStorageType == "storagemanager") return; - + string DataRedundancyConfig = "n"; try @@ -11197,4 +11197,3 @@ namespace procheartbeat */ } // end of namespace // vim:ts=4 sw=4: - diff --git a/oam/oamcpp/oamcache.cpp b/oam/oamcpp/oamcache.cpp index ed5097fda..bd2957764 100644 --- a/oam/oamcpp/oamcache.cpp +++ b/oam/oamcpp/oamcache.cpp @@ -111,7 +111,7 @@ void OamCache::checkReload() // Restore for Windows when we support multiple PMs while (it != uniquePids.end()) { -#if !defined(SKIP_OAM_INIT) + if (getenv("SKIP_OAM_INIT") == NULL) { try { @@ -180,10 +180,11 @@ void OamCache::checkReload() break; } } -#else - pmToConnectionMap[*it] = i++; - moduleIds.push_back(*it); -#endif + else + { + pmToConnectionMap[*it] = i++; + moduleIds.push_back(*it); + } it++; } @@ -346,4 +347,3 @@ string OamCache::getModuleName() } } /* namespace oam */ - diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 0b9e500cc..e2970769e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -174,11 +174,12 @@ void setupSignalHandlers() int main(int argc, char* argv[]) { -// print a warning if this is a developer build -#ifdef SKIP_OAM_INIT - cout << "SKIP_OAM_INIT is set" << endl; - sleep(2); -#endif + // print a warning if this is a developer build + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set" << endl; + sleep(2); + } #ifndef _MSC_VER setuid(0); // set effective ID to root; ignore return status @@ -246,7 +247,7 @@ int main(int argc, char* argv[]) if (user != 0) rootUser = false; - + // create/open command log file if not created logFile.open(DEFAULT_LOG_FILE.c_str(), ios::app); @@ -2622,7 +2623,7 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { string logFile = tmpDir + "/cc-stop.pdsh"; - + cmd = "pdsh -a 'columnstore stop' > " + logFile + " 2>&1"; system(cmd.c_str()); @@ -4932,7 +4933,7 @@ int processCommand(string* arguments) case 39: // getSystemDirectories { cout << endl << "System Installation and Temporary File Directories" << endl << endl; - + cout << "System Temporary File Directory = " << tmpDir << endl << endl; } break; @@ -5695,7 +5696,7 @@ int processCommand(string* arguments) umStorageNames umstoragenames; int hostArg; int dbrootPerPM = 0; - + //check if module type or module name was entered if ( arguments[1].size() == 2 ) { @@ -5711,7 +5712,7 @@ int processCommand(string* arguments) moduleType = arguments[1]; moduleCount = atoi(arguments[2].c_str()); hostArg = 4; - + // MCOL-1607. Check whether we should store host names or IP addresses. if (arguments[3] != "" && (arguments[3][0] == 'y' || arguments[3][0] == 'Y')) storeHostnames = true; @@ -5746,7 +5747,7 @@ int processCommand(string* arguments) // MCOL-1607. Check whether we should store host names or IP addresses. if (arguments[2] != "" && (arguments[2][0] == 'y' || arguments[2][0] == 'Y')) storeHostnames = true; - + if (arguments[4] != "") password = arguments[4]; else @@ -6024,7 +6025,7 @@ int processCommand(string* arguments) // prompt for IP Address string prompt = "IP Address of " + hostName + " not found, enter IP Address or enter 'abort'"; IPAddress = dataPrompt(prompt); - + if ( IPAddress == "abort" || !oam.isValidIP(IPAddress) ) return 1; } @@ -6065,7 +6066,7 @@ int processCommand(string* arguments) DataRedundancyHostname = dataPrompt(prompt); if (storeHostnames) DataRedundancyIPAddress = DataRedundancyHostname; - else + else { DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); @@ -6074,7 +6075,7 @@ int processCommand(string* arguments) // prompt for IP Address string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; DataRedundancyIPAddress = dataPrompt(prompt); - + if (!oam.isValidIP(DataRedundancyIPAddress)) return 1; } @@ -7896,7 +7897,7 @@ int processCommand(string* arguments) gracefulTemp = oam::FORCEFUL; int returnStatus = oam.restartSystem(gracefulTemp, ackTemp); switch (returnStatus) - { + { case API_SUCCESS: if ( waitForActive() ) cout << endl << " Successful restart of System " << endl << endl; @@ -10034,4 +10035,3 @@ bool SendToWES(Oam& oam, ByteStream bs) return true; } // vim:ts=4 sw=4: - diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1cb97e199..73d8d9ed2 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3730,9 +3730,8 @@ int main(int argc, char* argv[]) // startup MariaDB ColumnStore // - #ifdef SKIP_OAM_INIT - exit(0); - #endif + if (getenv("SKIP_OAM_INIT")) + exit(0); if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || pmNumber > 1 ) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 1ce786a75..f67bf9430 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -618,35 +618,35 @@ int RedistributeControlThread::executeRedistributePlan() try { // skip system status check in case no OAM -#if !defined(_MSC_VER) && !defined(SKIP_OAM_INIT) - // make sure system is in active state - bool isActive = false; - - while (!isActive) + if (getenv("SKIP_OAM_INIT") == NULL) { - bool noExcept = true; - SystemStatus systemstatus; + // make sure system is in active state + bool isActive = false; - try + while (!isActive) { - fControl->fOam->getSystemStatus(systemstatus); - } - catch (const std::exception& ex) - { - fErrorMsg += ex.what(); - noExcept = false; - } - catch (...) - { - noExcept = false; - } + bool noExcept = true; + SystemStatus systemstatus; - if (noExcept && ((isActive = (systemstatus.SystemOpState == oam::ACTIVE)) == false)) - sleep(1);; + try + { + fControl->fOam->getSystemStatus(systemstatus); + } + catch (const std::exception& ex) + { + fErrorMsg += ex.what(); + noExcept = false; + } + catch (...) + { + noExcept = false; + } + + if (noExcept && ((isActive = (systemstatus.SystemOpState == oam::ACTIVE)) == false)) + sleep(1);; + } } -#endif - if (fStopAction) return RED_EC_USER_STOP; @@ -851,4 +851,3 @@ void RedistributeControlThread::doStopAction() } // namespace // vim:ts=4 sw=4: - diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 47916d877..b23f61120 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -547,7 +547,7 @@ void WESDHandler::setup() } // getModuleStatus will take too long. Also to test in development -#if !defined(_MSC_VER) && !defined(SKIP_OAM_INIT) + if (getenv("SKIP_OAM_INIT") == NULL) { vector& aVec = fRef.fCmdArgs.getPmVec(); @@ -578,8 +578,6 @@ void WESDHandler::setup() } } } -#endif - int rtn = fDbrm.getSystemReady(); @@ -2997,4 +2995,3 @@ void WESDHandler::setDisconnectFailure(bool Flag) //------------------------------------------------------------------------------ } /* namespace WriteEngine */ - From 2691dbf79064abba32c4a72065a1e524e4619213 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 15 Jan 2020 16:52:18 -0500 Subject: [PATCH 02/84] postConfig goes down a different path "sometimes", added a SKIP_OAM_INIT check to another path to prevent steps that are redundant with our init scripts. --- oamapps/postConfigure/installer.cpp | 15 ++++++++++----- oamapps/postConfigure/postConfigure.cpp | 3 +++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 3266bac8d..8f072291b 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -183,14 +183,14 @@ int main(int argc, char* argv[]) if (p && *p) USER = p; - + string tmpDir = startup::StartUp::tmpDir(); // setup to start on reboot, for non-root amazon installs if ( !rootUser ) { system("sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); - } + } //copy Columnstore.xml.rpmsave if upgrade option is selected if ( installType == "upgrade" ) @@ -694,6 +694,12 @@ int main(int argc, char* argv[]) cmd = "chmod 755 -R /var/lib/columnstore/data1/systemFiles/dbrm > /dev/null 2>&1"; system(cmd.c_str()); } + + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + exit(0); + } string idbstartcmd = "columnstore start"; @@ -778,7 +784,7 @@ int main(int argc, char* argv[]) exit (1); } } - + cout << endl << "MariaDB ColumnStore Install Successfully Completed, System is Active" << endl << endl; cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; @@ -990,7 +996,7 @@ bool updateProcessConfig(int serverTypeInstall) */ bool makeRClocal(string moduleName, int IserverTypeInstall) { - + return true; string moduleType = moduleName.substr(0, MAX_MODULE_TYPE_SIZE); @@ -1175,4 +1181,3 @@ bool uncommentCalpontXml( string entry) } // vim:ts=4 sw=4: - diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 73d8d9ed2..b807876c3 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3731,7 +3731,10 @@ int main(int argc, char* argv[]) // if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; exit(0); + } if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || pmNumber > 1 ) From 65f6701b998aabdba8fa405295da8b58a13fd2ce Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 16 Jan 2020 10:33:30 -0500 Subject: [PATCH 03/84] Moved the early-exit code in postConfig s.t. it will still run the mysql setup steps. --- oamapps/postConfigure/installer.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 8f072291b..9a2c73b1c 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -695,12 +695,6 @@ int main(int argc, char* argv[]) system(cmd.c_str()); } - if (getenv("SKIP_OAM_INIT")) - { - cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; - exit(0); - } - string idbstartcmd = "columnstore start"; { @@ -717,6 +711,12 @@ int main(int argc, char* argv[]) mysqlSetup(); sleep(5); + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + exit(0); + } + //start on local module int rtnCode = system(idbstartcmd.c_str()); @@ -738,6 +738,12 @@ int main(int argc, char* argv[]) mysqlSetup(); sleep(5); + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + exit(0); + } + //startup mysqld and infinidb processes cout << endl; cmd = "clearShm > /dev/null 2>&1"; From 83676a1193bc989902a72f9eb6d5b63ca42ffcaa Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 17 Jan 2020 13:29:14 -0500 Subject: [PATCH 04/84] Removed the commented SKIP_OAM_INIT lines from cmake files. --- cmake/configureEngine.cmake | 4 ---- mcsconfig.h.in | 3 --- 2 files changed, 7 deletions(-) diff --git a/cmake/configureEngine.cmake b/cmake/configureEngine.cmake index 98d202345..7626f9edc 100644 --- a/cmake/configureEngine.cmake +++ b/cmake/configureEngine.cmake @@ -716,10 +716,6 @@ IF (NOT INLINE) SET (inline "") ENDIF() -#IF($ENV{SKIP_OAM_INIT}) -# set(SKIP_OAM_INIT 1 CACHE BOOL "Skip OAM initialization" FORCE) -#ENDIF() - EXECUTE_PROCESS( COMMAND rm -f conftest.data conftest.file conftest.sym WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/mcsconfig.h.in b/mcsconfig.h.in index dc392d947..76f17917a 100644 --- a/mcsconfig.h.in +++ b/mcsconfig.h.in @@ -2,9 +2,6 @@ #ifndef TEST_MCSCONFIG_H #define TEST_MCSCONFIG_H -/* Define to 1 to let the system come up without using OAM */ -// #cmakedefine SKIP_OAM_INIT 1 - /* Define to 1 if you have the `alarm' function. */ #cmakedefine HAVE_ALARM 1 From 474589e516dcf1b64c59501bfc5a103a700ddf4c Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 21 Jan 2020 11:34:01 -0800 Subject: [PATCH 05/84] MCOL-3707 Packaging pre/post functions Adds support for: * columnstore-post-install on RPM install * MCOL-3708 mcsadmin shutdown on uninstall There is a counterpart for the server tree to propagate the RPM settings up and add Debian packaging support. --- CMakeLists.txt | 4 ++++ build/preUn_platform.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9794324ea..cab7bd282 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -406,6 +406,10 @@ IF (INSTALL_LAYOUT) else () SETA(CPACK_RPM_columnstore-platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "MariaDB-columnstore-libs" "snappy" "jemalloc" "net-tools" PARENT_SCOPE) endif() + + SET(CPACK_RPM_columnstore-platform_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/build/postInstall_platform.sh PARENT_SCOPE) + SET(CPACK_RPM_columnstore-platform_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/build/preUn_platform.sh PARENT_SCOPE) + ENDIF () ELSE () # MariaDB has its own packaging routines diff --git a/build/preUn_platform.sh b/build/preUn_platform.sh index 3d5c2d200..a3cf9a653 100644 --- a/build/preUn_platform.sh +++ b/build/preUn_platform.sh @@ -1,3 +1,5 @@ + +mcsadmin shutdown y rpmmode=upgrade if [ "$1" -eq "$1" 2> /dev/null ]; then if [ $1 -ne 1 ]; then From b6dfdd6e9934e0799903fbbe14c265a98eac33d7 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Tue, 14 Jan 2020 22:28:15 -0600 Subject: [PATCH 06/84] Merge pull request #1000 from drrtuy/MCOL-1734_2 MCOL-1734 CS now behaves similar to MDB in case of NOT IN + correlated subquery if the subquery returns empty set. --- dbcon/mysql/ha_mcs_impl.cpp | 2 +- primitives/primproc/batchprimitiveprocessor.cpp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 766cfbea7..3b71dd5f9 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2303,7 +2303,7 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows *affected_rows) *affected_rows = ci->affectedRows; } - return 0; + return rc; } int ha_mcs_impl_rnd_init(TABLE* table) diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index b08c27de5..64a9d2455 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -1174,6 +1174,7 @@ void BatchPrimitiveProcessor::executeTupleJoin() break; } + if (LIKELY(!typelessJoin[j])) { //cout << "not typeless join\n"; @@ -1186,6 +1187,8 @@ void BatchPrimitiveProcessor::executeTupleJoin() largeKey = oldRow.getIntField(colIndex); uint bucket = bucketPicker((char *) &largeKey, 8, bpSeed) & ptMask; + bool joinerIsEmpty = tJoiners[j][bucket]->empty() ? true : false; + found = (tJoiners[j][bucket]->find(largeKey) != tJoiners[j][bucket]->end()); isNull = oldRow.isNullValue(colIndex); /* These conditions define when the row is NOT in the result set: @@ -1195,7 +1198,7 @@ void BatchPrimitiveProcessor::executeTupleJoin() */ if (((!found || isNull) && !(joinTypes[j] & (LARGEOUTER | ANTI))) || - ((joinTypes[j] & ANTI) && ((isNull && (joinTypes[j] & MATCHNULLS)) || (found && !isNull)))) + ((joinTypes[j] & ANTI) && !joinerIsEmpty && ((isNull && (joinTypes[j] & MATCHNULLS)) || (found && !isNull)))) { //cout << " - not in the result set\n"; break; From 3a3905b26ad77efc3925237e0113e58af6dd7144 Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Wed, 15 Jan 2020 10:15:12 -0800 Subject: [PATCH 07/84] Merge pull request #1001 from pleblanc1976/join-abort-fix MCOL-3713: Join abort fix --- dbcon/joblist/tuplehashjoin.cpp | 81 +++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 1f164a11f..f18b5d101 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -133,8 +133,11 @@ TupleHashJoinStep::~TupleHashJoinStep() delete outputDL; if (memUsedByEachJoin) + { for (uint i = 0 ; i < smallDLs.size(); i++) resourceManager->returnMemory(memUsedByEachJoin[i], sessionMemLimit); + } + //cout << "deallocated THJS, UM memory available: " << resourceManager.availableMemory() << endl; } @@ -338,32 +341,33 @@ void TupleHashJoinStep::startSmallRunners(uint index) extendedInfo += "\n"; ostringstream oss; - if (joiner->inPM()) - { - oss << "PM join (" << index << ")" << endl; - #ifdef JLF_DEBUG - cout << oss.str(); - #endif - extendedInfo += oss.str(); - } - else if (joiner->inUM() && !joiner->onDisk()) - { - oss << "UM join (" << index << ")" << endl; - #ifdef JLF_DEBUG - cout << oss.str(); - #endif - extendedInfo += oss.str(); - } - - /* Trying to get the extended info to match the original version - It's kind of kludgey at the moment, need to clean it up at some point */ if (!joiner->onDisk()) { - joiner->doneInserting(); - boost::mutex::scoped_lock lk(*fStatsMutexPtr); - fExtendedInfo += extendedInfo; - formatMiniStats(index); + // add extended info, and if not aborted then tell joiner + // we're done reading the small side. + if (joiner->inPM()) + { + oss << "PM join (" << index << ")" << endl; + #ifdef JLF_DEBUG + cout << oss.str(); + #endif + extendedInfo += oss.str(); + } + else if (joiner->inUM()) + { + oss << "UM join (" << index << ")" << endl; + #ifdef JLF_DEBUG + cout << oss.str(); + #endif + extendedInfo += oss.str(); + } + if (!cancelled()) + joiner->doneInserting(); } + + boost::mutex::scoped_lock lk(*fStatsMutexPtr); + fExtendedInfo += extendedInfo; + formatMiniStats(index); } /* Index is which small input to read. */ @@ -405,9 +409,16 @@ void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t * gotMem = resourceManager->getMemory(rgSize, sessionMemLimit, false); if (!gotMem) { + /* Mem went over the limit. + If DML or a syscat query, abort. + if disk join is enabled, use it. + else abort. + */ boost::unique_lock sl(saneErrMsg); - if (!joinIsTooBig && (isDML || !allowDJS || (fSessionId & 0x80000000) || - (tableOid() < 3000 && tableOid() >= 1000))) + if (cancelled()) + return; + if (!allowDJS || isDML || (fSessionId & 0x80000000) || + (tableOid() < 3000 && tableOid() >= 1000)) { joinIsTooBig = true; fLogger->logMessage(logging::LOG_TYPE_INFO, logging::ERR_JOIN_TOO_BIG); @@ -415,13 +426,10 @@ void TupleHashJoinStep::smallRunnerFcn(uint32_t index, uint threadID, uint64_t * status(logging::ERR_JOIN_TOO_BIG); cout << "Join is too big, raise the UM join limit for now (small runner)" << endl; abort(); - break; } - else - { + else if (allowDJS) joiner->setConvertToDiskJoin(); - return; - } + return; } joiner->insertRGData(smallRG, threadID); @@ -700,7 +708,9 @@ void TupleHashJoinStep::hjRunner() segregateJoiners(); /* Need to clean this stuff up. If the query was cancelled before this, and this would have had - a disk join, it's still necessary to construct the DJS objects to finish the abort. */ + a disk join, it's still necessary to construct the DJS objects to finish the abort. + Update: Is this more complicated than scanning joiners for either ondisk() or (not isFinished()) + and draining the corresponding inputs & telling downstream EOF? todo, think about it */ if (!djsJoiners.empty()) { joinIsTooBig = false; @@ -1895,8 +1905,6 @@ void TupleHashJoinStep::segregateJoiners() bool anyTooLarge = false; uint32_t smallSideCount = smallDLs.size(); - boost::mutex::scoped_lock sl(djsLock); - for (i = 0; i < smallSideCount; i++) { allInnerJoins &= (joinTypes[i] == INNER); @@ -1921,8 +1929,9 @@ void TupleHashJoinStep::segregateJoiners() // Debugging code, this makes all eligible joins disk-based. else { cout << "making all joins disk-based" << endl; + joinIsTooBig = true; for (i = 0; i < smallSideCount; i++) { - joinIsTooBig = true; + joiner[i]->setConvertToDiskJoin(); djsJoiners.push_back(joiners[i]); djsJoinerMap.push_back(i); } @@ -1930,6 +1939,7 @@ void TupleHashJoinStep::segregateJoiners() } #endif + boost::mutex::scoped_lock sl(djsLock); /* For now if there is no largeBPS all joins need to either be DJS or not, not mixed */ if (!largeBPS) { @@ -1939,6 +1949,7 @@ void TupleHashJoinStep::segregateJoiners() for (i = 0; i < smallSideCount; i++) { + joiners[i]->setConvertToDiskJoin(); djsJoiners.push_back(joiners[i]); djsJoinerMap.push_back(i); } @@ -1967,6 +1978,7 @@ void TupleHashJoinStep::segregateJoiners() else { joinIsTooBig = true; + joiners[i]->setConvertToDiskJoin(); //cout << "1joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> DJS" << endl; djsJoiners.push_back(joiners[i]); djsJoinerMap.push_back(i); @@ -1991,6 +2003,7 @@ void TupleHashJoinStep::segregateJoiners() for (; i < smallSideCount; i++) { joinIsTooBig = true; + joiners[i]->setConvertToDiskJoin(); //cout << "2joiner " << i << " " << hex << (uint64_t) joiners[i].get() << dec << " -> DJS" << endl; djsJoiners.push_back(joiners[i]); djsJoinerMap.push_back(i); From c6aaa3f227ac3980b1d4e29651300e96cc539728 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 14 Jan 2020 22:44:35 +0000 Subject: [PATCH 08/84] This commit fixes MDB crash that happens when table lock timeout fires with multiply DML run in separate transactions. This commit also fixes GCC unused variable warning. --- dbcon/mysql/ha_mcs_impl.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 3b71dd5f9..a417ea990 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2206,9 +2206,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) //@Bug 2540. Set error status instead of warning thd->raise_error_printf(ER_INTERNAL_ERROR, errorMsg.c_str()); ci->rc = b; - // WIP - //thd->get_stmt_da()->set_overwrite_status(true); - //cout << " error status " << ci->rc << endl; + rc = ER_INTERNAL_ERROR; } if (b == dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING) @@ -2249,7 +2247,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) delete ci->dmlProc; ci->dmlProc = NULL; - return 0; + return rc; } } //anon namespace From d0ffede1354332c0232d2364f5333c286f84760a Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 23 Jan 2020 17:41:02 +0000 Subject: [PATCH 09/84] MCOL-3680 mysqld will abort if Columnstore.xml is missing Add try-catch blocks in the plugin code to prevent mysqld from aborting if configcpp cannot access Columnstore.xml. --- dbcon/mysql/ha_mcs.cpp | 316 ++++++++++++++++++++++----- dbcon/mysql/ha_mcs.h | 2 +- dbcon/mysql/is_columnstore_files.cpp | 14 +- utils/configcpp/configcpp.cpp | 16 +- 4 files changed, 277 insertions(+), 71 deletions(-) diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index 61fda11c3..67b683e21 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -30,14 +30,14 @@ #define COLUMNSTORE_MATURITY MariaDB_PLUGIN_MATURITY_STABLE #endif -static handler* calpont_create_handler(handlerton* hton, - TABLE_SHARE* table, - MEM_ROOT* mem_root); +static handler* mcs_create_handler(handlerton* hton, + TABLE_SHARE* table, + MEM_ROOT* mem_root); -static int calpont_commit(handlerton* hton, THD* thd, bool all); +static int mcs_commit(handlerton* hton, THD* thd, bool all); -static int calpont_rollback(handlerton* hton, THD* thd, bool all); -static int calpont_close_connection ( handlerton* hton, THD* thd ); +static int mcs_rollback(handlerton* hton, THD* thd, bool all); +static int mcs_close_connection(handlerton* hton, THD* thd ); handlerton* mcs_hton; char cs_version[25]; char cs_commit_hash[41]; // a commit hash is 40 characters @@ -59,11 +59,11 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* sel); Hash used to track the number of open tables; variable for example share methods */ -static HASH calpont_open_tables; +static HASH mcs_open_tables; #ifndef _MSC_VER /* The mutex used to init the hash; variable for example share methods */ -pthread_mutex_t calpont_mutex; +pthread_mutex_t mcs_mutex; #endif #ifdef DEBUG_ENTER @@ -79,20 +79,20 @@ pthread_mutex_t calpont_mutex; Function we use in the creation of our hash to get key. */ -static uchar* calpont_get_key(COLUMNSTORE_SHARE* share, size_t* length, - my_bool not_used __attribute__((unused))) +static uchar* mcs_get_key(COLUMNSTORE_SHARE* share, size_t* length, + my_bool not_used __attribute__((unused))) { *length = share->table_name_length; return (uchar*) share->table_name; } // This one is unused -int calpont_discover(handlerton* hton, THD* thd, TABLE_SHARE* share) +int mcs_discover(handlerton* hton, THD* thd, TABLE_SHARE* share) { - DBUG_ENTER("calpont_discover"); - DBUG_PRINT("calpont_discover", ("db: '%s' name: '%s'", share->db.str, share->table_name.str)); + DBUG_ENTER("mcs_discover"); + DBUG_PRINT("mcs_discover", ("db: '%s' name: '%s'", share->db.str, share->table_name.str)); #ifdef INFINIDB_DEBUG - fprintf(stderr, "calpont_discover()\n"); + fprintf(stderr, "mcs_discover()\n"); #endif uchar* frm_data = NULL; @@ -114,8 +114,8 @@ int calpont_discover(handlerton* hton, THD* thd, TABLE_SHARE* share) } // This f() is also unused -int calpont_discover_existence(handlerton* hton, const char* db, - const char* table_name) +int mcs_discover_existence(handlerton* hton, const char* db, + const char* table_name) { return ha_mcs_impl_discover_existence(db, table_name); } @@ -144,19 +144,19 @@ static int columnstore_init_func(void* p) mcs_hton = (handlerton*)p; #ifndef _MSC_VER - (void) pthread_mutex_init(&calpont_mutex, MY_MUTEX_INIT_FAST); + (void) pthread_mutex_init(&mcs_mutex, MY_MUTEX_INIT_FAST); #endif - (void) my_hash_init(&calpont_open_tables, system_charset_info, 32, 0, 0, - (my_hash_get_key) calpont_get_key, 0, 0); + (void) my_hash_init(&mcs_open_tables, system_charset_info, 32, 0, 0, + (my_hash_get_key) mcs_get_key, 0, 0); mcs_hton->state = SHOW_OPTION_YES; - mcs_hton->create = calpont_create_handler; + mcs_hton->create = mcs_create_handler; mcs_hton->flags = HTON_CAN_RECREATE; -// mcs_hton->discover_table= calpont_discover; -// mcs_hton->discover_table_existence= calpont_discover_existence; - mcs_hton->commit = calpont_commit; - mcs_hton->rollback = calpont_rollback; - mcs_hton->close_connection = calpont_close_connection; +// mcs_hton->discover_table = mcs_discover; +// mcs_hton->discover_table_existence = mcs_discover_existence; + mcs_hton->commit = mcs_commit; + mcs_hton->rollback = mcs_rollback; + mcs_hton->close_connection = mcs_close_connection; mcs_hton->create_group_by = create_columnstore_group_by_handler; mcs_hton->create_derived = create_columnstore_derived_handler; mcs_hton->create_select = create_columnstore_select_handler; @@ -166,37 +166,64 @@ static int columnstore_init_func(void* p) static int columnstore_done_func(void* p) { - DBUG_ENTER("calpont_done_func"); + DBUG_ENTER("columnstore_done_func"); - my_hash_free(&calpont_open_tables); + my_hash_free(&mcs_open_tables); #ifndef _MSC_VER - pthread_mutex_destroy(&calpont_mutex); + pthread_mutex_destroy(&mcs_mutex); #endif DBUG_RETURN(0); } -static handler* calpont_create_handler(handlerton* hton, - TABLE_SHARE* table, - MEM_ROOT* mem_root) +static handler* mcs_create_handler(handlerton* hton, + TABLE_SHARE* table, + MEM_ROOT* mem_root) { return new (mem_root) ha_mcs(hton, table); } -static int calpont_commit(handlerton* hton, THD* thd, bool all) +static int mcs_commit(handlerton* hton, THD* thd, bool all) { - int rc = ha_mcs_impl_commit( hton, thd, all); + int rc; + try + { + rc = ha_mcs_impl_commit(hton, thd, all); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } return rc; } -static int calpont_rollback(handlerton* hton, THD* thd, bool all) +static int mcs_rollback(handlerton* hton, THD* thd, bool all) { - int rc = ha_mcs_impl_rollback( hton, thd, all); + int rc; + try + { + rc = ha_mcs_impl_rollback(hton, thd, all); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } return rc; } -static int calpont_close_connection ( handlerton* hton, THD* thd ) +static int mcs_close_connection(handlerton* hton, THD* thd) { - int rc = ha_mcs_impl_close_connection( hton, thd); + int rc; + try + { + rc = ha_mcs_impl_close_connection(hton, thd); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } return rc; } @@ -257,7 +284,16 @@ int ha_mcs::open(const char* name, int mode, uint32_t test_if_locked) { DBUG_ENTER("ha_mcs::open"); - int rc = ha_mcs_impl_open(name, mode, test_if_locked); + int rc; + try + { + rc = ha_mcs_impl_open(name, mode, test_if_locked); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -283,7 +319,16 @@ int ha_mcs::close(void) { DBUG_ENTER("ha_mcs::close"); - int rc = ha_mcs_impl_close(); + int rc; + try + { + rc = ha_mcs_impl_close(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -304,7 +349,16 @@ int ha_mcs::close(void) int ha_mcs::write_row(const uchar* buf) { DBUG_ENTER("ha_mcs::write_row"); - int rc = ha_mcs_impl_write_row(buf, table); + int rc; + try + { + rc = ha_mcs_impl_write_row(buf, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -312,14 +366,30 @@ int ha_mcs::write_row(const uchar* buf) void ha_mcs::start_bulk_insert(ha_rows rows, uint flags) { DBUG_ENTER("ha_mcs::start_bulk_insert"); - ha_mcs_impl_start_bulk_insert(rows, table); + try + { + ha_mcs_impl_start_bulk_insert(rows, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + } DBUG_VOID_RETURN; } int ha_mcs::end_bulk_insert() { DBUG_ENTER("ha_mcs::end_bulk_insert"); - int rc = ha_mcs_impl_end_bulk_insert(false, table); + int rc; + try + { + rc = ha_mcs_impl_end_bulk_insert(false, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -340,7 +410,16 @@ int ha_mcs::update_row(const uchar* old_data, uchar* new_data) { DBUG_ENTER("ha_mcs::update_row"); - int rc = ha_mcs_impl_update_row(); + int rc; + try + { + rc = ha_mcs_impl_update_row(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -366,15 +445,33 @@ int ha_mcs::direct_update_rows_init(List *update_fields) int ha_mcs::direct_update_rows(ha_rows *update_rows) { DBUG_ENTER("ha_mcs::direct_update_rows"); - int rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } int ha_mcs::direct_update_rows(ha_rows *update_rows, ha_rows *found_rows) { DBUG_ENTER("ha_mcs::direct_update_rows"); - int rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); - *found_rows = *update_rows; + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); + *found_rows = *update_rows; + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -387,7 +484,16 @@ int ha_mcs::direct_delete_rows_init() int ha_mcs::direct_delete_rows(ha_rows *deleted_rows) { DBUG_ENTER("ha_mcs::direct_delete_rows"); - int rc = ha_mcs_impl_direct_update_delete_rows(true, deleted_rows); + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(true, deleted_rows); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } /** @@ -413,7 +519,16 @@ int ha_mcs::direct_delete_rows(ha_rows *deleted_rows) int ha_mcs::delete_row(const uchar* buf) { DBUG_ENTER("ha_mcs::delete_row"); - int rc = ha_mcs_impl_delete_row(); + int rc; + try + { + rc = ha_mcs_impl_delete_row(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -512,7 +627,15 @@ int ha_mcs::rnd_init(bool scan) int rc = 0; if(scan) { - rc = ha_mcs_impl_rnd_init(table); + try + { + rc = ha_mcs_impl_rnd_init(table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } } DBUG_RETURN(rc); @@ -522,7 +645,16 @@ int ha_mcs::rnd_end() { DBUG_ENTER("ha_mcs::rnd_end"); - int rc = ha_mcs_impl_rnd_end(table); + int rc; + try + { + rc = ha_mcs_impl_rnd_end(table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -546,7 +678,16 @@ int ha_mcs::rnd_next(uchar* buf) { DBUG_ENTER("ha_mcs::rnd_next"); - int rc = ha_mcs_impl_rnd_next(buf, table); + int rc; + try + { + rc = ha_mcs_impl_rnd_next(buf, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -602,7 +743,16 @@ void ha_mcs::position(const uchar* record) int ha_mcs::rnd_pos(uchar* buf, uchar* pos) { DBUG_ENTER("ha_mcs::rnd_pos"); - int rc = ha_mcs_impl_rnd_pos(buf, pos); + int rc; + try + { + rc = ha_mcs_impl_rnd_pos(buf, pos); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -755,11 +905,20 @@ int ha_mcs::external_lock(THD* thd, int lock_type) { DBUG_ENTER("ha_mcs::external_lock"); - //@Bug 2526 Only register the transaction when autocommit is off - if ((thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) - trans_register_ha( thd, true, mcs_hton); + int rc; + try + { + //@Bug 2526 Only register the transaction when autocommit is off + if ((thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) + trans_register_ha( thd, true, mcs_hton); - int rc = ha_mcs_impl_external_lock(thd, table, lock_type); + rc = ha_mcs_impl_external_lock(thd, table, lock_type); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -840,7 +999,17 @@ int ha_mcs::delete_table(const char* name) DBUG_ENTER("ha_mcs::delete_table"); /* This is not implemented but we want someone to be able that it works. */ - int rc = ha_mcs_impl_delete_table(name); + int rc; + + try + { + rc = ha_mcs_impl_delete_table(name); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -863,7 +1032,16 @@ int ha_mcs::delete_table(const char* name) int ha_mcs::rename_table(const char* from, const char* to) { DBUG_ENTER("ha_mcs::rename_table "); - int rc = ha_mcs_impl_rename_table(from, to); + int rc; + try + { + rc = ha_mcs_impl_rename_table(from, to); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -913,14 +1091,32 @@ int ha_mcs::create(const char* name, TABLE* table_arg, { DBUG_ENTER("ha_mcs::create"); - int rc = ha_mcs_impl_create(name, table_arg, create_info); + int rc; + try + { + rc = ha_mcs_impl_create(name, table_arg, create_info); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } const COND* ha_mcs::cond_push(const COND* cond) { DBUG_ENTER("ha_mcs::cond_push"); - DBUG_RETURN(ha_mcs_impl_cond_push(const_cast(cond), table)); + COND* ret_cond = NULL; + try + { + ret_cond = ha_mcs_impl_cond_push(const_cast(cond), table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + } + DBUG_RETURN(ret_cond); } diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index cd29a165b..8baf31e7e 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -28,7 +28,7 @@ extern handlerton* mcs_hton; COLUMNSTORE_SHARE is a structure that will be shared among all open handlers. This example implements the minimum of what you will probably need. */ -typedef struct st_calpont_share +typedef struct st_mcs_share { char* table_name; uint32_t table_name_length, use_count; diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 429b5c650..cad0b8528 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -93,7 +93,19 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th char oidDirName[WriteEngine::FILE_NAME_SIZE]; char fullFileName[WriteEngine::FILE_NAME_SIZE]; char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; - config::Config* config = config::Config::makeConfig(); + + config::Config* config; + + try + { + config = config::Config::makeConfig(); + } + catch (std::runtime_error& e) + { + thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + return ER_INTERNAL_ERROR; + } + WriteEngine::Config we_config; off_t fileSize = 0; off_t compressedFileSize = 0; diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 1bea5c09b..823e6b2a6 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -114,19 +114,17 @@ Config* Config::makeConfig(const char* cf) Config::Config(const string& configFile) : fDoc(0), fConfigFile(configFile), fMtime(0), fParser() { - for ( int i = 0 ; i < 20 ; i++ ) + int i = 0; + for ( ; i < 2 ; i++ ) { - if (access(fConfigFile.c_str(), R_OK) != 0) - { - if ( i >= 15 ) - throw runtime_error("Config::Config: error accessing config file " + fConfigFile); - - sleep (1); - } - else + if (access(fConfigFile.c_str(), R_OK) == 0) break; + sleep (1); } + if ( i == 2 ) + throw runtime_error("Config::Config: error accessing config file " + fConfigFile); + struct stat statbuf; if (stat(configFile.c_str(), &statbuf) == 0) From 5b794c862c416a0affa62297c2b0293f39014b6d Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 23 Jan 2020 12:47:56 -0500 Subject: [PATCH 10/84] Fixed a compiler warning that broke debugging builds. --- utils/common/poolallocator.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/common/poolallocator.cpp b/utils/common/poolallocator.cpp index 04f31121c..684e7049c 100644 --- a/utils/common/poolallocator.cpp +++ b/utils/common/poolallocator.cpp @@ -68,7 +68,6 @@ void PoolAllocator::newBlock() void * PoolAllocator::allocOOB(uint64_t size) { - bool _false = false; OOBMemInfo memInfo; memUsage += size; From dc94725dd55e4274fb2107660aa608e4b8a029f0 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 23 Jan 2020 12:48:51 -0500 Subject: [PATCH 11/84] Fixed a problem with the early-exit code in postConfig, where it would turn replication on in the config file but not specify whether it was the master or slave. Made the early-exit code turn rep off. --- oamapps/postConfigure/installer.cpp | 8 ++++++-- oamapps/postConfigure/postConfigure.cpp | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 9a2c73b1c..cb94d7c23 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -713,7 +713,9 @@ int main(int argc, char* argv[]) if (getenv("SKIP_OAM_INIT")) { - cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + cout << "(2) SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + sysConfig->setConfig("Installation", "MySQLRep", "n"); + sysConfig->write(); exit(0); } @@ -740,7 +742,9 @@ int main(int argc, char* argv[]) if (getenv("SKIP_OAM_INIT")) { - cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + cout << "(3) SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + sysConfig->setConfig("Installation", "MySQLRep", "n"); + sysConfig->write(); exit(0); } diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index b807876c3..1ec9b023e 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3732,7 +3732,9 @@ int main(int argc, char* argv[]) if (getenv("SKIP_OAM_INIT")) { - cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + cout << "(1) SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + sysConfig->setConfig("Installation", "MySQLRep", "n"); + sysConfig->write(); exit(0); } From 57bf5303f174cb72fa59c04fa8f1af9d5cd43ab3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 23 Jan 2020 14:10:15 -0600 Subject: [PATCH 12/84] MCOL-3662 Restore error message for multi parameter aggregates save for UDAnF and GROUP_CONCAT --- dbcon/mysql/ha_mcs_execplan.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 63dbb91a8..f7602b9bb 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4559,10 +4559,9 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) if (gwi.clauseType == SELECT) gwi.aggOnSelect = true; - // N.B. argument_count() is the # of formal parms to the agg fcn. InifniDB only supports 1 argument - // TODO: Support more than one parm -#if 0 - + // Argument_count() is the # of formal parms to the agg fcn. Columnstore + // only supports 1 argument except UDAnF and GROUP_CONCAT + // TODO: Support more than one parm for COUNT(DISTINCT) if (isp->argument_count() != 1 && isp->sum_func() != Item_sum::GROUP_CONCAT_FUNC && isp->sum_func() != Item_sum::UDF_SUM_FUNC) { @@ -4571,7 +4570,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) return NULL; } -#endif AggregateColumn* ac = NULL; if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) From 4b86890cf701c9f79918321f12c1d4e1bfe4bbae Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 20 Jan 2020 14:26:39 -0800 Subject: [PATCH 13/84] MCOL-128 Support ALTER TABLE...ENGINE=Columnstore Also implements: * ALTER TABLE from Columnstore to another engine * MCOL-3349 CREATE TABLE ... AS SELECT. --- dbcon/mysql/ha_mcs_ddl.cpp | 36 +++++++++++++++++++++++------------- dbcon/mysql/ha_mcs_impl.cpp | 5 ++++- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 43671055e..8454b8cce 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -2400,14 +2400,10 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea return 1; } - // @bug 3908. error out primary key for now. + // Send notice if primary key specified that it is not supported if (table_arg->key_info && table_arg->key_info->name.length && string(table_arg->key_info->name.str) == "PRIMARY") { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS); - setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - ci.alterTableState = cal_connection_info::NOT_ALTER; - ci.isAlter = false; - return 1; + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "INDEXES"); } int compressiontype = get_compression_type(thd); @@ -2452,22 +2448,25 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea return 1; } - // - // Check if this is a "CREATE TABLE ... LIKE " statement. + // Check if this is one of + // * "CREATE TABLE ... LIKE " + // * "ALTER TABLE ... ENGINE=Columnstore" + // * "CREATE TABLE ... AS ..." // If so generate a full create table statement using the properties of // the source table. Note that source table has to be a columnstore table and // we only check for currently supported options. // - if (thd->lex->create_info.like()) + if ((thd->lex->sql_command == SQLCOM_CREATE_TABLE && thd->lex->used_tables) || + (thd->lex->sql_command == SQLCOM_ALTER_TABLE && create_info->used_fields & HA_CREATE_USED_ENGINE) || + (thd->lex->create_info.like())) { TABLE_SHARE *share = table_arg->s; my_bitmap_map *old_map; // To save the read_set char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH]; String datatype, def_value; ostringstream oss; - string tbl_name (name+2); - std::replace(tbl_name.begin(), tbl_name.end(), '/', '.'); + string tbl_name = string(share->db.str) + "." + string(share->table_name.str); // Save the current read_set map and mark it for read old_map= tmp_use_all_columns(table_arg, table_arg->read_set); @@ -2505,10 +2504,18 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea // Process table level options + /* TODO: uncomment when we support AUTO_INCREMENT if (create_info->auto_increment_value > 1) { oss << " AUTO_INCREMENT=" << create_info->auto_increment_value; } + */ + + if (create_info->auto_increment_value > 1) + { + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "AUTO INCREMENT"); + } + if (share->table_charset) { @@ -2539,7 +2546,7 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea oss << ";"; stmt = oss.str(); - + tmp_restore_column_map(table_arg->read_set, old_map); } @@ -2600,7 +2607,7 @@ int ha_mcs_impl_delete_table_(const char* db, const char* name, cal_connection_i string emsg; - if (thd->lex->sql_command == SQLCOM_DROP_DB) + if ((thd->lex->sql_command == SQLCOM_DROP_DB) || (thd->lex->sql_command == SQLCOM_ALTER_TABLE)) { std::string tableName(name); tableName.erase(0, tableName.rfind("/") + 1); @@ -2636,6 +2643,9 @@ int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_connection_i decode_file_path(to, decodedDbTo, decodedTblTo); string stmt; + // This case is already handled + if (thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE) + return 0; if (thd->slave_thread && !get_replication_slave(thd)) return 0; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 766cfbea7..532e67843 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2362,8 +2362,11 @@ int ha_mcs_impl_rnd_init(TABLE* table) } #endif - if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) + // If ALTER TABLE and not ENGINE= we don't need rnd_init (gets us in a bad state) + if ((thd->lex->sql_command == SQLCOM_ALTER_TABLE) && !(thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE)) + { return 0; + } //Update and delete code if ( ((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) From bbd5a13d4abd5c6ebad989d9ab8dc843c49574d7 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 23 Jan 2020 18:58:10 -0500 Subject: [PATCH 14/84] MCOL-3675 and MCOL-3676 Detect in postConfigure if columnstore-post-install is run or not --- oamapps/postConfigure/postConfigure.cpp | 34 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index b807876c3..3fda29b34 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -58,6 +58,7 @@ #include /* for strncpy */ #include +#include #include #include #include @@ -498,6 +499,31 @@ int main(int argc, char* argv[]) cout << "IMPORTANT: This tool requires to run on the Performance Module #1" << endl; cout << endl; + // MCOL-3675 + struct stat dir_info; + if (stat(tmpDir.c_str(), &dir_info) != 0) + { + cerr<getConfig(InstallSection, "ProfileFile"); + } + catch (...) + {} + + // MCOL-3676 + if (ProfileFile.empty()) + { + cerr<getConfig(InstallSection, "ProfileFile"); - } - catch (...) - {} - if ( waitForActive() ) { cout << " DONE" << endl; From 2c814bd3edc0dbbcfae2e60f1096b4c324dcd45a Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 24 Jan 2020 12:33:03 -0600 Subject: [PATCH 15/84] MCOL-3744 mcssystemready to work with SKIP_OAM_INIT --- dbcon/mysql/ha_mcs_client_udfs.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 51ce2917f..8395b364b 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -294,13 +294,24 @@ extern "C" try { - oam.getSystemStatus(systemstatus); - - if (systemstatus.SystemOpState == ACTIVE - && dbrm.getSystemReady() - && dbrm.getSystemQueryReady()) + if (getenv("SKIP_OAM_INIT")) { - return 1; + if (dbrm.getSystemReady() + && dbrm.getSystemQueryReady()) + { + return 1; + } + } + else + { + oam.getSystemStatus(systemstatus); + + if (systemstatus.SystemOpState == ACTIVE + && dbrm.getSystemReady() + && dbrm.getSystemQueryReady()) + { + return 1; + } } } catch (...) From 275b1db857b312736d70488a658f8dc9f2267582 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 28 Jan 2020 11:16:18 -0600 Subject: [PATCH 16/84] MCOL-3632 Restore error message for ref item type Function or Operator with sub query on the SELECT clause is currently not supported. --- dbcon/mysql/ha_mcs_execplan.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 63dbb91a8..59f013cd4 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3306,6 +3306,11 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp case Item::WINDOW_FUNC_ITEM: return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); + case Item::SUBSELECT_ITEM: + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + break; + default: gwi.fatalParseError = true; gwi.parseErrorText = "Unknown REF item"; From acc87f4cfc5c82eb7359fe5b5dc7704b26ba5380 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Sat, 1 Feb 2020 16:02:03 +0100 Subject: [PATCH 17/84] MCOL-3721 Allow collate and warn on ORDER BY COLLATE is now allowed but ignored on DDL and ORDER BY. For an ORDER BY query a note is generated. --- dbcon/ddlpackage/ddl.l | 1 + dbcon/ddlpackage/ddl.y | 8 +++++++- dbcon/mysql/ha_mcs_execplan.cpp | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index c644bddf0..ee585f048 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -171,6 +171,7 @@ FLOAT {return IDB_FLOAT;} DOUBLE {return DOUBLE;} REAL {return REAL;} CHARSET {return CHARSET;} +COLLATE {return COLLATE;} IF {return IDB_IF;} EXISTS {return EXISTS;} CHANGE {return CHANGE;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 177a64caa..6fbbb67f2 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -111,7 +111,7 @@ MATCH MAX_ROWS MEDIUMBLOB MEDIUMTEXT MIN_ROWS MODIFY NO NOT NULL_TOK NUMBER NUMERIC ON PARTIAL PRECISION PRIMARY REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY -VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE +VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET COLLATE IDB_IF EXISTS CHANGE TRUNCATE BOOL BOOLEAN MEDIUMINT TIMESTAMP %token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE TIME @@ -485,8 +485,14 @@ table_option: } | DEFAULT CHARSET opt_equal ident {$$ = new pair("default charset", $4);} + | + CHARSET opt_equal ident {$$ = new pair("default charset", $3);} | DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair("default charset", $5);} + | + DEFAULT COLLATE opt_equal ident {$$ = new pair("default collate", $4);} + | + COLLATE opt_equal ident {$$ = new pair("default collate", $3);} ; alter_table_statement: diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 90dbaa2d3..d367686a5 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -7555,6 +7555,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { gwi.fatalParseError = true; } + else if ((ord_item->type() == Item::FUNC_ITEM) && (((Item_func*)ord_item)->functype() == Item_func::COLLATE_FUNC)) + { + push_warning(gwi.thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "COLLATE is ignored in ColumnStore"); + continue; + } else { rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); From 608fba77fde1daab09dd86e682192715806cbf74 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 4 Feb 2020 08:26:15 +0000 Subject: [PATCH 18/84] MCOL-3747 This patch properly enables queries with IN + uncorrelated subquery processing that was broken after vtable had been removed. Before this patch IN-INTO-EXISTS rewrite doesn't add equi-JOIN predicate when IN+subquery was in a nested subquery. --- dbcon/mysql/ha_mcs_pushdown.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 2e26e65bc..d267ad896 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -42,6 +42,26 @@ void disable_indices_for_CEJ(THD *thd_) } } +bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) +{ + bool result = false; + TABLE_LIST *tbl; + List_iterator_fast li(select_lex->leaf_tables); + while (!result && (tbl= li++)) + { + if (tbl->is_view_or_derived()) + { + SELECT_LEX *dsl = tbl->derived->first_select(); + result = optimize_unflattened_subqueries_(dsl); + } + } + + result = (!result) ? + select_lex->optimize_unflattened_subqueries(false) : true; + + return result; +} + void mutate_optimizer_flags(THD *thd_) { // MCOL-2178 Disable all optimizer flags as it was in the fork. @@ -813,7 +833,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) } COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); - select_lex->optimize_unflattened_subqueries(false); + // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add + // an equi-JOIN condition. + optimize_unflattened_subqueries_(select_lex); if (conds) { From 009a4aea1d5d7cef72f2e2ada5ebc9707789117d Mon Sep 17 00:00:00 2001 From: jmrojas2332 Date: Tue, 28 Jan 2020 17:43:40 +0000 Subject: [PATCH 19/84] MCOL 2139 Fix subquery not being filtered when using view --- dbcon/mysql/ha_mcs_execplan.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 90dbaa2d3..099b3c388 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -5664,6 +5664,11 @@ void gp_walk(const Item* item, void* arg) { boost::shared_ptr scsp(sc->clone()); gwip->scsp = scsp; + + if (col->type() == Item::FIELD_ITEM) + { + gwip->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(((Item_field*)item)->field_name.str), scsp)); + } } bool cando = true; From 7c358957dc75c89fcb8cd9579e2fe47b98c4dee3 Mon Sep 17 00:00:00 2001 From: jmrojas2332 Date: Mon, 3 Feb 2020 20:13:55 +0000 Subject: [PATCH 20/84] MCOL-2139 syntax update based on feedback --- dbcon/mysql/ha_mcs_execplan.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 099b3c388..10f0b96fa 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -5667,7 +5667,9 @@ void gp_walk(const Item* item, void* arg) if (col->type() == Item::FIELD_ITEM) { - gwip->columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(((Item_field*)item)->field_name.str), scsp)); + const auto &field_name = string(((Item_field*)item)->field_name.str); + auto col = CalpontSelectExecutionPlan::ColumnMap::value_type(field_name, scsp); + gwip->columnMap.insert(col); } } From 979d2bcd036573c4f61a975d3aeb2d07d088e936 Mon Sep 17 00:00:00 2001 From: jmrojas2332 Date: Mon, 3 Feb 2020 20:49:02 +0000 Subject: [PATCH 21/84] MCOL-2139 syntax update --- dbcon/mysql/ha_mcs_execplan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 10f0b96fa..0f5bb6b88 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -5668,8 +5668,8 @@ void gp_walk(const Item* item, void* arg) if (col->type() == Item::FIELD_ITEM) { const auto &field_name = string(((Item_field*)item)->field_name.str); - auto col = CalpontSelectExecutionPlan::ColumnMap::value_type(field_name, scsp); - gwip->columnMap.insert(col); + auto colMap = CalpontSelectExecutionPlan::ColumnMap::value_type(field_name, scsp); + gwip->columnMap.insert(colMap); } } From 65de8b8e639a2e4b888533579d8789fc8de7f42e Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 4 Feb 2020 16:50:18 -0500 Subject: [PATCH 22/84] MCOL-3520. Checkpointing some mods. --- writeengine/splitter/we_cmdargs.cpp | 5 +- writeengine/splitter/we_cmdargs.h | 51 ++++++++++++++------ writeengine/splitter/we_filereadthread.cpp | 56 ++++++++++++++++++++-- writeengine/splitter/we_filereadthread.h | 12 +++++ 4 files changed, 104 insertions(+), 20 deletions(-) diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index f10dc690c..7235d4935 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -432,6 +432,8 @@ bool WECmdArgs::checkForCornerCases() throw(runtime_error("Mode 2 require remote file opts -f and -l or "\ "a fully qualified path for the remote file." "\nTry 'cpimport -h' for more information.")); + if (!fS3Key.empty()) + throw(runtime_error("Mode 2 & an input file on S3 does not make sense.")); } if (fMode == 3) @@ -973,7 +975,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) } if (optind < argc) // see if input file name is given - { + { // 3rd pos parm fLocFile = argv[optind]; @@ -1233,7 +1235,6 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) throw (runtime_error("No schema or local filename specified.")); } } - } std::string WECmdArgs::getJobFileName() diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index d02600070..47117932f 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -37,7 +37,7 @@ class WECmdArgs public: WECmdArgs(int argc, char** argv); virtual ~WECmdArgs() {} -public: + void appTestFunction(); void parseCmdLineArgs(int argc, char** argv); std::string getCpImportCmdLine(); @@ -54,7 +54,6 @@ public: char* pBuff, int FileIdx); void updateWithJobFile(int Idx); -public: std::string getJobFileName(); std::string getBrmRptFileName(); std::string getTmpFileDir(); @@ -195,19 +194,6 @@ public: return fTimeZone; } - -private: // variables for SplitterApp - typedef std::vector VecArgs; - VecArgs fVecArgs; - typedef std::vector VecInts; - VecInts fPmVec; - - VecArgs fVecJobFiles; //JobFiles splitter from master JobFile - int fMultiTableCount; //MultiTable count - VecArgs fColFldsFromJobFile;//List of columns from any job file, that - // represent fields in the import data - -public: bool getPmStatus(int Id); bool str2PmList(std::string& PmList, VecInts& V); int getPmVecSize() @@ -252,11 +238,46 @@ public: { fMultiTableCount = Count; } + + bool isS3Import() const + { + return !fS3Key.empty(); + } + std::string getS3Key() const + { + return fS3Key; + } + std::string getS3Bucket() const + { + return fS3Bucket; + } + std::string getS3Host() const + { + return fS3Host; + } + std::string getS3Secret() const + { + return fS3Secret; + } + std::string getS3Region() const + { + return fS3Region; + } std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342 void getColumnList( std::set& columnList ) const; private: // variables for SplitterApp + typedef std::vector VecArgs; + VecArgs fVecArgs; + typedef std::vector VecInts; + VecInts fPmVec; + + VecArgs fVecJobFiles; //JobFiles splitter from master JobFile + int fMultiTableCount; //MultiTable count + VecArgs fColFldsFromJobFile;//List of columns from any job file, that + // represent fields in the import data + std::string fJobId; // JobID std::string fOrigJobId; // Original JobID, in case we have to split it bool fJobLogOnly; // Job number is only for log filename only diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index 2d7fb4731..4836dfb46 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -98,7 +98,22 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh): fSdh(aSdh), } fBuff = new char [fBuffSize]; - + + /* Get S3 import params from fSdh.fRef, which is a ref to the we_splitterapp + */ + const WECmdArgs &args = fSdh.fRef.fCmdArgs; + initS3Connection(args); + doS3Import = args.isS3Import(); + if (doS3Import) + { + s3Key = args.getS3Key(); + s3Secret = args.getS3Secret(); + s3Bucket = args.getS3Bucket(); + s3Region = args.getS3Region(); + s3Host = args.getS3Host(); + ms3_library_init(); + connection = ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str()));) + } } //WEFileReadThread::WEFileReadThread(const WEFileReadThread& rhs):fSdh(rhs.fSdh) @@ -122,6 +137,12 @@ WEFileReadThread::~WEFileReadThread() fpThread = 0; delete []fBuff; //cout << "WEFileReadThread destructor called" << endl; + + if (doS3Import) + { + ms3_deinit(connection); + ms3_library_deinit(); + } } //------------------------------------------------------------------------------ @@ -451,8 +472,22 @@ void WEFileReadThread::openInFile() { try { + /* If an S3 transfer + use ms3 lib to d/l data into mem + use boost::iostreams to wrap the mem in an fstream interface + assign fiffile and/or finfile to it? + + example (change file_destriptor_source to array_source) + FILE* pipe = popen("find . -type f", "r"); + + io::stream_buffer fpstream(fileno(pipe)); + std::istream in(&fpstream); + */ + if (fSdh.getDebugLvl()) cout << "Input FileName: " << fInFileName << endl; + WORKING HERE + if (fInFileName == "/dev/stdin") { char aDefCon[16], aGreenCol[16]; @@ -465,8 +500,6 @@ void WEFileReadThread::openInFile() << aDefCon << endl; } - cout.flush(); - //@BUG 4326 if (fInFileName != "/dev/stdin") { @@ -592,6 +625,23 @@ int WEFileReadThread::getNextRow(istream& ifs, char* pBuf, int MaxLen) return pEnd - pBuf; } +void WEFileReadThread::initS3Connection(const WE_CmdArgs &args) +{ + doS3Import = args.isS3Import(); + if (doS3Import) + { + s3Key = args.getS3Key(); + s3Secret = args.getS3Secret(); + s3Bucket = args.getS3Bucket(); + s3Region = args.getS3Region(); + s3Host = args.getS3Host(); + ms3_library_init(); + connection = ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str()));) + if (!connection) + throw runtime_error("WEFileReadThread::initS3Connection(): Failed to init S3 connection"); + } +} + //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_filereadthread.h b/writeengine/splitter/we_filereadthread.h index 2fb92332c..5056fdeed 100644 --- a/writeengine/splitter/we_filereadthread.h +++ b/writeengine/splitter/we_filereadthread.h @@ -30,6 +30,8 @@ #ifndef WE_FILEREADTHREAD_H_ #define WE_FILEREADTHREAD_H_ +#include "libmarias3/marias3.h" + namespace WriteEngine { @@ -153,6 +155,16 @@ private: char fDelim; // Column Delimit char char* fBuff; // main data buffer int fBuffSize; + + /* To support mode 1 imports from objects on S3 */ + void initS3Connection(const WECmdArgs &); + bool doS3Import; + std::string s3Key; + std::string s3Secret; + std::string s3Bucket; + std::string s3Region; + std::string s3Host; + ms3_st *s3connection; }; } /* namespace WriteEngine */ From 6db7b96fe8c025fa5cdf4a97a07c0daa5da4af6b Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 4 Feb 2020 15:52:56 -0600 Subject: [PATCH 23/84] MCOL-3736: Change AND to OR. Loop could exit with ops still in pending/progress. --- storage-manager/src/Synchronizer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index 7e86f8e2d..ee47b9dfa 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -305,7 +305,7 @@ void Synchronizer::syncNow(const bf::path &prefix) // this should be redone to only remove items of given prefix eventually blockNewJobs = true; - while (pendingOps.size() != 0 && opsInProgress.size() != 0) + while (pendingOps.size() != 0 || opsInProgress.size() != 0) { for (auto &job : pendingOps) makeJob(job.first); @@ -329,7 +329,7 @@ void Synchronizer::syncNow() // Leaving S3 storage and local metadata directories sync'd for snapshot backups. blockNewJobs = true; - while (pendingOps.size() != 0 && opsInProgress.size() != 0) + while (pendingOps.size() != 0 || opsInProgress.size() != 0) { for (auto &job : pendingOps) makeJob(job.first); From f9f7b79a2be60c2a3aace3ca778d886d834f14e4 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 5 Feb 2020 11:10:54 -0600 Subject: [PATCH 24/84] MCOL-3749 Assertion on error Correct error code return stops assertion --- dbcon/mysql/ha_mcs_impl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index ce072d2fa..7ab1cbe61 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2217,6 +2217,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) ci->rc = b; // Turn this on as MariaDB doesn't do it until the next phase thd->abort_on_warning = thd->is_strict_mode(); + rc = ER_INTERNAL_ERROR; } else { From 6cf6519019bc662f1b8321875000ca8d9bcdcbf1 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Sat, 1 Feb 2020 14:25:35 +0100 Subject: [PATCH 25/84] MCOL-3718 Use systemd instead of mysql-Columnstore MariaDB server is now managed using systemd instead of our own mysql-Columnstore script. This means that MariaDB server can be stopped / started independently of ColumnStore --- dbcon/mysql/CMakeLists.txt | 2 +- dbcon/mysql/mysql-Columnstore | 486 ------------------ .../columnstore-post-install.in | 2 - .../columnstore-pre-uninstall.in | 3 +- oam/install_scripts/columnstore.in | 2 +- oam/install_scripts/post-mysql-install | 12 +- oam/oamcpp/liboamcpp.cpp | 19 +- oamapps/columnstoreSupport/configReport.sh | 10 +- procmon/processmonitor.cpp | 2 +- .../clusterTester/columnstoreClusterTester.sh | 4 +- 10 files changed, 24 insertions(+), 518 deletions(-) delete mode 100755 dbcon/mysql/mysql-Columnstore diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 8f1610c02..85c13f2a1 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -52,7 +52,7 @@ install(FILES syscatalog_mysql.sql calshowprocesslist.sql columnstore_info.sql DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) -install(PROGRAMS install_mcs_mysql.sh mysql-Columnstore +install(PROGRAMS install_mcs_mysql.sh DESTINATION ${ENGINE_SBINDIR} COMPONENT columnstore-engine) install(FILES columnstore.cnf diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore deleted file mode 100755 index 5c5305726..000000000 --- a/dbcon/mysql/mysql-Columnstore +++ /dev/null @@ -1,486 +0,0 @@ -#!/bin/sh -# Copyright Abandoned 1996 TCX DataKonsult AB & Monty Program KB & Detron HB -# This file is public domain and comes with NO WARRANTY of any kind - -# MySQL daemon start/stop script. - -# Usually this is put in /etc/init.d (at least on machines SYSV R4 based -# systems) and linked to /etc/rc3.d/S99mysql and /etc/rc0.d/K01mysql. -# When this is done the mysql server will be started when the machine is -# started and shut down when the systems goes down. - -# Comments to support chkconfig on RedHat Linux -# chkconfig: 2345 64 36 -# description: A very fast and reliable SQL database engine. - -# Comments to support LSB init script conventions -### BEGIN INIT INFO -# Provides: mysql -# Required-Start: $local_fs $network $remote_fs -# Should-Start: ypbind nscd ldap ntpd xntpd -# Required-Stop: $local_fs $network $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start and stop MySQL -# Description: MySQL is a very fast and reliable SQL database engine. -### END INIT INFO - -# If you install MySQL on some other places than /usr/local/mariadb/columnstore/mysql, then you -# have to do one of the following things for this script to work: -# -# - Run this script from within the MySQL installation directory -# - Create a /etc/my.cnf file with the following information: -# [mysqld] -# basedir= -# - Add the above to any other configuration file (for example ~/.my.ini) -# and copy my_print_defaults to /usr/bin -# - Add the path to the mysql-installation-directory to the basedir variable -# below. -# -# If you want to affect other MySQL variables, you should make your changes -# in the /etc/my.cnf, ~/.my.cnf or other MySQL configuration files. - -# If you change base dir, you must also change datadir. These may get -# overwritten by settings in the MySQL configuration files. - - -# Source function library. -if [ -f /etc/init.d/functions ]; then - . /etc/init.d/functions -fi - -# Default value, in seconds, afterwhich the script should timeout waiting -# for server start. -# Value here is overriden by value in my.cnf. -# 0 means don't wait at all -# Negative numbers mean to wait indefinitely -service_startup_timeout=90 -user=`whoami 2>/dev/null` - -# Lock directory -lockdir=/var/lock/subsys - -lock_file_path="$lockdir/mysql-Columnstore" - -# The following variables are only set for letting mysql.server find things. - -# Set some defaults -mysqld_pid_file_path= -if test -z "$basedir" -then - basedir=/usr - bindir=/usr/bin - if test -z "$datadir" - then - datadir=/var/lib/mysql - fi - sbindir=/usr/bin - bindir=/usr/bin -else - bindir="$basedir/bin" - if test -z "$datadir" - then - datadir="$basedir/data" - fi - sbindir="$basedir/sbin" - if test -f "$basedir/sbin/mysqld" - then - bindir="$basedir/bin" - else - bindir="$basedir/bin" - fi -fi - -# datadir_set is used to determine if datadir was set (and so should be -# *not* set inside of the --basedir= handler.) -datadir_set= - -# -# Use LSB init script functions for printing messages, if possible -# -lsb_functions="/lib/lsb/init-functions" -if test -f $lsb_functions ; then - . $lsb_functions >/dev/null 2>&1 -else - # Include non-LSB RedHat init functions to make systemctl redirect work - init_functions="/etc/init.d/functions" - if test -f $init_functions; then - . $init_functions >/dev/null 2>&1 - fi - log_success_msg() - { - echo " SUCCESS! $@" - } - log_failure_msg() - { - echo " ERROR! $@" - } -fi - -PATH="/sbin:/usr/sbin:/bin:/usr/bin:$basedir/bin" -export PATH - -mode=$1 # start or stop - -[ $# -ge 1 ] && shift - -case `echo "testing\c"`,`echo -n testing` in - *c*,-n*) echo_n= echo_c= ;; - *c*,*) echo_n=-n echo_c= ;; - *) echo_n= echo_c='\c' ;; -esac - -parse_server_arguments() { - for arg do - case "$arg" in - --basedir=*) basedir=`echo "$arg" | sed -e 's/^[^=]*=//'` - bindir="$basedir/bin" - if test -z "$datadir_set"; then - datadir="$basedir/data" - fi - sbindir="$basedir/sbin" - if test -f "$basedir/sbin/mysqld" - then - bindir="$basedir/bin" - else - bindir="$basedir/bin" - fi - #bindir="$basedir/bin" - ;; - --datadir=*) datadir=`echo "$arg" | sed -e 's/^[^=]*=//'` - datadir_set=1 - ;; - --log-basename=*|--hostname=*|--loose-log-basename=*) - mysqld_pid_file_path=`echo "$arg.pid" | sed -e 's/^[^=]*=//'` - ;; - --pid-file=*) mysqld_pid_file_path=`echo "$arg" | sed -e 's/^[^=]*=//'` ;; - --service-startup-timeout=*) service_startup_timeout=`echo "$arg" | sed -e 's/^[^=]*=//'` ;; - esac - done -} - -# Get arguments from the my.cnf file, -# the only group, which is read from now on is [mysqld] -if test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x $bindir/my_print_defaults -then - print_defaults="$bindir/my_print_defaults" -elif test -x $bindir/mysql_print_defaults -then - print_defaults="$bindir/mysql_print_defaults" -else - # Try to find basedir in /etc/my.cnf - conf=/etc/my.cnf - print_defaults= - if test -r $conf - then - subpat='^[^=]*basedir[^=]*=\(.*\)$' - dirs=`sed -e "/$subpat/!d" -e 's//\1/' $conf` - for d in $dirs - do - d=`echo $d | sed -e 's/[ ]//g'` - if test -x "$d/bin/my_print_defaults" - then - print_defaults="$d/bin/my_print_defaults" - break - fi - if test -x "$d/bin/mysql_print_defaults" - then - print_defaults="$d/bin/mysql_print_defaults" - break - fi - done - fi - - # Hope it's in the PATH ... but I doubt it - test -z "$print_defaults" && print_defaults="my_print_defaults" -fi - -# -# Read defaults file from 'basedir'. If there is no defaults file there -# check if it's in the old (depricated) place (datadir) and read it from there -# - -extra_args="" -if test -r "$basedir/my.cnf" -then - extra_args="-e $basedir/my.cnf" -else - if test -r "$datadir/my.cnf" - then - extra_args="-e $datadir/my.cnf" - fi -fi - -parse_server_arguments `$print_defaults $extra_args --mysqld mysql.server` -parse_server_arguments "$@" - -# wait for the pid file to disappear -wait_for_gone () { - pid="$1" # process ID of the program operating on the pid-file - pid_file_path="$2" # path to the PID file. - - i=0 - crash_protection="by checking again" - - while test $i -ne $service_startup_timeout ; do - - if kill -0 "$pid" 2>/dev/null; then - : # the server still runs - else - if test ! -s "$pid_file_path"; then - # no server process and no pid-file? great, we're done! - log_success_msg - return 0 - fi - - # pid-file exists, the server process doesn't. - # it must've crashed, and mysqld_safe will restart it - if test -n "$crash_protection"; then - crash_protection="" - sleep 5 - continue # Check again. - fi - - kill_by_pid - # Cannot help it - log_failure_msg "The server quit without updating PID file ($pid_file_path)." - return 1 # not waiting any more. - fi - - echo $echo_n ".$echo_c" - i=`expr $i + 1` - sleep 1 - - done - - log_failure_msg - kill_by_pid - - return 1 -} - -wait_for_ready () { - - i=0 - while test $i -ne $service_startup_timeout ; do - - if $bindir/mysqladmin ping >/dev/null 2>&1; then - log_success_msg - return 0 - elif kill -0 $! 2>/dev/null ; then - : # mysqld_safe is still running - else - # mysqld_safe is no longer running, abort the wait loop - break - fi - - echo $echo_n ".$echo_c" - i=`expr $i + 1` - sleep 1 - - done - - log_failure_msg - return 1 -} -# -# Set pid file if not given -# -if test -z "$mysqld_pid_file_path" -then - mysqld_pid_file_path=$datadir/`hostname`.pid -else - case "$mysqld_pid_file_path" in - /* ) ;; - * ) mysqld_pid_file_path="$datadir/$mysqld_pid_file_path" ;; - esac -fi - -# source other config files -[ -f /etc/default/mysql ] && . /etc/default/mysql -[ -f /etc/sysconfig/mysql ] && . /etc/sysconfig/mysql -[ -f /etc/conf.d/mysql ] && . /etc/conf.d/mysql - -kill_by_pid() { - # let's see if we can kill the 2 mysql procs by hand - # get the our mysql from ps - eval $(ps -ef | grep "bin/mysqld" | grep -v grep | head -1 | awk '{printf "pid=%d\n", $2}') - - if [ -n "$pid" ]; then - ppid=$(ps -o ppid= -p $pid) - kill -9 $ppid - sleep 1 - kill -9 $pid - echo $echo_n "Force shutting down (no/bad pid file)" - log_success_msg - exit 0 - fi - return -} - -case "$mode" in - 'start') - # Start daemon - - # Safeguard (relative paths, core dumps..) - cd $basedir - - echo $echo_n "Starting MySQL" - if test -x $bindir/mysqld_safe - then - # Give extra arguments to mysqld with the my.cnf file. This script - # may be overwritten at next upgrade. - $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" "$@" >/dev/null 2>&1 & - wait_for_ready; return_value=$? - - # Make lock for RedHat / SuSE - if test -w "$lockdir" - then - touch "$lock_file_path" - fi - - exit $return_value - else - log_failure_msg "Couldn't find MySQL server ($bindir/mysqld_safe)" - fi - ;; - - 'stop') - # Stop daemon. We use a signal here to avoid having to know the - # root password. - - if test -s "$mysqld_pid_file_path" - then - mysqld_pid=`cat "$mysqld_pid_file_path"` - - if (kill -0 $mysqld_pid 2>/dev/null) - then - echo $echo_n "Shutting down MySQL" - kill $mysqld_pid - # mysqld should remove the pid file when it exits, so wait for it. - wait_for_gone $mysqld_pid "$mysqld_pid_file_path"; return_value=$? - else - log_failure_msg "MySQL server process #$mysqld_pid is not running!" - kill_by_pid - rm "$mysqld_pid_file_path" - fi - - # Delete lock for RedHat / SuSE - if test -f "$lock_file_path" - then - rm -f "$lock_file_path" - fi - exit $return_value - else - kill_by_pid - log_failure_msg "MySQL server PID file could not be found!" - fi - ;; - - 'restart') - # Stop the service and regardless of whether it was - # running or not, start it again. - if $0 stop "$@"; then - if ! $0 start "$@"; then - log_failure_msg "Failed to restart server." - exit 1 - fi - else - log_failure_msg "Failed to stop running server, so refusing to try to start." - exit 1 - fi - ;; - - 'reload'|'force-reload') - if test -s "$mysqld_pid_file_path" ; then - read mysqld_pid < "$mysqld_pid_file_path" - kill -HUP $mysqld_pid && log_success_msg "Reloading service MySQL" - touch "$mysqld_pid_file_path" - else - log_failure_msg "MySQL PID file could not be found!" - exit 1 - fi - ;; - 'status') - # First, check to see if pid file exists - if test -s "$mysqld_pid_file_path" ; then - read mysqld_pid < "$mysqld_pid_file_path" - if kill -0 $mysqld_pid 2>/dev/null ; then - log_success_msg "MySQL running ($mysqld_pid)" - exit 0 - else - log_failure_msg "MySQL is not running, but PID file exists" - exit 1 - fi - else - # Try to find appropriate mysqld process - mysqld_pid=`pidof $bindir/mysqld` - - # test if multiple pids exist - pid_count=`echo $mysqld_pid | wc -w` - if test $pid_count -gt 1 ; then - log_failure_msg "Multiple MySQL running but PID file could not be found ($mysqld_pid)" - exit 5 - elif test -z $mysqld_pid ; then - if test -f "$lock_file_path" ; then - log_failure_msg "MySQL is not running, but lock file ($lock_file_path) exists" - exit 2 - fi - log_failure_msg "MySQL is not running" - exit 3 - else - log_failure_msg "MySQL is running but PID file could not be found" - exit 4 - fi - fi - ;; - 'configtest') - # Safeguard (relative paths, core dumps..) - cd $basedir - echo $echo_n "Testing MySQL configuration syntax" - daemon=$bindir/mysqld - if test -x $bindir/mysqld - then - daemon=$bindir/mysqld - elif test -x $sbindir/mysqld - then - daemon=$sbindir/mysqld - elif test -x `which mysqld` - then - daemon=`which mysqld` - else - log_failure_msg "Unable to locate the mysqld binary!" - exit 1 - fi - help_out=`$daemon --help 2>&1`; r=$? - if test "$r" != 0 ; then - log_failure_msg "$help_out" - log_failure_msg "There are syntax errors in the server configuration. Please fix them!" - else - log_success_msg "Syntax OK" - fi - exit $r - ;; - 'bootstrap') - if test "$_use_systemctl" == 1 ; then - log_failure_msg "Please use galera_new_cluster to start the mariadb service with --wsrep-new-cluster" - exit 1 - fi - # Bootstrap the cluster, start the first node - # that initiate the cluster - echo $echo_n "Bootstrapping the cluster.. " - $0 start $other_args --wsrep-new-cluster - exit $? - ;; - *) - # usage - basename=`basename "$0"` - echo "Usage: $basename {start|stop|restart|reload|force-reload|status|configtest|bootstrap} [ MySQL server options ]" - exit 1 - ;; -esac - -exit 0 diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index a5f12555e..15ad7fcc7 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -107,7 +107,6 @@ fi if [ $user = "root" ]; then #setup the columnstore service script rm -f /etc/init.d/columnstore >/dev/null 2>&1 - rm -f /etc/init.d/mysql-Columnstore >/dev/null 2>&1 rm -f /etc/default/columnstore systemctl=`which systemctl 2>/dev/null` @@ -169,7 +168,6 @@ mcsSetConfig -d Installation LockFileDirectory $lockdir mkdir $lockdir >/dev/null 2>&1 -rm -f $lockdir/mysql-Columnstore rm -f $lockdir/columnstore #backup copy of Alarm Config File diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 38f20686e..6c549a677 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -25,7 +25,7 @@ done #stop services columnstore stop > /dev/null 2>&1 -mysql-Columnstore stop > /dev/null 2>&1 +systemctl stop mariadb.service > /dev/null 2>&1 cloud=`mcsGetConfig Installation Cloud` if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then @@ -55,7 +55,6 @@ rm -f /var/lib/columnstore/local/moveDbrootTransactionLog lockdir=`mcsGetConfig Installation LockFileDirectory` rm -f $lockdir/columnstore -rm -f $lockdir/mysql-Columnstore # delete core files #rm -f /var/log/mariadb/columnstore/corefiles/* > /dev/null 2>&1 diff --git a/oam/install_scripts/columnstore.in b/oam/install_scripts/columnstore.in index 858395063..4e4383f93 100644 --- a/oam/install_scripts/columnstore.in +++ b/oam/install_scripts/columnstore.in @@ -90,7 +90,7 @@ stop() { RETVAL=$? rm -f $lockdir/columnstore fuser -k 8604/tcp > /dev/null 2>&1 - mysql-Columnstore stop > /dev/null 2>&1 + systemctl stop mariadb.service > /dev/null 2>&1 pkill ProcMon pkill ProcMgr return $RETVAL diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index d8deaff73..2566e39b3 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -11,7 +11,7 @@ checkForError() { if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then echo "MySQL Password file missing or incorrect, check .my.cnf file" rm -f ${tmpdir}/error.check - mysql-Columnstore stop + systemctl stop mysqld.service sleep 2 exit 2; fi @@ -54,9 +54,11 @@ for arg in "$@"; do done # Restart in the same way that mysqld will be started normally. -mysql-Columnstore stop >/dev/null 2>&1 +systemctl stop mariadb.service >/dev/null 2>&1 sleep 2 -mysql-Columnstore start --skip-grant-tables +export MYSQL_OPTS="--skip-grant-tables" +systemctl start mariadb.service +unset MYSQL_OPTS sleep 5 @@ -65,12 +67,12 @@ install_mcs_mysql.sh --tmpdir=$tmpdir checkForError if [ $? -ne 0 ]; then echo "ERROR: Invalid password in .my.cnf, or Columnstore plugin install missing" - mysql-Columnstore stop + systemctl stop mariadb.service sleep 2 exit 2; fi -mysql-Columnstore stop +systemctl stop mariadb.service exit 0 diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 5fb72f387..48205af38 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -7750,7 +7750,7 @@ int Oam::sendDeviceNotification(std::string deviceName, NOTIFICATION_TYPE type, * * Function: actionMysqlCalpont * - * Purpose: mysql-Columnstore service command + * Purpose: systemctl mariadb.service command * ****************************************************************************/ @@ -7790,8 +7790,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) else return; - // check if mysql-Columnstore is installed - string mysqlscript = "mysql-Columnstore"; + string mysqlscript = "systemctl"; string command; @@ -7801,13 +7800,13 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) { case MYSQL_START: { - command = "start > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + command = "start"; break; } case MYSQL_STOP: { - command = "stop > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + command = "stop"; //set process status try @@ -7822,25 +7821,25 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) case MYSQL_RESTART: { - command = "restart > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + command = "restart"; break; } case MYSQL_RELOAD: { - command = "reload > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + command = "reload"; break; } case MYSQL_FORCE_RELOAD: { - command = "force-reload > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + command = "force-reload"; break; } case MYSQL_STATUS: { - command = "status > " + tmpdir + "/mysql.status"; + command = "status"; break; } @@ -7852,7 +7851,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) } //RUN COMMAND - string cmd = mysqlscript + " " + command; + string cmd = mysqlscript + " " + command + " mariadb.service > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; system(cmd.c_str()); if (action == MYSQL_START || action == MYSQL_RESTART) diff --git a/oamapps/columnstoreSupport/configReport.sh b/oamapps/columnstoreSupport/configReport.sh index a74bc995d..b81125127 100755 --- a/oamapps/columnstoreSupport/configReport.sh +++ b/oamapps/columnstoreSupport/configReport.sh @@ -25,9 +25,6 @@ if [ -n "$chkconfig" ]; then echo "################# chkconfig --list | grep columnstore #################" echo " " chkconfig --list | grep columnstore 2>/dev/null - echo "################# chkconfig --list | grep mysql-Columnstore #################" - echo " " - chkconfig --list | grep mysql-Columnstore 2>/dev/null fi systemctl=`which systemctl 2>/dev/null` @@ -37,9 +34,9 @@ if [ -n "$systemctl" ]; then echo "################# systemctl list-unit-files --type=service | grep columnstore #################" echo " " systemctl list-unit-files --type=service | grep columnstore 2>/dev/null - echo "################# systemctl list-unit-files --type=service | grep mysql-Columnstore #################" + echo "################# systemctl list-unit-files --type=service | grep mariadb #################" echo " " - systemctl list-unit-files --type=service | grep mysql-Columnstore 2>/dev/null + systemctl list-unit-files --type=service | grep mariadb 2>/dev/null fi updaterc=`which update-rc.d 2>/dev/null` @@ -49,9 +46,6 @@ if [ -n "$updaterc" ]; then echo "################# service --status-all | grep columnstore #################" echo " " service --status-all | grep columnstore 2>/dev/null - echo "################# service --status-all | grep mysql-Columnstore #################" - echo " " - service --status-all | grep mysql-Columnstore 2>/dev/null fi diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 6138a305c..53930a7a8 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1514,7 +1514,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO system(cmd.c_str()); cmd = "post-mysql-install >> " + tmpLogDir + "/rpminstall"; system(cmd.c_str()); - cmd = "mysql-Columnstore start > " + tmpLogDir + "/mysqldstart"; + cmd = "systemctl start mariadb.service > " + tmpLogDir + "/mysqldstart"; system(cmd.c_str()); string tmpFile = tmpLogDir + "/mysqldstart"; diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 282a4fade..ce968ec8f 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -753,7 +753,7 @@ checkMysqlPassword() #get MariaDB password pass=true - `mysql-Columnstore start > /dev/null 2>&1` + `systemctl start mariadb.service > /dev/null 2>&1` `mariadb-command-line.sh > /dev/null 2>&1` if [ "$?" -eq 2 ]; then echo "${bold}Failed${normal}, Local Node MariaDB login failed with missing password file, /root/.my.cnf" @@ -763,7 +763,7 @@ checkMysqlPassword() `/bin/cp -f mariadb-command-line.sh ${tmpDir}/.` for ipadd in "${NODE_IPADDRESS[@]}"; do - `remote_command.sh $ipadd $PASSWORD mysql-Columnstore start > /dev/null 2>&1` + `remote_command.sh $ipadd $PASSWORD systemctl start mariadb.service > /dev/null 2>&1` `remote_scp_put.sh $ipadd $PASSWORD ${tmpDir}/mariadb-command-line.sh 1 > ${tmpDir}/remote_scp_put_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_put.sh to $ipadd Node, check ${tmpDir}/remote_scp_put_check" From 7df731a3a6e204aae150cc1d0207968264757b7d Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 4 Feb 2020 14:16:49 +0000 Subject: [PATCH 26/84] Make mysqld work without systemd ColumnStore should now start/stop mysqld where the systemd service is not available. --- .../columnstore-pre-uninstall.in | 9 ++- oam/install_scripts/columnstore.in | 8 ++- oam/install_scripts/post-mysql-install | 40 +++++++++++-- oam/oamcpp/liboamcpp.cpp | 56 ++++++++++++++++--- procmon/processmonitor.cpp | 13 ++++- 5 files changed, 109 insertions(+), 17 deletions(-) diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 6c549a677..41896c937 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -25,7 +25,14 @@ done #stop services columnstore stop > /dev/null 2>&1 -systemctl stop mariadb.service > /dev/null 2>&1 + +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 +else + pkill mysqld +fi cloud=`mcsGetConfig Installation Cloud` if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then diff --git a/oam/install_scripts/columnstore.in b/oam/install_scripts/columnstore.in index 4e4383f93..fc0d952bc 100644 --- a/oam/install_scripts/columnstore.in +++ b/oam/install_scripts/columnstore.in @@ -90,7 +90,13 @@ stop() { RETVAL=$? rm -f $lockdir/columnstore fuser -k 8604/tcp > /dev/null 2>&1 - systemctl stop mariadb.service > /dev/null 2>&1 + # Test we are using systemd + systemctl cat mariadb.service > /dev/null 2>&1 + if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 + else + pkill mysqld + fi pkill ProcMon pkill ProcMgr return $RETVAL diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index 2566e39b3..81808c5da 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -11,7 +11,13 @@ checkForError() { if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then echo "MySQL Password file missing or incorrect, check .my.cnf file" rm -f ${tmpdir}/error.check - systemctl stop mysqld.service + # Test we are using systemd + systemctl cat mariadb.service > /dev/null 2>&1 + if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 + else + pkill mysqld + fi sleep 2 exit 2; fi @@ -54,10 +60,22 @@ for arg in "$@"; do done # Restart in the same way that mysqld will be started normally. -systemctl stop mariadb.service >/dev/null 2>&1 +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 +else + pkill mysqld +fi sleep 2 export MYSQL_OPTS="--skip-grant-tables" -systemctl start mariadb.service +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl start mariadb.service +else + /usr/bin/mysqld_safe --skip-grant-tables & +fi unset MYSQL_OPTS sleep 5 @@ -67,12 +85,24 @@ install_mcs_mysql.sh --tmpdir=$tmpdir checkForError if [ $? -ne 0 ]; then echo "ERROR: Invalid password in .my.cnf, or Columnstore plugin install missing" - systemctl stop mariadb.service + # Test we are using systemd + systemctl cat mariadb.service > /dev/null 2>&1 + if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 + else + pkill mysqld + fi sleep 2 exit 2; fi -systemctl stop mariadb.service +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 +else + pkill mysqld +fi exit 0 diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 48205af38..4cfbc3d70 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -7796,16 +7796,26 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) string pidtmp = tmpdir + "/mysql.pid"; + int no_systemd = system("systemctl cat mariadb.server > /dev/null 2>&1"); + switch (action) { case MYSQL_START: { + if (no_systemd) + { + system("/usr/bin/mysqld_safe &"); + } command = "start"; break; } case MYSQL_STOP: { + if (no_systemd) + { + system("pkill mysqld"); + } command = "stop"; //set process status @@ -7821,18 +7831,31 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) case MYSQL_RESTART: { + if (no_systemd) + { + system("pkill mysqld"); + system("/usr/bin/mysqld_safe &"); + } command = "restart"; break; } case MYSQL_RELOAD: { + if (no_systemd) + { + system("pkill -HUP mysqld"); + } command = "reload"; break; } case MYSQL_FORCE_RELOAD: { + if (no_systemd) + { + system("pkill -HUP mysqld"); + } command = "force-reload"; break; } @@ -7851,19 +7874,36 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) } //RUN COMMAND - string cmd = mysqlscript + " " + command + " mariadb.service > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; - system(cmd.c_str()); + if (!no_systemd) + { + string cmd = mysqlscript + " " + command + " mariadb.service > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + system(cmd.c_str()); + } if (action == MYSQL_START || action == MYSQL_RESTART) { - //get pid - char buf[512]; - FILE *cmd_pipe = popen("pidof -s mysqld", "r"); + pid_t pid = 0; + // Loop check because we mysqld may not start immediately + for (int i=0; i < 10; i++) + { + //get pid + char buf[512]; + FILE *cmd_pipe = popen("pidof -s mysqld", "r"); - fgets(buf, 512, cmd_pipe); - pid_t pid = strtoul(buf, NULL, 10); + fgets(buf, 512, cmd_pipe); + pid = strtoul(buf, NULL, 10); - pclose( cmd_pipe ); + pclose( cmd_pipe ); + + if (pid) + { + break; + } + else + { + sleep(2); + } + } if (!pid) { diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 53930a7a8..e2da321ee 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1514,8 +1514,17 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO system(cmd.c_str()); cmd = "post-mysql-install >> " + tmpLogDir + "/rpminstall"; system(cmd.c_str()); - cmd = "systemctl start mariadb.service > " + tmpLogDir + "/mysqldstart"; - system(cmd.c_str()); + int ret = system("systemctl cat mariadb.service > /dev/null 2>&1"); + if (!ret) + { + cmd = "systemctl start mariadb.service > " + tmpLogDir + "/mysqldstart"; + system(cmd.c_str()); + } + else + { + cmd = "/usr/bin/mysqld_safe & > " + tmpLogDir + "/mysqldstart"; + system(cmd.c_str()); + } string tmpFile = tmpLogDir + "/mysqldstart"; ifstream file (tmpFile.c_str()); From cd5e742a2a13cbc7ff15511d6e0f645a75c78815 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Thu, 6 Feb 2020 11:52:38 +0000 Subject: [PATCH 27/84] MCOL-3764 Fix RENAME TABLE When ALTER TABLE ... ENGINE= is called the following happens: 1. A temporary table is created in the new engine with the real table name but a temporary file name supplied 2. A bulk insert is started from old to new table 3. The old table is dropped 4. The new table is renamed For #1 we use the real table name instead of the temporary file name (otherwise step #2 breaks), we were therefore trying to skip #4. This broke regular RENAME TABLE commands. With this patch we detect if the rename is for a temporary to real table and skip it. Since this is the only instance where we support temporary tables. This patch also fixes issues with extracting table names from file names and some other irrelevant stuff. Longer term if we want to support temporary tables we need to store the provided filename in our metadata since it could be different from table name. --- dbcon/ddlpackage/ddl.l | 2 +- dbcon/mysql/ha_mcs_ddl.cpp | 87 ++++++++++++-------------------------- 2 files changed, 27 insertions(+), 62 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index c644bddf0..4cef8182e 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -70,7 +70,7 @@ self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) digit [0-9] -ident_start [A-Za-z\200-\377_0-9] +ident_start [A-Za-z\200-\377_0-9#] ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* extended_identifier {ident_start}{extended_ident_cont}* diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 8454b8cce..05a3717cb 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -390,32 +390,6 @@ bool validateNextValue( int type, int64_t value ) return validValue; } -static void decode_objectname(char *buf, const char *path, size_t buf_size) -{ - size_t new_path_len = filename_to_tablename(path, buf, buf_size); - buf[new_path_len] = '\0'; -} - -static void decode_file_path(const char *path, char *decoded_dbname, - char *decoded_tbname) -{ - // The format cont ains './' in the beginning of a path. - char *dbname_start = (char*) path + 2; - char *dbname_end = dbname_start; - while (*dbname_end != '/') - dbname_end++; - - int cnt = dbname_end - dbname_start; - char *dbname = (char *)my_alloca(cnt + 1); - memcpy(dbname, dbname_start, cnt); - dbname[cnt] = '\0'; - decode_objectname(decoded_dbname, dbname, FN_REFLEN); - my_afree(dbname); - - char *tbname_start = dbname_end + 1; - decode_objectname(decoded_tbname, tbname_start, FN_REFLEN); -} - bool anyRowInTable(string& schema, string& tableName, int sessionID) { //find a column in the table @@ -914,18 +888,6 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl { CreateTableStatement* createTable = dynamic_cast ( &stmt ); - //@Bug 5767. To handle key words inside `` for a tablename. - if (!(boost::iequals(schema, createTable->fTableDef->fQualifiedName->fSchema)) || !(boost::iequals(table, createTable->fTableDef->fQualifiedName->fName))) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CREATE_DATATYPE_NOT_SUPPORT)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - bool matchedCol = false; bool isFirstTimestamp = true; @@ -2466,7 +2428,7 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH]; String datatype, def_value; ostringstream oss; - string tbl_name = string(share->db.str) + "." + string(share->table_name.str); + string tbl_name = string(share->db.str) + "." + string(share->table_name.str); // Save the current read_set map and mark it for read old_map= tmp_use_all_columns(table_arg, table_arg->read_set); @@ -2634,22 +2596,28 @@ int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_connection_i THD* thd = current_thd; string emsg; - string dbFrom, tblFrom, dbTo, tblTo; - char decodedDbFrom[FN_REFLEN]; - char decodedTblFrom[FN_REFLEN]; - char decodedDbTo[FN_REFLEN]; - char decodedTblTo[FN_REFLEN]; - decode_file_path(from, decodedDbFrom, decodedTblFrom); - decode_file_path(to, decodedDbTo, decodedTblTo); - string stmt; + string tblFrom (from+2); + size_t pos = tblFrom.find("/"); + std::string dbFrom = tblFrom.substr(0, pos); + tblFrom = tblFrom.erase(0, pos + 1); - // This case is already handled - if (thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE) - return 0; + string tblTo (to+2); + pos = tblTo.find("/"); + std::string dbTo = tblTo.substr(0, pos); + tblTo = tblTo.erase(0, pos + 1); + + string stmt; if (thd->slave_thread && !get_replication_slave(thd)) return 0; + // This is a temporary table rename, we don't use the temporary table name + // so this is a NULL op + if (tblFrom.compare(0, 4, "#sql") == 0) + { + return 0; + } + //@bug 5660. Error out REAL DDL/DML on slave node. // When the statement gets here, it's NOT SSO or RESTRICT if (ci.isSlaveNode) @@ -2659,24 +2627,21 @@ int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_connection_i return 1; } - // User moves tables b/w namespaces. - size_t tblFromLength= strlen(decodedTblFrom); - if (!memcmp(decodedTblFrom, decodedTblTo, tblFromLength)) - { - return 0; - } - - stmt.assign("alter table `"); - stmt.append(decodedTblFrom); + stmt.assign("alter table `"); + stmt.append(dbFrom); + stmt.append("`.`"); + stmt.append(tblFrom); stmt.append("` rename to `"); - stmt.append(decodedTblTo); + stmt.append(dbTo); + stmt.append("`.`"); + stmt.append(tblTo); stmt.append("`;"); string db; if (thd->db.length) db = thd->db.str; else - db.assign(decodedDbFrom); + db.assign(dbFrom); int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); From 53afb3c71efdca534123f5e32c15b0a681466db8 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 6 Feb 2020 18:09:52 -0500 Subject: [PATCH 28/84] MCOL-3520: Fix cpimport S3 multi-PM imports Impl'd mode-1 imports from S3 & fixed a few other minor things I noticed. --- writeengine/splitter/CMakeLists.txt | 4 +- writeengine/splitter/we_cmdargs.cpp | 14 +- writeengine/splitter/we_cmdargs.h | 5 +- writeengine/splitter/we_filereadthread.cpp | 144 +++++++++++++-------- writeengine/splitter/we_filereadthread.h | 8 +- writeengine/splitter/we_sdhandler.cpp | 34 +---- writeengine/splitter/we_sdhandler.h | 2 +- 7 files changed, 117 insertions(+), 94 deletions(-) diff --git a/writeengine/splitter/CMakeLists.txt b/writeengine/splitter/CMakeLists.txt index c02645534..1ab3f4b20 100644 --- a/writeengine/splitter/CMakeLists.txt +++ b/writeengine/splitter/CMakeLists.txt @@ -1,5 +1,5 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ) +include_directories( ${ENGINE_COMMON_INCLUDES} ${S3API_DIR} ) ########### next target ############### @@ -17,7 +17,7 @@ set(cpimport_SRCS add_executable(cpimport ${cpimport_SRCS}) -target_link_libraries(cpimport ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} batchloader threadpool) +target_link_libraries(cpimport ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} batchloader threadpool marias3) install(TARGETS cpimport DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index 7235d4935..eafa6e559 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -217,10 +217,10 @@ std::string WECmdArgs::getCpImportCmdLine() if (fbTruncationAsError) aSS << " -S "; - if (!fS3Key.empty()) + if (!fS3Key.empty() && !(fMode == 0 || fMode == 1)) { if (fS3Secret.empty() || fS3Bucket.empty() || fS3Region.empty()) - throw (runtime_error("Not all requried S3 options provided")); + throw (runtime_error("Not all required S3 options provided")); aSS << " -y " << fS3Key; aSS << " -K " << fS3Secret; aSS << " -t " << fS3Bucket; @@ -433,7 +433,7 @@ bool WECmdArgs::checkForCornerCases() "a fully qualified path for the remote file." "\nTry 'cpimport -h' for more information.")); if (!fS3Key.empty()) - throw(runtime_error("Mode 2 & an input file on S3 does not make sense.")); + throw(runtime_error("Mode 2 & an input file from S3 does not make sense.")); } if (fMode == 3) @@ -975,7 +975,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) } if (optind < argc) // see if input file name is given - { + { // 3rd pos parm fLocFile = argv[optind]; @@ -1235,6 +1235,12 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) throw (runtime_error("No schema or local filename specified.")); } } + + /* check for all-or-nothing cmdline args to enable S3 import */ + int s3Tmp = (fS3Key.empty() ? 0 : 1) + (fS3Bucket.empty() ? 0 : 1) + + (fS3Secret.empty() ? 0 : 1) + (fS3Region.empty() ? 0 : 1); + if (s3Tmp != 0 && s3Tmp != 4) + throw runtime_error("The access key, secret, bucket, and region are all required to import from S3"); } std::string WECmdArgs::getJobFileName() diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index 47117932f..8c2b89c97 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -38,6 +38,9 @@ public: WECmdArgs(int argc, char** argv); virtual ~WECmdArgs() {} + typedef std::vector VecInts; + typedef std::vector VecArgs; + void appTestFunction(); void parseCmdLineArgs(int argc, char** argv); std::string getCpImportCmdLine(); @@ -268,9 +271,7 @@ public: void getColumnList( std::set& columnList ) const; private: // variables for SplitterApp - typedef std::vector VecArgs; VecArgs fVecArgs; - typedef std::vector VecInts; VecInts fPmVec; VecArgs fVecJobFiles; //JobFiles splitter from master JobFile diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index 4836dfb46..f6aa8f3b1 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -30,6 +30,7 @@ #include "we_messages.h" #include "we_sdhandler.h" +#include "we_splitterapp.h" #include #include @@ -98,22 +99,9 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh): fSdh(aSdh), } fBuff = new char [fBuffSize]; - - /* Get S3 import params from fSdh.fRef, which is a ref to the we_splitterapp - */ + const WECmdArgs &args = fSdh.fRef.fCmdArgs; initS3Connection(args); - doS3Import = args.isS3Import(); - if (doS3Import) - { - s3Key = args.getS3Key(); - s3Secret = args.getS3Secret(); - s3Bucket = args.getS3Bucket(); - s3Region = args.getS3Region(); - s3Host = args.getS3Host(); - ms3_library_init(); - connection = ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str()));) - } } //WEFileReadThread::WEFileReadThread(const WEFileReadThread& rhs):fSdh(rhs.fSdh) @@ -140,8 +128,14 @@ WEFileReadThread::~WEFileReadThread() if (doS3Import) { - ms3_deinit(connection); + ms3_deinit(s3Connection); ms3_library_deinit(); + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + } } } @@ -162,6 +156,14 @@ void WEFileReadThread::reset() fpThread = 0; //cout << "WEFileReadThread destructor called" << endl; this->setContinue(true); + + if (buf) + { + arrSource.reset(); + s3Stream.reset(); + free(buf); + buf = NULL; + } } //------------------------------------------------------------------------------ @@ -221,37 +223,33 @@ bool WEFileReadThread::chkForListOfFiles(std::string& FileName) std::string aFileName = FileName; istringstream iss(aFileName); + ostringstream oss; size_t start = 0, end = 0; const char* sep = " ,|"; - - end = aFileName.find_first_of(sep); + ms3_status_st ms3status; do { - if (end != string::npos) - { - std::string aFile = aFileName.substr(start, end - start); - - if (fSdh.getDebugLvl() > 2) - cout << "File: " << aFileName.substr(start, end - start) << endl; - - start = end + 1; - fInfileList.push_back(aFile); - } - else - { - std::string aFile = aFileName.substr(start, end - start); - - if (fSdh.getDebugLvl() > 1) - cout << "Next Input File " << aFileName.substr(start, end - start) << endl; - - fInfileList.push_back(aFile); - break; - } - end = aFileName.find_first_of(sep, start); + std::string aFile = aFileName.substr(start, end - start); + if (aFile == "STDIN" || aFile == "stdin") + aFile = "/dev/stdin"; + + if (fSdh.getDebugLvl() > 1) + cout << "Next Input File " << aFile << endl; + + if ((!doS3Import && access(aFile.c_str(), O_RDONLY) != 0) || + (doS3Import && ms3_status(s3Connection, s3Bucket.c_str(), + aFile.c_str(), &ms3status) != 0)) + { + oss << "Could not access " << aFile; + throw runtime_error(oss.str()); + } + + fInfileList.push_back(aFile); + start = end + 1; } - while (start != end); + while (end != string::npos); //cout << "Going out chkForListOfFiles("<< FileName << ")" << endl; @@ -288,6 +286,13 @@ void WEFileReadThread::shutdown() //if(fInFile.is_open()) fInFile.close(); //@BUG 4326 if (fIfFile.is_open()) fIfFile.close(); + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + buf = NULL; + } } //------------------------------------------------------------------------------ @@ -474,21 +479,46 @@ void WEFileReadThread::openInFile() { /* If an S3 transfer use ms3 lib to d/l data into mem - use boost::iostreams to wrap the mem in an fstream interface - assign fiffile and/or finfile to it? - - example (change file_destriptor_source to array_source) - FILE* pipe = popen("find . -type f", "r"); - - io::stream_buffer fpstream(fileno(pipe)); - std::istream in(&fpstream); + use boost::iostreams to wrap the mem in a stream interface + point infile's stream buffer to it. */ - if (fSdh.getDebugLvl()) cout << "Input FileName: " << fInFileName << endl; + if (fSdh.getDebugLvl()) cout << "Input Filename: " << fInFileName << endl; - WORKING HERE + if (doS3Import) + { + size_t bufLen = 0; + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + buf = NULL; + } + if (fSdh.getDebugLvl()) + cout << "Downloading " << fInFileName << endl; + int err = ms3_get(s3Connection, s3Bucket.c_str(), fInFileName.c_str(), + &buf, &bufLen); + if (fSdh.getDebugLvl()) + cout << "Download complete." << endl; + if (err) + { + ostringstream os; + if (ms3_server_error(s3Connection)) + os << "Download of '" << fInFileName << "' failed. Error from the server: " + << ms3_server_error(s3Connection); + else + os << "Download of '" << fInFileName << "' failed. Got '" << ms3_error(err) + << "'."; + throw runtime_error(os.str()); + } + + arrSource.reset(new boost::iostreams::array_source((char *) buf, bufLen)); + s3Stream.reset(new boost::iostreams::stream(*arrSource)); + fInFile.rdbuf(s3Stream->rdbuf()); + } - if (fInFileName == "/dev/stdin") + else if (fInFileName == "/dev/stdin") { char aDefCon[16], aGreenCol[16]; snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); @@ -498,10 +528,11 @@ void WEFileReadThread::openInFile() cout << aGreenCol << "trying to read from STDIN... " << aDefCon << endl; + fInFile.rdbuf(cin.rdbuf()); } //@BUG 4326 - if (fInFileName != "/dev/stdin") + else if (fInFileName != "/dev/stdin") { if (!fIfFile.is_open()) { @@ -625,7 +656,7 @@ int WEFileReadThread::getNextRow(istream& ifs, char* pBuf, int MaxLen) return pEnd - pBuf; } -void WEFileReadThread::initS3Connection(const WE_CmdArgs &args) +void WEFileReadThread::initS3Connection(const WECmdArgs &args) { doS3Import = args.isS3Import(); if (doS3Import) @@ -636,10 +667,13 @@ void WEFileReadThread::initS3Connection(const WE_CmdArgs &args) s3Region = args.getS3Region(); s3Host = args.getS3Host(); ms3_library_init(); - connection = ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str()));) - if (!connection) - throw runtime_error("WEFileReadThread::initS3Connection(): Failed to init S3 connection"); + s3Connection = ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str())); + if (!s3Connection) + throw runtime_error("failed to get an S3 connection"); } + else + s3Connection = NULL; + buf = NULL; } //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_filereadthread.h b/writeengine/splitter/we_filereadthread.h index 5056fdeed..f81f5d492 100644 --- a/writeengine/splitter/we_filereadthread.h +++ b/writeengine/splitter/we_filereadthread.h @@ -30,7 +30,10 @@ #ifndef WE_FILEREADTHREAD_H_ #define WE_FILEREADTHREAD_H_ +#include "we_cmdargs.h" #include "libmarias3/marias3.h" +#include +#include namespace WriteEngine { @@ -164,7 +167,10 @@ private: std::string s3Bucket; std::string s3Region; std::string s3Host; - ms3_st *s3connection; + ms3_st *s3Connection; + uint8_t *buf; + std::unique_ptr arrSource; + std::unique_ptr > s3Stream; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index b23f61120..a819dbda2 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -501,14 +501,9 @@ void WESDHandler::setup() //fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), false); fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), getConsoleLog()); - // In mode 0 and Mode 1, we need to check for local file availability - if ((0 == fRef.fCmdArgs.getMode()) || (1 == fRef.fCmdArgs.getMode())) - { - if (!check4InputFile(fRef.getLocFile())) - { - throw (runtime_error("Could not open Input file " + fRef.getLocFile())); - } - } + // In mode 0 and Mode 1, we need to construct the input file list and check availability + if (0 == fRef.fCmdArgs.getMode() || 1 == fRef.fCmdArgs.getMode()) + setInputFileList(fRef.getLocFile()); fImportRslt.startTimer(); @@ -2726,28 +2721,9 @@ std::string WESDHandler::getTime2Str() const //------------------------------------------------------------------------------ -bool WESDHandler::check4InputFile(std::string InFileName) +void WESDHandler::setInputFileList(std::string InFileName) { - bool aRet = false; - - if ((0 == InFileName.compare("STDIN")) || (0 == InFileName.compare("stdin"))) - { - fFileReadThread.add2InputDataFileList(InFileName); - return true; - } - else - { - //BUG 4342 - Need to support "list of infiles" - fFileReadThread.chkForListOfFiles(InFileName); - std::string aFileName = fFileReadThread.getNextInputDataFile(); - std::ifstream aFile(aFileName.c_str()); - aRet = (aFile.good()) ? true : false; - - // add back to list, which we pop_front for checking the file. - if (aRet) fFileReadThread.add2InputDataFileList(aFileName); - } - - return aRet; + fFileReadThread.chkForListOfFiles(InFileName); } //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_sdhandler.h b/writeengine/splitter/we_sdhandler.h index 551df4d9f..e8d624e1b 100644 --- a/writeengine/splitter/we_sdhandler.h +++ b/writeengine/splitter/we_sdhandler.h @@ -111,7 +111,7 @@ public: bool releaseTableLocks(); void check4CpiInvokeMode(); bool check4PmArguments(); - bool check4InputFile(std::string InFileName); + void setInputFileList(std::string InFileName); bool check4CriticalErrMsgs(std::string& Entry); void onStartCpiResponse(int PmId); From 4d528cc6b848d1158e6a82b1ffdc7bd84d775d99 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 10 Feb 2020 10:20:03 +0000 Subject: [PATCH 29/84] MCOL-3766 EXPLAIN now doesn't execute query via SH thus doesn't fail every other run. --- dbcon/mysql/ha_mcs_pushdown.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index d267ad896..bd3b2cd44 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -883,8 +883,13 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) { handler= new ha_columnstore_select_handler(thd, select_lex); mcs_handler_info mhi= mcs_handler_info(reinterpret_cast(handler), SELECT); - // this::table is the place for the result set - int rc= ha_cs_impl_pushdown_init(&mhi, handler->table); + // handler::table is the place for the result set + int rc= 0; + // Skip execution for EXPLAIN queries + if (!thd->lex->describe) + { + rc= ha_cs_impl_pushdown_init(&mhi, handler->table); + } // Return SH even if init fails b/c CS changed SELECT_LEX structures // with simplify_joins_() From c334a6fb1eafba7e8dc33393ec66cfaea7d93bf7 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 10 Feb 2020 16:24:02 +0000 Subject: [PATCH 30/84] Also add support for collate in column definitions --- dbcon/ddlpackage/ddl.y | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 6fbbb67f2..d0f408e86 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -732,13 +732,25 @@ optional_braces: | '(' ')' {} ; +opt_column_charset: + /* empty */ {} + | + IDB_CHAR SET ident {} + ; + +opt_column_collate: + /* empty */ {} + | + COLLATE ident {} + ; + data_type: - character_string_type + character_string_type opt_column_charset opt_column_collate | binary_string_type | numeric_type | datetime_type | blob_type - | text_type + | text_type opt_column_charset opt_column_collate | IDB_BLOB { $$ = new ColumnType(DDL_BLOB); From b7156eb244f80091693742701ef562c2367e6622 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 10 Feb 2020 11:29:31 -0600 Subject: [PATCH 31/84] MCOL-3663 revert to using posix regex for LIKE --- dbcon/execplan/treenode.h | 5 ----- primitives/linux-port/dictionary.cpp | 1 + primitives/linux-port/primitiveprocessor.h | 6 +++--- utils/dataconvert/dataconvert.h | 9 +++++++++ 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 2eb431aa2..ca1196b27 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -29,11 +29,6 @@ #include #include -#ifdef _MSC_VER -#include -#else -#include -#endif #include #include diff --git a/primitives/linux-port/dictionary.cpp b/primitives/linux-port/dictionary.cpp index b1405589f..003fd3816 100644 --- a/primitives/linux-port/dictionary.cpp +++ b/primitives/linux-port/dictionary.cpp @@ -31,6 +31,7 @@ using namespace std; #include "messageobj.h" #include "exceptclasses.h" #include "utils_utf8.h" +#include "dataconvert.h" #include using namespace funcexp; diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index a02f88b29..366e90daf 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -33,9 +33,9 @@ #include #endif -//#ifdef __linux__ -//#define POSIX_REGEX -//#endif +#ifdef __linux__ +#define POSIX_REGEX +#endif #ifdef POSIX_REGEX #include diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index b3654bc9b..c2d27b1b8 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -36,7 +36,16 @@ #else #include #endif + +#ifdef __linux__ +#define POSIX_REGEX +#endif + +#ifdef POSIX_REGEX +#include +#else #include +#endif #include "calpontsystemcatalog.h" #include "columnresult.h" From 7656a1ead3a25e789c3ee4850bf1e5d7501e5348 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 10 Feb 2020 22:16:03 +0000 Subject: [PATCH 32/84] MCOL-2101 Set up proper permissions for /tmp/columnstore_tmp_files to allow non-root users to write to it. --- oam/install_scripts/columnstore-post-install.in | 1 + oam/install_scripts/columnstore.in | 3 +++ 2 files changed, 4 insertions(+) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index a5f12555e..2e253de53 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -79,6 +79,7 @@ else TempFileDir=`mcsGetConfig SystemConfig TempFileDir` tmpDir=${tmpDir}${TempFileDir} mkdir $tmpDir >/dev/null 2>&1 + chmod 777 $tmpDir fi #set base columnstore temp file directory diff --git a/oam/install_scripts/columnstore.in b/oam/install_scripts/columnstore.in index 858395063..8686b6683 100644 --- a/oam/install_scripts/columnstore.in +++ b/oam/install_scripts/columnstore.in @@ -39,6 +39,9 @@ lockdir=`mcsGetConfig Installation LockFileDirectory` #get temp directory tmpDir=`mcsGetConfig SystemConfig SystemTempFileDir` mkdir $tmpDir >/dev/null 2>&1 +if [ $USER == "root" ]; then + chmod 777 $tmpDir +fi checkInstallSetup() { InitialInstallFlag=`mcsGetConfig -c @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml Installation InitialInstallFlag` From 9e24412681e2034d619df759a77e31602016358c Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Fri, 31 Jan 2020 10:44:08 -0600 Subject: [PATCH 33/84] Merge pull request #1015 from pleblanc1976/mcol-3716 Mcol-3716: Wrong min/max after cpimport which causes wrong select results --- writeengine/bulk/we_bulkloadbuffer.cpp | 12 ++++++------ writeengine/bulk/we_bulkloadbuffer.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 1c65c34d3..60a0a457e 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -719,7 +719,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, errno = 0; origVal = strtoll(field, 0, 10); - + if (errno == ERANGE) bSatVal = true; } @@ -804,7 +804,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL ) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { @@ -979,7 +979,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { @@ -1411,7 +1411,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, strcpy(field, "1"); fieldLength = 1; } - + if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { @@ -1642,7 +1642,7 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) tokenNullFlag = true; } - // convert the data into appropriate format. + // convert the data into appropriate format and update CP values convert(field, tokenLength, tokenNullFlag, buf + i * columnInfo.column.width, columnInfo.column, bufStats); @@ -1673,7 +1673,7 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) lastInputRowInExtent += columnInfo.rowsPerExtent(); - if (isUnsigned(columnInfo.column.dataType)) + if (isUnsigned(columnInfo.column.dataType) || isCharType(columnInfo.column.dataType)) { bufStats.minBufferVal = static_cast(MAX_UBIGINT); bufStats.maxBufferVal = static_cast(MIN_UBIGINT); diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 6f2a431fa..2d1094d5a 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -44,7 +44,7 @@ public: int64_t satCount; BLBufferStats(ColDataType colDataType) : satCount(0) { - if (isUnsigned(colDataType)) + if (isUnsigned(colDataType) || isCharType(colDataType)) { minBufferVal = static_cast(MAX_UBIGINT); maxBufferVal = static_cast(MIN_UBIGINT); From 5ce8994e14dccef43d5eb14285bc57f595738841 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 10 Feb 2020 23:44:11 -0600 Subject: [PATCH 34/84] MCOL-3748: Modify signal handling for different default actions. --- storage-manager/src/main.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/storage-manager/src/main.cpp b/storage-manager/src/main.cpp index e8d68390c..e541f230a 100644 --- a/storage-manager/src/main.cpp +++ b/storage-manager/src/main.cpp @@ -57,6 +57,15 @@ void printKPIs(int sig) } void shutdownSM(int sig) +{ + if (!signalCaught) + { + (SessionManager::get())->shutdownSM(sig); + } + signalCaught = true; +} + +void coreSM(int sig) { if (!signalCaught) { @@ -78,15 +87,20 @@ int main(int argc, char** argv) struct sigaction sa; memset(&sa, 0, sizeof(sa)); - std::vector shutdownSignals{ SIGHUP, SIGINT, SIGQUIT, SIGILL, - SIGTRAP, SIGABRT, SIGBUS, SIGFPE, - SIGSEGV, SIGALRM, SIGTERM, SIGXCPU, - SIGXFSZ, SIGVTALRM, SIGPROF, SIGPOLL, - SIGPWR, SIGSYS}; + std::vector shutdownSignals{ SIGALRM, SIGHUP, SIGINT, SIGKILL, + SIGPOLL, SIGPROF, SIGTERM, SIGVTALRM}; + + std::vector coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL, + SIGQUIT, SIGSEGV, SIGSYS, SIGTRAP, + SIGXCPU, SIGXFSZ}; + sa.sa_handler = shutdownSM; for (int sig : shutdownSignals) sigaction(sig, &sa, NULL); - + + sa.sa_handler = coreSM; + for (int sig : coreSignals) + sigaction(sig, &sa, NULL); sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, NULL); From 89569c25047a5c6a8fb48d4938d1235ca18e3420 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 11 Feb 2020 14:23:17 -0600 Subject: [PATCH 35/84] MCOL-3784 Add boost/scoped_ptr.hpp to rowaggregation.h We were inadvertantly relying on boost/regex.hpp to include boost/scoped_ptr.hpp. When we removed boost regex, we lost scoped_ptr too. --- utils/rowgroup/rowaggregation.h | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 17e903f3f..aeb7dbf53 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -42,6 +42,7 @@ #include #include #include +#include #include "serializeable.h" #include "bytestream.h" From c4d8d3d87d831d01367675080e8200bb7a856e62 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 11 Feb 2020 17:22:30 -0600 Subject: [PATCH 36/84] MCOL-3748: added missing signal to shutdown. Don't shutdown if fatalHandler is called. --- storage-manager/src/main.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage-manager/src/main.cpp b/storage-manager/src/main.cpp index e541f230a..6ce2a1db8 100644 --- a/storage-manager/src/main.cpp +++ b/storage-manager/src/main.cpp @@ -69,7 +69,6 @@ void coreSM(int sig) { if (!signalCaught) { - (SessionManager::get())->shutdownSM(sig); fatalHandler(sig); } signalCaught = true; @@ -88,7 +87,7 @@ int main(int argc, char** argv) memset(&sa, 0, sizeof(sa)); std::vector shutdownSignals{ SIGALRM, SIGHUP, SIGINT, SIGKILL, - SIGPOLL, SIGPROF, SIGTERM, SIGVTALRM}; + SIGPOLL, SIGPROF, SIGPWR, SIGTERM, SIGVTALRM}; std::vector coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGQUIT, SIGSEGV, SIGSYS, SIGTRAP, From fe227c7977f4562edda843fed9b65676df0bbb1a Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 11 Feb 2020 17:29:06 -0600 Subject: [PATCH 37/84] Remove SIGKILL from list. --- storage-manager/src/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-manager/src/main.cpp b/storage-manager/src/main.cpp index 6ce2a1db8..3d0108fe4 100644 --- a/storage-manager/src/main.cpp +++ b/storage-manager/src/main.cpp @@ -86,8 +86,8 @@ int main(int argc, char** argv) struct sigaction sa; memset(&sa, 0, sizeof(sa)); - std::vector shutdownSignals{ SIGALRM, SIGHUP, SIGINT, SIGKILL, - SIGPOLL, SIGPROF, SIGPWR, SIGTERM, SIGVTALRM}; + std::vector shutdownSignals{ SIGALRM, SIGHUP, SIGINT, SIGPOLL, + SIGPROF, SIGPWR, SIGTERM, SIGVTALRM}; std::vector coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGQUIT, SIGSEGV, SIGSYS, SIGTRAP, From 0c13c40201a4132b88771459ce18137376f9aa61 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 12 Feb 2020 08:01:01 +0000 Subject: [PATCH 38/84] MCOL-3769 This commit refactors create_SH() to return an early error if any of optimizer rewrites fails. --- dbcon/mysql/ha_mcs_opt_rewrites.cpp | 10 ++--- dbcon/mysql/ha_mcs_opt_rewrites.h | 3 +- dbcon/mysql/ha_mcs_pushdown.cpp | 62 +++++++++++++++++------------ 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp index 00c08d9e0..6d0d24e68 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.cpp +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -18,7 +18,7 @@ // Search simplify_joins() function in the server's code for detail COND * -simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, +simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj) { TABLE_LIST *table; @@ -26,7 +26,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, TABLE_LIST *prev_table= 0; List_iterator li(*join_list); bool straight_join= MY_TEST(join->select_options & SELECT_STRAIGHT_JOIN); - DBUG_ENTER("simplify_joins"); + DBUG_ENTER("simplify_joins_cs"); /* Try to simplify join operations from join_list. @@ -54,7 +54,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, the outer join is converted to an inner join and the corresponding on expression is added to E. */ - expr= simplify_joins_(join, &nested_join->join_list, + expr= simplify_joins_cs(join, &nested_join->join_list, expr, FALSE, in_sj || table->sj_on_expr); if (!table->prep_on_expr || expr != table->on_expr) @@ -67,7 +67,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, } nested_join->used_tables= (table_map) 0; nested_join->not_null_tables=(table_map) 0; - conds= simplify_joins_(join, &nested_join->join_list, conds, top, + conds= simplify_joins_cs(join, &nested_join->join_list, conds, top, in_sj || table->sj_on_expr); used_tables= nested_join->used_tables; not_null_tables= nested_join->not_null_tables; @@ -241,5 +241,5 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, li.replace(repl_list); } } - DBUG_RETURN(conds); + DBUG_RETURN(conds); } diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h index d433c06a5..30d6b4dbe 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.h +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -20,7 +20,6 @@ #include "idb_mysql.h" -COND *simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); - +COND *simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); #endif diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index bd3b2cd44..65b369661 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -21,7 +21,6 @@ void check_walk(const Item* item, void* arg); - void disable_indices_for_CEJ(THD *thd_) { TABLE_LIST* global_list; @@ -42,7 +41,7 @@ void disable_indices_for_CEJ(THD *thd_) } } -bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) +bool optimize_unflattened_subqueries_cs(SELECT_LEX *select_lex) { bool result = false; TABLE_LIST *tbl; @@ -52,7 +51,7 @@ bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) if (tbl->is_view_or_derived()) { SELECT_LEX *dsl = tbl->derived->first_select(); - result = optimize_unflattened_subqueries_(dsl); + result = optimize_unflattened_subqueries_cs(dsl); } } @@ -546,7 +545,7 @@ create_columnstore_derived_handler(THD* thd, TABLE_LIST *derived) //To search for CROSS JOIN-s we use tree invariant //G(V,E) where [V] = [E]+1 List join_preds_list; - TABLE_LIST *tl; + TABLE_LIST *tl; for (tl = sl->get_table_list(); !unsupported_feature && tl; tl = tl->next_local) { Item_cond* where_icp= 0; @@ -808,7 +807,6 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } - bool unsupported_feature = false; // Select_handler use the short-cut that effectively disables // INSERT..SELECT, LDI, SELECT..INTO OUTFILE if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT @@ -816,28 +814,41 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) || (thd->lex)->exchange) { - unsupported_feature = true; + return handler; } + // We apply dedicated rewrites from MDB here so the data structures + // becomes dirty and CS has to raise an error in case of any problem. + bool unsupported_feature = false; + logging::Message::Args args; JOIN *join= select_lex->join; - // Next block tries to execute the query using SH very early to fallback - // if execution fails. - if (!unsupported_feature) { disable_indices_for_CEJ(thd); if (select_lex->handle_derived(thd->lex, DT_MERGE)) { - // early quit b/c of the error in handle_derived - return handler; + // set to true b/c of an error in handle_derived + unsupported_feature = true; + args.add("in handle_derived()"); } - COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); + COND *conds; + if (!unsupported_feature) + { + conds = simplify_joins_cs(join, select_lex->join_list, + join->conds, TRUE, FALSE); + } + // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add // an equi-JOIN condition. - optimize_unflattened_subqueries_(select_lex); + if (!unsupported_feature + && optimize_unflattened_subqueries_cs(select_lex)) + { + unsupported_feature = true; + args.add("in optimize_unflattened_subqueries_cs()"); + } - if (conds) + if (!unsupported_feature && conds) { #ifdef DEBUG_WALK_COND conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); @@ -845,15 +856,6 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) join->conds = conds; } - // Impossible HAVING or WHERE - // TODO replace with function call - if (unsupported_feature - || select_lex->having_value == Item::COND_FALSE - || select_lex->cond_value == Item::COND_FALSE ) - { - unsupported_feature = true; - restore_optimizer_flags(thd); - } } // Restore back the saved group_list @@ -874,6 +876,7 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) item_check(item, &unsupported_feature); if (unsupported_feature) { + args.add("in item_check()"); break; } } @@ -890,13 +893,20 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) { rc= ha_cs_impl_pushdown_init(&mhi, handler->table); } + unsupported_feature = (rc) ? true : unsupported_feature; // Return SH even if init fails b/c CS changed SELECT_LEX structures - // with simplify_joins_() - if (rc) - unsupported_feature = true; + // with simplify_joins_cs and other rewrites() + // the handler will return error on next_row() return handler; } + else + { + std::string emsg= + logging::IDBErrorInfo::instance()->errorMsg(ER_INTERNAL_ERROR, args); + thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); + } + restore_optimizer_flags(thd); return NULL; } From e43d9be05b51e39a198d573cf422891845f3f40b Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 12 Feb 2020 09:14:07 +0000 Subject: [PATCH 39/84] MCOL-3721 Fix quoting issues for COLLATE --- dbcon/ddlpackage/ddl.y | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index d0f408e86..f95f3e3fb 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -199,7 +199,7 @@ BOOL BOOLEAN MEDIUMINT TIMESTAMP %type trunc_table_statement %type rename_table_statement %type ident - +%type opt_quoted_literal %% stmtblock: stmtmulti { x->fParseTree = $1; } ; @@ -490,9 +490,9 @@ table_option: | DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair("default charset", $5);} | - DEFAULT COLLATE opt_equal ident {$$ = new pair("default collate", $4);} + DEFAULT COLLATE opt_equal opt_quoted_literal {$$ = new pair("default collate", $4);} | - COLLATE opt_equal ident {$$ = new pair("default collate", $3);} + COLLATE opt_equal opt_quoted_literal {$$ = new pair("default collate", $3);} ; alter_table_statement: @@ -735,13 +735,13 @@ optional_braces: opt_column_charset: /* empty */ {} | - IDB_CHAR SET ident {} + IDB_CHAR SET opt_quoted_literal {} ; opt_column_collate: /* empty */ {} | - COLLATE ident {} + COLLATE opt_quoted_literal {} ; data_type: @@ -864,6 +864,12 @@ string_literal: '\'' SCONST '\'' {$$ = $2;} ; +opt_quoted_literal: + string_literal + | + ident + ; + character_string_type: CHARACTER { From f9e5bdbc8d2f519e5a8610062a514759f3702e9a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 12 Feb 2020 09:57:22 +0000 Subject: [PATCH 40/84] Fix restart hang mcsadmin restart will do a status call. Before this would log status output to a file which would be checked. If that file check failed a PID check would be made. We do not log the status to a file any more since it could have multiple different outputs. The PID check could hit a race condition where it is being checked before the process is up. This would mean there is no module state change causing a hang. This fix does the following: 1. Remove the file status check completely 2. Loop the PID check to give it time to come up 3. If the PID check fails drop to a DEGRADED state This makes mcsadmin restart work correctly again. --- oam/oamcpp/liboamcpp.cpp | 55 +++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 4cfbc3d70..c0a020a10 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -7927,11 +7927,11 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) getProcessStatus("mysqld", moduleName, procstat); int state = procstat.ProcessOpState; pid_t pidStatus = procstat.ProcessID; - + pid_t pid = 0; string mysqlStatus = tmpdir + "/mysql.status"; - if (checkLogStatus(mysqlStatus, "MySQL running")) + if ( state != ACTIVE ) { - if ( state != ACTIVE ) + for (int i=0; i < 10; i++) { //get pid char buf[512]; @@ -7942,17 +7942,26 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) pclose( cmd_pipe ); - //set process status - try + if (pid) { - setProcessStatus("mysqld", moduleName, ACTIVE, pid); + //set process status + try + { + setProcessStatus("mysqld", moduleName, ACTIVE, pid); + } + catch (...) + {} + return; + } + else + { + sleep(2); } - catch (...) - {} - - return; } - else + } + else + { + for (int i=0; i < 10; i++) { //check if pid has changed char buf[512]; @@ -7963,18 +7972,28 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) pclose( cmd_pipe ); - if ( pidStatus != pid ) + if (pid) { - //set process status - try + if ( pidStatus != pid ) { - setProcessStatus("mysqld", moduleName, ACTIVE, pid); + //set process status + try + { + setProcessStatus("mysqld", moduleName, ACTIVE, pid); + } + catch (...) + {} + break; } - catch (...) - {} + } + else + { + sleep(2); } } - + } + if (pid) + { //check module status, if DEGRADED set to ACTIVE int opState; bool degraded; From abad95bc04c320ec112326a72846cbbcf2786eea Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 13 Feb 2020 16:54:32 -0500 Subject: [PATCH 41/84] MCOL-3776: PrimProc crash Added sync to timezone var accesses. --- utils/funcexp/func_add_time.cpp | 2 +- utils/funcexp/func_between.cpp | 2 +- utils/funcexp/func_bitand.cpp | 2 +- utils/funcexp/func_bitwise.cpp | 22 +++++++++++----------- utils/funcexp/func_cast.cpp | 18 +++++++++--------- utils/funcexp/func_ceil.cpp | 4 ++-- utils/funcexp/func_char_length.cpp | 2 +- utils/funcexp/func_date.cpp | 2 +- utils/funcexp/func_date_add.cpp | 2 +- utils/funcexp/func_date_format.cpp | 4 ++-- utils/funcexp/func_day.cpp | 2 +- utils/funcexp/func_dayname.cpp | 2 +- utils/funcexp/func_dayofweek.cpp | 2 +- utils/funcexp/func_dayofyear.cpp | 2 +- utils/funcexp/func_extract.cpp | 2 +- utils/funcexp/func_floor.cpp | 4 ++-- utils/funcexp/func_hour.cpp | 2 +- utils/funcexp/func_if.cpp | 18 +++++++++--------- utils/funcexp/func_last_day.cpp | 2 +- utils/funcexp/func_math.cpp | 2 +- utils/funcexp/func_minute.cpp | 2 +- utils/funcexp/func_month.cpp | 2 +- utils/funcexp/func_monthname.cpp | 2 +- utils/funcexp/func_nullif.cpp | 2 +- utils/funcexp/func_quarter.cpp | 2 +- utils/funcexp/func_regexp.cpp | 2 +- utils/funcexp/func_round.cpp | 2 +- utils/funcexp/func_second.cpp | 2 +- utils/funcexp/func_str_to_date.cpp | 14 +++++++------- utils/funcexp/func_time.cpp | 2 +- utils/funcexp/func_time_format.cpp | 2 +- utils/funcexp/func_time_to_sec.cpp | 2 +- utils/funcexp/func_timediff.cpp | 6 +++--- utils/funcexp/func_timestampdiff.cpp | 4 ++-- utils/funcexp/func_to_days.cpp | 2 +- utils/funcexp/func_truncate.cpp | 2 +- utils/funcexp/func_week.cpp | 2 +- utils/funcexp/func_weekday.cpp | 2 +- utils/funcexp/func_year.cpp | 2 +- utils/funcexp/func_yearweek.cpp | 2 +- utils/funcexp/functor.cpp | 2 +- utils/funcexp/functor.h | 7 ++++++- 42 files changed, 85 insertions(+), 80 deletions(-) diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index a95eff53d..f5f501949 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -181,7 +181,7 @@ int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, TimeStamp timestamp(val1); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 8330c4fc0..d742c2d16 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -344,7 +344,7 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal if (cc) { Result result = cc->result(); - result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone()); cc->result(result); } } diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 64fa4078b..2863eadf8 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -162,7 +162,7 @@ int64_t Func_bitand::getIntVal(Row& row, TimeStamp timestamp(parm[i]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); values.push_back((m_time.month * 100000000000000) + (m_time.day * 1000000000000) + (m_time.hour * 10000000000) + (m_time.minute * 100000000) + (m_time.second * 1000000) + timestamp.msecond); } diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 028aedc1c..2154f125b 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -195,8 +195,8 @@ int64_t Func_bitand::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "bitand: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -231,8 +231,8 @@ int64_t Func_leftshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "leftshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -267,8 +267,8 @@ int64_t Func_rightshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "rightshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -303,8 +303,8 @@ int64_t Func_bitor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "bitor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -347,8 +347,8 @@ int64_t Func_bitxor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "bitxor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -382,7 +382,7 @@ int64_t Func_bit_count::getIntVal(Row& row, uint64_t val = 0; - if (!getUIntValFromParm(row, parm[0], val, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val, isNull, timeZone())) { std::ostringstream oss; oss << "bit_count: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index d335ddc81..ce3d6a531 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -218,7 +218,7 @@ int64_t Func_cast_signed::getIntVal(Row& row, int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(fTimeZone); + return dt.convertToMySQLint(timeZone()); } break; @@ -379,7 +379,7 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(fTimeZone); + return dt.convertToMySQLint(timeZone()); } break; @@ -509,7 +509,7 @@ string Func_cast_char::getStrVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { - return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone).substr(0, length); + return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()).substr(0, length); } break; @@ -676,7 +676,7 @@ int32_t Func_cast_date::getDateIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - string value = dataconvert::DataConvert::timestampToString(val1, fTimeZone); + string value = dataconvert::DataConvert::timestampToString(val1, timeZone()); value = value.substr(0, 10); return dataconvert::DataConvert::stringToDate(value); } @@ -803,7 +803,7 @@ int64_t Func_cast_date::getDatetimeIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -985,7 +985,7 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -1099,7 +1099,7 @@ int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); Time time; time.hour = m_time.hour; time.minute = m_time.minute; @@ -1478,7 +1478,7 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, { int32_t s = 0; - string value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + string value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); //strip off micro seconds string date = value.substr(0, 14); @@ -1599,7 +1599,7 @@ double Func_cast_double::getDoubleVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); // strip off micro seconds str = str.substr(0, 14); diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index b1f886166..f9f9c1891 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -164,7 +164,7 @@ int64_t Func_ceil::getIntVal(Row& row, TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); if (!isNull) - ret = dt.convertToMySQLint(fTimeZone); + ret = dt.convertToMySQLint(timeZone()); } break; @@ -269,7 +269,7 @@ uint64_t Func_ceil::getUintVal(Row& row, TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); if (!isNull) - ret = dt.convertToMySQLint(fTimeZone); + ret = dt.convertToMySQLint(timeZone()); } break; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index c7c382c94..72bdfdfda 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -103,7 +103,7 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { - string date = dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + string date = dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); return (int64_t)date.size(); } diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 7280e838a..76a50a90f 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -78,7 +78,7 @@ int64_t Func_date::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - value = dataconvert::DataConvert::timestampToString(val1, fTimeZone); + value = dataconvert::DataConvert::timestampToString(val1, timeZone()); value = value.substr(0, 10); break; } diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 5ed920e44..5004f5521 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -782,7 +782,7 @@ int64_t Func_date_add::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index e62bc3952..105069c59 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -275,7 +275,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, TimeStamp timestamp(val); int64_t seconds = timestamp.second; MySQLTime time; - gmtSecToMySQLTime(seconds, time, fTimeZone); + gmtSecToMySQLTime(seconds, time, timeZone()); dt.year = time.year; dt.month = time.month; dt.day = time.day; @@ -419,7 +419,7 @@ int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), fTimeZone); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); } diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index c8ba97e6c..c414fd5d4 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -67,7 +67,7 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.day; } diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index 02955fcf4..e46bfa46f 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -79,7 +79,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); year = time.year; month = time.month; day = time.day; diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index e66e1d864..3a29f592d 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -77,7 +77,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); year = time.year; month = time.month; day = time.day; diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 6d4fc9fcf..ee13730c1 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -76,7 +76,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index b8b1f90bc..f69b8dcbe 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -252,7 +252,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); dataconvert::DateTime dt; dt.year = m_time.year; dt.month = m_time.month; diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index 50dd3d919..1e0ee7f09 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -161,7 +161,7 @@ int64_t Func_floor::getIntVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); // strip off micro seconds str = str.substr(0, 14); @@ -278,7 +278,7 @@ uint64_t Func_floor::getUintVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); // strip off micro seconds str = str.substr(0, 14); diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 6959ce526..666e84ce0 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -114,7 +114,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.hour; } diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index b17ad7317..ef2827bbe 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -134,7 +134,7 @@ int64_t Func_if::getIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getIntVal(row, isNull); } @@ -151,7 +151,7 @@ string Func_if::getStrVal(Row& row, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getStrVal(row, isNull); } @@ -167,7 +167,7 @@ IDB_Decimal Func_if::getDecimalVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDecimalVal(row, isNull); } @@ -183,7 +183,7 @@ double Func_if::getDoubleVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDoubleVal(row, isNull); } @@ -198,7 +198,7 @@ long double Func_if::getLongDoubleVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getLongDoubleVal(row, isNull); } @@ -214,7 +214,7 @@ int32_t Func_if::getDateIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDateIntVal(row, isNull); } @@ -230,7 +230,7 @@ int64_t Func_if::getDatetimeIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDatetimeIntVal(row, isNull); } @@ -246,7 +246,7 @@ int64_t Func_if::getTimestampIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getTimestampIntVal(row, isNull); } @@ -262,7 +262,7 @@ int64_t Func_if::getTimeIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getTimeIntVal(row, isNull); } diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index c201898db..2099ea71a 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -77,7 +77,7 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 705a13620..13c738217 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1834,7 +1834,7 @@ string Func_format::getStrVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { - value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); } break; diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index 10b3f1ef6..6cdd109a6 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -114,7 +114,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.minute; } diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index 94708fb1b..ffb3f5fc1 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -66,7 +66,7 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.month; } diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 831d16586..7c27c9581 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -107,7 +107,7 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); return time.month; } diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index a81b439dc..3af8a7d4b 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -978,7 +978,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, string value; if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), fTimeZone); + value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), timeZone()); else value = DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index e402444a7..6cc4316c4 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -70,7 +70,7 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); month = m_time.month; break; } diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 809cb239a..120a727e6 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -244,7 +244,7 @@ bool Func_regexp::getBoolVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct, fTimeZone) && !isNull; + return getBool(row, pm, isNull, ct, timeZone()) && !isNull; } diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 293f94021..0064ae8df 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -490,7 +490,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, string value; if (op_ct.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); else value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 4f12cfca8..45cca7e21 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -113,7 +113,7 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.second; } diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index 3432ba100..c4b0226f2 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -187,7 +187,7 @@ string Func_str_to_date::getStrVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); string convertedDate = dataconvert::DataConvert::datetimeToString(*((long long*) &dateTime)); return convertedDate; } @@ -198,7 +198,7 @@ int32_t Func_str_to_date::getDateIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return ((((int32_t)(time >> 32)) & 0xFFFFFFC0) | 0x3E); } @@ -209,7 +209,7 @@ int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } @@ -220,7 +220,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); dataconvert::TimeStamp timestamp; dataconvert::MySQLTime m_time; m_time.year = dateTime.year; @@ -230,7 +230,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, m_time.minute = dateTime.minute; m_time.second = dateTime.second; bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, fTimeZone, isValid); + int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone(), isValid); if (!isValid) { timestamp = -1; @@ -252,7 +252,7 @@ int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, { dataconvert::DateTime dateTime; dataconvert::Time retTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); retTime.day = 0; retTime.is_neg = false; retTime.hour = dateTime.hour; @@ -269,7 +269,7 @@ int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index b83c678db..70202ac63 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -123,7 +123,7 @@ string Func_time::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); dataconvert::Time time; time.hour = m_time.hour; time.minute = m_time.minute; diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index 08739b046..5dd2f666a 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -77,7 +77,7 @@ string Func_time_format::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); hour = m_time.hour; min = m_time.minute; sec = m_time.second; diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index f8391d12f..e19ce2f81 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -76,7 +76,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); hour = time.hour; min = time.minute; sec = time.second; diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 881c04017..5bf16e413 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -152,7 +152,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -239,7 +239,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -332,7 +332,7 @@ int64_t Func_timediff::getTimestampIntVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), fTimeZone); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); } int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index 60b18cdc7..3972554d3 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -57,7 +57,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; @@ -84,7 +84,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[1]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); dt2.year = m_time.year; dt2.month = m_time.month; dt2.day = m_time.day; diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index b51eb5aff..26a993d1a 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -90,7 +90,7 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 441047f74..057a1c299 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -497,7 +497,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, int64_t x = 0; string value = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); s = parm[1]->data()->getIntVal(row, isNull); diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index 326f1c13a..af024cfe4 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -80,7 +80,7 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index 9b6cc8c63..04ca938b4 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -76,7 +76,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 94abdd79a..2c5ec41f9 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -66,7 +66,7 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.year; } diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 3ae72a7f2..91bb60a8c 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -83,7 +83,7 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 06999c9fd..7fb55b47e 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -112,7 +112,7 @@ uint64_t Func::stringToDatetime(const string str) uint64_t Func::stringToTimestamp(const string str) { - int64_t ret = DataConvert::stringToTimestamp(str, fTimeZone); + int64_t ret = DataConvert::stringToTimestamp(str, timeZone()); if (ret == -1) { diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 357d8639d..12dadc83a 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "parsetree.h" #include "exceptclasses.h" @@ -73,10 +74,12 @@ public: const std::string timeZone() const { + std::unique_lock l(tzMutex); return fTimeZone; } void timeZone(const std::string timeZone) { + std::unique_lock l(tzMutex); fTimeZone = timeZone; } @@ -201,7 +204,6 @@ protected: virtual int64_t addTime(dataconvert::Time& dt1, dataconvert::Time& dt2); std::string fFuncName; - std::string fTimeZone; private: //defaults okay @@ -213,6 +215,9 @@ private: float fFloatNullVal; double fDoubleNullVal; long double fLongDoubleNullVal; + + std::string fTimeZone; + mutable std::mutex tzMutex; }; From 6d98de39223835dfb333c976975c1fd0027aeec1 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 14 Feb 2020 00:58:02 +0000 Subject: [PATCH 42/84] Use const_cast to assign to field_length due to change in server 10.4 commit 1394216e3db67152a1356fddd8ddcd563b4f38c3 --- dbcon/mysql/ha_mcs_impl.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 7ab1cbe61..3ffa33ccd 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -752,7 +752,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h // bug 3485, reserve enough space for the longest float value // -3.402823466E+38 to -1.175494351E-38, 0, and // 1.175494351E-38 to 3.402823466E+38. - (*f)->field_length = 40; + const_cast((*f)->field_length) = 40; f2->store(dl); @@ -774,7 +774,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h // bug 3483, reserve enough space for the longest double value // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; + const_cast((*f)->field_length) = 310; // The server converts dl=-0 to dl=0 in f2->store(). // This happens in the call to truncate_double(). @@ -819,7 +819,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h // bug 3483, reserve enough space for the longest double value // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; + const_cast((*f)->field_length) = 310; f2->store(static_cast(dl)); if ((*f)->null_ptr) From ae515a36b85f715fb08673072c74833d64ca2478 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 14 Feb 2020 18:23:54 -0500 Subject: [PATCH 43/84] Revert "Merge pull request #1046 from drrtuy/MCOL-3769" This has an uninit'd var warning from the compiler. It fixes one test and breaks 2 others in test001. This reverts commit cf668897eff4395cc929abffb0cd87df0305c88b, reversing changes made to 25084d002073f9cc1bc5fe6f9feb8ddbbd51ef4d. --- dbcon/mysql/ha_mcs_opt_rewrites.cpp | 10 ++--- dbcon/mysql/ha_mcs_opt_rewrites.h | 3 +- dbcon/mysql/ha_mcs_pushdown.cpp | 62 ++++++++++++----------------- 3 files changed, 33 insertions(+), 42 deletions(-) diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp index 6d0d24e68..00c08d9e0 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.cpp +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -18,7 +18,7 @@ // Search simplify_joins() function in the server's code for detail COND * -simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top, +simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj) { TABLE_LIST *table; @@ -26,7 +26,7 @@ simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top TABLE_LIST *prev_table= 0; List_iterator li(*join_list); bool straight_join= MY_TEST(join->select_options & SELECT_STRAIGHT_JOIN); - DBUG_ENTER("simplify_joins_cs"); + DBUG_ENTER("simplify_joins"); /* Try to simplify join operations from join_list. @@ -54,7 +54,7 @@ simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top the outer join is converted to an inner join and the corresponding on expression is added to E. */ - expr= simplify_joins_cs(join, &nested_join->join_list, + expr= simplify_joins_(join, &nested_join->join_list, expr, FALSE, in_sj || table->sj_on_expr); if (!table->prep_on_expr || expr != table->on_expr) @@ -67,7 +67,7 @@ simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top } nested_join->used_tables= (table_map) 0; nested_join->not_null_tables=(table_map) 0; - conds= simplify_joins_cs(join, &nested_join->join_list, conds, top, + conds= simplify_joins_(join, &nested_join->join_list, conds, top, in_sj || table->sj_on_expr); used_tables= nested_join->used_tables; not_null_tables= nested_join->not_null_tables; @@ -241,5 +241,5 @@ simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top li.replace(repl_list); } } - DBUG_RETURN(conds); + DBUG_RETURN(conds); } diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h index 30d6b4dbe..d433c06a5 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.h +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -20,6 +20,7 @@ #include "idb_mysql.h" -COND *simplify_joins_cs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); +COND *simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); + #endif diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 65b369661..bd3b2cd44 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -21,6 +21,7 @@ void check_walk(const Item* item, void* arg); + void disable_indices_for_CEJ(THD *thd_) { TABLE_LIST* global_list; @@ -41,7 +42,7 @@ void disable_indices_for_CEJ(THD *thd_) } } -bool optimize_unflattened_subqueries_cs(SELECT_LEX *select_lex) +bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) { bool result = false; TABLE_LIST *tbl; @@ -51,7 +52,7 @@ bool optimize_unflattened_subqueries_cs(SELECT_LEX *select_lex) if (tbl->is_view_or_derived()) { SELECT_LEX *dsl = tbl->derived->first_select(); - result = optimize_unflattened_subqueries_cs(dsl); + result = optimize_unflattened_subqueries_(dsl); } } @@ -545,7 +546,7 @@ create_columnstore_derived_handler(THD* thd, TABLE_LIST *derived) //To search for CROSS JOIN-s we use tree invariant //G(V,E) where [V] = [E]+1 List join_preds_list; - TABLE_LIST *tl; + TABLE_LIST *tl; for (tl = sl->get_table_list(); !unsupported_feature && tl; tl = tl->next_local) { Item_cond* where_icp= 0; @@ -807,6 +808,7 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } + bool unsupported_feature = false; // Select_handler use the short-cut that effectively disables // INSERT..SELECT, LDI, SELECT..INTO OUTFILE if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT @@ -814,41 +816,28 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) || (thd->lex)->exchange) { - return handler; + unsupported_feature = true; } - // We apply dedicated rewrites from MDB here so the data structures - // becomes dirty and CS has to raise an error in case of any problem. - bool unsupported_feature = false; - logging::Message::Args args; JOIN *join= select_lex->join; + // Next block tries to execute the query using SH very early to fallback + // if execution fails. + if (!unsupported_feature) { disable_indices_for_CEJ(thd); if (select_lex->handle_derived(thd->lex, DT_MERGE)) { - // set to true b/c of an error in handle_derived - unsupported_feature = true; - args.add("in handle_derived()"); + // early quit b/c of the error in handle_derived + return handler; } - COND *conds; - if (!unsupported_feature) - { - conds = simplify_joins_cs(join, select_lex->join_list, - join->conds, TRUE, FALSE); - } - + COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add // an equi-JOIN condition. - if (!unsupported_feature - && optimize_unflattened_subqueries_cs(select_lex)) - { - unsupported_feature = true; - args.add("in optimize_unflattened_subqueries_cs()"); - } + optimize_unflattened_subqueries_(select_lex); - if (!unsupported_feature && conds) + if (conds) { #ifdef DEBUG_WALK_COND conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); @@ -856,6 +845,15 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) join->conds = conds; } + // Impossible HAVING or WHERE + // TODO replace with function call + if (unsupported_feature + || select_lex->having_value == Item::COND_FALSE + || select_lex->cond_value == Item::COND_FALSE ) + { + unsupported_feature = true; + restore_optimizer_flags(thd); + } } // Restore back the saved group_list @@ -876,7 +874,6 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) item_check(item, &unsupported_feature); if (unsupported_feature) { - args.add("in item_check()"); break; } } @@ -893,20 +890,13 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) { rc= ha_cs_impl_pushdown_init(&mhi, handler->table); } - unsupported_feature = (rc) ? true : unsupported_feature; // Return SH even if init fails b/c CS changed SELECT_LEX structures - // with simplify_joins_cs and other rewrites() - // the handler will return error on next_row() + // with simplify_joins_() + if (rc) + unsupported_feature = true; return handler; } - else - { - std::string emsg= - logging::IDBErrorInfo::instance()->errorMsg(ER_INTERNAL_ERROR, args); - thd->raise_error_printf(ER_INTERNAL_ERROR, emsg.c_str()); - } - restore_optimizer_flags(thd); return NULL; } From e7703f44dd4e7707d8bcc123cafc61aa939f8a36 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 16 Feb 2020 07:13:34 +0000 Subject: [PATCH 44/84] MCOL-3791 This patch implicitly enables disk space preallocation for non-compressed abbreviated extents as it worked before MCOL-498. --- writeengine/shared/we_fileop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index d4f9f1665..f4c65ed52 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -1130,7 +1130,7 @@ int FileOp::initColumnExtent( // Couldn't avoid preallocation for full extents, // e.g. ADD COLUMN DDL b/c CS has to fill the file // with empty magics. - if ( !bOptExtension ) + if ( !bOptExtension || !m_compressionType ) { #ifdef PROFILE Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); From 9605c951dc1b2fab3e5f5bcf8b3e5b8b551be005 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 17 Feb 2020 09:53:02 +0000 Subject: [PATCH 45/84] MCOL-3793 Fix UM ACTIVE / DEGRADED flip-flop OAM mysqld status check was recording the pid of mysqld but this was being masked by another variable. So on every ACTIVE check it would flip to DEGRADED and on every DEGRADED check it would flip to ACTIVE. Variable masking is now removed. Also fixed a typo in the systemd check. --- oam/oamcpp/liboamcpp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index c0a020a10..1a02a9cb9 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -7796,7 +7796,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) string pidtmp = tmpdir + "/mysql.pid"; - int no_systemd = system("systemctl cat mariadb.server > /dev/null 2>&1"); + int no_systemd = system("systemctl cat mariadb.service > /dev/null 2>&1"); switch (action) { @@ -7938,7 +7938,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) FILE *cmd_pipe = popen("pidof -s mysqld", "r"); fgets(buf, 512, cmd_pipe); - pid_t pid = strtoul(buf, NULL, 10); + pid = strtoul(buf, NULL, 10); pclose( cmd_pipe ); @@ -7968,7 +7968,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) FILE *cmd_pipe = popen("pidof -s mysqld", "r"); fgets(buf, 512, cmd_pipe); - pid_t pid = strtoul(buf, NULL, 10); + pid = strtoul(buf, NULL, 10); pclose( cmd_pipe ); From 437e8650a73bea3615f4f9d634dbe3f925069fa1 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 17 Feb 2020 17:35:05 +0000 Subject: [PATCH 46/84] MCOL-3769 This commit divides create_SH in two parts: before rewrites and after rewrites. We can safely fallback from SH to DH and table mode from 'before rewrites' part. We also returns meaningful warning. --- dbcon/mysql/ha_mcs.h | 1 + dbcon/mysql/ha_mcs_pushdown.cpp | 73 ++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index 8baf31e7e..9a184e58e 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -23,6 +23,7 @@ #include "ha_mcs_sysvars.h" extern handlerton* mcs_hton; +#define CS_WARNING_ID 9999 /** @brief COLUMNSTORE_SHARE is a structure that will be shared among all open handlers. diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index bd3b2cd44..28d855d3d 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -786,16 +786,31 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) { ha_columnstore_select_handler* handler = NULL; - // MCOL-2178 Disable SP support in the select_handler for now. - // Check the session variable value to enable/disable use of + // Check the session variable value to enable/disable // select_handler + if (!get_select_handler(thd)) + { + return handler; + } + + // Disable SP support in the select_handler for now. + if ((thd->lex)->sphead) + { + std::string warnMsg("Select Handler doesn't support SP."); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + CS_WARNING_ID, warnMsg.c_str()); + return handler; + } + // Disable processing of select_result_interceptor classes // which intercept and transform result set rows. E.g.: // select a,b into @a1, @a2 from t1; - if (!get_select_handler(thd) || (thd->lex)->sphead || - ((thd->lex)->result && + if (((thd->lex)->result && !((select_dumpvar *)(thd->lex)->result)->var_list.is_empty())) { + std::string warnMsg("Select Handler doesn't assign variables values."); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + CS_WARNING_ID, warnMsg.c_str()); return handler; } @@ -808,7 +823,6 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } - bool unsupported_feature = false; // Select_handler use the short-cut that effectively disables // INSERT..SELECT, LDI, SELECT..INTO OUTFILE if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT @@ -816,9 +830,36 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) || (thd->lex)->exchange) { - unsupported_feature = true; + std::string warnMsg("Query inserts its result. Select Handler doesn't work."); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + CS_WARNING_ID, warnMsg.c_str()); + return handler; } + bool unsupported_feature = false; + logging::Message::Args args; + // Iterate and traverse through the item list and do not create SH + // if the unsupported (set/get_user_var) functions are present. + TABLE_LIST* table_ptr = select_lex->get_table_list(); + for (; !unsupported_feature && table_ptr; table_ptr = table_ptr->next_global) + { + List_iterator_fast it(table_ptr->select_lex->item_list); + Item* item; + while ((item = it++)) + { + item_check(item, &unsupported_feature); + if (unsupported_feature) + { + std::string warnMsg("Select Handler doesn't work b/c of the unsupported item found in item_check()"); + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + CS_WARNING_ID, warnMsg.c_str()); + return handler; + } + } + } + + // We apply dedicated rewrites from MDB here so the data structures + // becomes dirty and CS has to raise an error in case of any problem. JOIN *join= select_lex->join; // Next block tries to execute the query using SH very early to fallback // if execution fails. @@ -832,7 +873,8 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } - COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); + COND *conds = NULL; + conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add // an equi-JOIN condition. optimize_unflattened_subqueries_(select_lex); @@ -862,23 +904,6 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) restore_group_list(select_lex, group_list_ptrs); } - // Iterate and traverse through the item list and do not create SH - // if the unsupported (set/get_user_var) functions are present. - TABLE_LIST* table_ptr = select_lex->get_table_list(); - for (; !unsupported_feature && table_ptr; table_ptr = table_ptr->next_global) - { - List_iterator_fast it(table_ptr->select_lex->item_list); - Item* item; - while ((item = it++)) - { - item_check(item, &unsupported_feature); - if (unsupported_feature) - { - break; - } - } - } - if (!unsupported_feature) { handler= new ha_columnstore_select_handler(thd, select_lex); From d5ec6341360076ce1618408f276e7d5971ba43db Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 18 Feb 2020 11:42:29 -0500 Subject: [PATCH 47/84] Fixed a problem with the fix for MCOL-3769 & the default sql_mode. It appears that STRICT_TRANS_TABLE treats these warnings as if they are errors. Commenting out the warnings for now. --- dbcon/mysql/ha_mcs_pushdown.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 28d855d3d..17dd7ee56 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -796,9 +796,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) // Disable SP support in the select_handler for now. if ((thd->lex)->sphead) { - std::string warnMsg("Select Handler doesn't support SP."); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - CS_WARNING_ID, warnMsg.c_str()); + //std::string warnMsg("Select Handler doesn't support SP."); + //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + // CS_WARNING_ID, warnMsg.c_str()); return handler; } @@ -808,9 +808,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) if (((thd->lex)->result && !((select_dumpvar *)(thd->lex)->result)->var_list.is_empty())) { - std::string warnMsg("Select Handler doesn't assign variables values."); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - CS_WARNING_ID, warnMsg.c_str()); + //std::string warnMsg("Select Handler doesn't assign variables values."); + //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + // CS_WARNING_ID, warnMsg.c_str()); return handler; } @@ -830,9 +830,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) || (thd->lex)->exchange) { - std::string warnMsg("Query inserts its result. Select Handler doesn't work."); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - CS_WARNING_ID, warnMsg.c_str()); + //std::string warnMsg("Query inserts its result. Select Handler doesn't work."); + //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + // CS_WARNING_ID, warnMsg.c_str()); return handler; } @@ -850,9 +850,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) item_check(item, &unsupported_feature); if (unsupported_feature) { - std::string warnMsg("Select Handler doesn't work b/c of the unsupported item found in item_check()"); - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - CS_WARNING_ID, warnMsg.c_str()); + //std::string warnMsg("Select Handler doesn't work b/c of the unsupported item found in item_check()"); + //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + // CS_WARNING_ID, warnMsg.c_str()); return handler; } } From 295ba657246d96dce2b97400b46f27e992cac774 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 24 Feb 2020 10:11:55 -0500 Subject: [PATCH 48/84] MCOL-3829: CS not starting correctly if rebooted at specific point Found that the IDBDataFile path in BRM journal writing code needs to seek to the end of the file before writing. If save_brm is run (as it is during init), it truncates the journal file, but the workernode retained its original offset, resulting in the front of the file being 0-filled on the next journal write. Load_brm can't load that. --- versioning/BRM/slavecomm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index d2106d0dd..eef66bea4 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -2266,7 +2266,7 @@ void SlaveComm::saveDelta() { uint32_t len = delta.length(); - + journalh->seek(0, SEEK_END); journalh->write((const char*) &len, sizeof(len)); journalh->write((const char*) delta.buf(), delta.length()); journalh->flush(); From 3b47a128d4ca0db2c114ea9ad0c27ce9381dbfbd Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 24 Feb 2020 13:53:21 -0500 Subject: [PATCH 49/84] Version bumped to 1.4.3-2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 90db58b20..53b56e047 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=4 COLUMNSTORE_VERSION_PATCH=3 -COLUMNSTORE_VERSION_RELEASE=1 +COLUMNSTORE_VERSION_RELEASE=2 From afe4495d09982b38632324fd420c028e812db6d9 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Tue, 25 Feb 2020 18:26:26 +0300 Subject: [PATCH 50/84] MCOL-3810: Change from system() to fork/execl inside liboam to avoid SIGCHLD handler in procmon. --- oam/oamcpp/liboamcpp.cpp | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 1a02a9cb9..a23039f61 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -32,6 +32,8 @@ #include #include #include +#include + #elif defined (_MSC_VER) #elif defined (__FreeBSD__) #include @@ -7794,9 +7796,39 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) string command; - string pidtmp = tmpdir + "/mysql.pid"; + int no_systemd = -1; - int no_systemd = system("systemctl cat mariadb.service > /dev/null 2>&1"); + // This is here because calling system() is problematic with ProcMon + // which has its own signalHandler for SIGCHLD. Therefore since this + // is only needed when doing non MYSQL_STATUS commands, only check + // for systemctl and mariadb.service when doing other commands. + if (action != MYSQL_STATUS) + { + pid_t cPid; + int status; + + cPid = fork(); + + if (cPid == 0) + { + execlp("systemctl","systemctl","cat","mariadb.service",NULL); + } + else if (cPid > 0) + { + waitpid(cPid, &status, 0); + if (!WIFEXITED(status)) + { + writeLog("Oam::actionMysqlCalpont: systemctl Failed", LOG_TYPE_ERROR); + exceptionControl("actionMysqlCalpont", API_FAILURE); + } + no_systemd = WEXITSTATUS(status); + } + else + { + writeLog("Oam::actionMysqlCalpont: Fork Failed", LOG_TYPE_ERROR); + exceptionControl("actionMysqlCalpont", API_FAILURE); + } + } switch (action) { @@ -7876,7 +7908,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) //RUN COMMAND if (!no_systemd) { - string cmd = mysqlscript + " " + command + " mariadb.service > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + string cmd = mysqlscript + " " + command + " mariadb.service > " + tmpdir + "/" + command + ".log 2>&1"; system(cmd.c_str()); } @@ -7928,7 +7960,6 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) int state = procstat.ProcessOpState; pid_t pidStatus = procstat.ProcessID; pid_t pid = 0; - string mysqlStatus = tmpdir + "/mysql.status"; if ( state != ACTIVE ) { for (int i=0; i < 10; i++) From 04166ff1f7cb02cf3790be2059a13de19782a5d0 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Tue, 25 Feb 2020 17:13:42 -0500 Subject: [PATCH 51/84] Bumped version to 1.4.3-3 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 53b56e047..985f99c01 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=4 COLUMNSTORE_VERSION_PATCH=3 -COLUMNSTORE_VERSION_RELEASE=2 +COLUMNSTORE_VERSION_RELEASE=3 From 886160516455c4f977e565f51e2da22f55767b33 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 26 Feb 2020 18:37:35 -0600 Subject: [PATCH 52/84] add storagemanager to save files. --- oam/install_scripts/columnstore-post-install.in | 5 +++++ oam/install_scripts/columnstore-pre-uninstall.in | 1 + 2 files changed, 6 insertions(+) diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index c0384295f..5cb22c35e 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -31,6 +31,11 @@ if [ ! -f @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml ]; then exit 1 fi +if [ -f @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf.rpmsave ]; then + /bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf.new + /bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf.rpmsave @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf +fi + touch /dev/shm/columnstore-test && rm /dev/shm/columnstore-test if [ $? -ne 0 ] ; then echo "User $user will need R/W access to /dev/shm." diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 41896c937..070633a41 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -94,6 +94,7 @@ fi if [ $quiet != 1 ]; then #make copy of Columnstore.xml /bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml.rpmsave > /dev/null 2>&1 + /bin/cp -f @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf @ENGINE_SYSCONFDIR@/columnstore/storagemanager.cnf.rpmsave > /dev/null 2>&1 /bin/cp -f @MARIADB_MYCNFDIR@/columnstore.cnf @MARIADB_MYCNFDIR@/columnstore.cnf.rpmsave > /dev/null 2>&1 cp @ENGINE_SUPPORTDIR@/myCnf-include-args.text @ENGINE_SUPPORTDIR@/myCnf-include-args.text.rpmsave >& /dev/null rm -f @ENGINE_SYSCONFDIR@/columnstore/AlarmConfig.xml.installSave From 4d5460ec73cf029a00538f286728e65a71d425e4 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 26 Feb 2020 18:41:51 -0600 Subject: [PATCH 53/84] Add presinstall script for upgrading RPMs from 1.4.2 --- CMakeLists.txt | 1 + build/preInstall_platform.sh | 7 +++++++ cmake/cpackEngineRPM.cmake | 1 + 3 files changed, 9 insertions(+) create mode 100644 build/preInstall_platform.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index cab7bd282..aefcc420a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -407,6 +407,7 @@ IF (INSTALL_LAYOUT) SETA(CPACK_RPM_columnstore-platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "MariaDB-columnstore-libs" "snappy" "jemalloc" "net-tools" PARENT_SCOPE) endif() + SET(CPACK_RPM_columnstore-platform_PRE_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/build/preInstall_platform.sh PARENT_SCOPE) SET(CPACK_RPM_columnstore-platform_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/build/postInstall_platform.sh PARENT_SCOPE) SET(CPACK_RPM_columnstore-platform_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/build/preUn_platform.sh PARENT_SCOPE) diff --git a/build/preInstall_platform.sh b/build/preInstall_platform.sh new file mode 100644 index 000000000..baa626d93 --- /dev/null +++ b/build/preInstall_platform.sh @@ -0,0 +1,7 @@ + +if [ "$1" == "2" ]; then + #Perform tasks to prepare for the upgrade installation + mcsadmin shutdown y > /dev/null 2>&1 + columnstore-pre-uninstall +fi +exit 0 diff --git a/cmake/cpackEngineRPM.cmake b/cmake/cpackEngineRPM.cmake index 717f752f0..0d361718e 100644 --- a/cmake/cpackEngineRPM.cmake +++ b/cmake/cpackEngineRPM.cmake @@ -98,6 +98,7 @@ endif() SETA(CPACK_RPM_columnstore-engine_PACKAGE_REQUIRES "MariaDB-columnstore-libs") +SET(CPACK_RPM_columnstore-platform_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/preInstall_platform.sh) SET(CPACK_RPM_columnstore-platform_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/postInstall_platform.sh) SET(CPACK_RPM_columnstore-libs_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/postInstall_libs.sh) SET(CPACK_RPM_columnstore-engine_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/build/postInstall_storage_engine.sh) From e55fe458c53d98563e84067a6739938e147ca9d0 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 26 Feb 2020 19:43:44 -0600 Subject: [PATCH 54/84] Add fix for mcsadmin startSystem --- oamapps/mcsadmin/mcsadmin.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index e2970769e..c2a13ab0f 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -2781,7 +2781,7 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { string logFile = tmpDir + "/cc-restart.pdsh"; - cmd = "pdsh -a 'columnstore restart' > " + logFile + " 2>&1"; + cmd = "pdsh -a 'columnstore start' > " + logFile + " 2>&1"; system(cmd.c_str()); if (oam.checkLogStatus(logFile, "exit") ) @@ -2818,12 +2818,12 @@ int processCommand(string* arguments) if ( modulename == localModule ) { - cmd = "columnstore restart > " + tmpDir + "/start.log 2>&1"; + cmd = "columnstore start > " + tmpDir + "/startSystem.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) { - cout << endl << "error with running 'columnstore restart' on local module " << endl; + cout << endl << "error with running 'columnstore start' on local module " << endl; cout << endl << "**** startSystem Failed" << endl; break; } @@ -2836,7 +2836,7 @@ int processCommand(string* arguments) for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) { //run remote command script - cmd = "remote_command.sh " + (*pt1).IPAddr + " " + password + " 'columnstore restart' 0"; + cmd = "remote_command.sh " + (*pt1).IPAddr + " " + password + " 'columnstore start' 0"; int rtnCode = system(cmd.c_str()); if (WEXITSTATUS(rtnCode) < 0) @@ -2885,7 +2885,7 @@ int processCommand(string* arguments) //just kick off local server cout << endl << " System being started, please wait..."; cout.flush(); - cmd = "columnstore restart > " + tmpDir + "/start.log 2>&1"; + cmd = "columnstore start > " + tmpDir + "/startSystem.log 2>&1"; int rtnCode = system(cmd.c_str()); if (geteuid() == 0 && WEXITSTATUS(rtnCode) != 0) From 5bb937ed0b93f85cb7c69fdc60d0683868fb6e05 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 27 Feb 2020 00:51:35 -0600 Subject: [PATCH 55/84] this needs to happen because 1.4.2 columnstore-pre-uninstall is missing this line. --- build/preInstall_platform.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/preInstall_platform.sh b/build/preInstall_platform.sh index baa626d93..6b6ef1984 100644 --- a/build/preInstall_platform.sh +++ b/build/preInstall_platform.sh @@ -2,6 +2,7 @@ if [ "$1" == "2" ]; then #Perform tasks to prepare for the upgrade installation mcsadmin shutdown y > /dev/null 2>&1 + /bin/cp -f /etc/columnstore/storagemanager.cnf /etc/columnstore/storagemanager.cnf.rpmsave > /dev/null 2>&1 columnstore-pre-uninstall fi exit 0 From eb26fa65393fa72badc42a671a003d8395efe480 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 27 Feb 2020 16:03:09 -0500 Subject: [PATCH 56/84] Bumped version to 1.4.3-4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 985f99c01..11078936f 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=4 COLUMNSTORE_VERSION_PATCH=3 -COLUMNSTORE_VERSION_RELEASE=3 +COLUMNSTORE_VERSION_RELEASE=4 From 1e9946e23ac394cddbd6af4bc226e1ba2c5a1d69 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Wed, 18 Mar 2020 14:37:02 -0400 Subject: [PATCH 57/84] Merge pull request #1095 from mariadb-corporation/MCOL-3769_3 MCOL-3769 This commit divides create_SH() in two parts. Conflicts: dbcon/mysql/ha_mcs_pushdown.cpp --- dbcon/mysql/ha_mcs_opt_rewrites.cpp | 10 +-- dbcon/mysql/ha_mcs_opt_rewrites.h | 2 +- dbcon/mysql/ha_mcs_pushdown.cpp | 121 ++++++++++++---------------- dbcon/mysql/ha_mcs_pushdown.h | 21 ++--- 4 files changed, 68 insertions(+), 86 deletions(-) diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp index 00c08d9e0..62c4c057a 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.cpp +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -18,7 +18,7 @@ // Search simplify_joins() function in the server's code for detail COND * -simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, +simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj) { TABLE_LIST *table; @@ -26,7 +26,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, TABLE_LIST *prev_table= 0; List_iterator li(*join_list); bool straight_join= MY_TEST(join->select_options & SELECT_STRAIGHT_JOIN); - DBUG_ENTER("simplify_joins"); + DBUG_ENTER("simplify_joins_mcs"); /* Try to simplify join operations from join_list. @@ -54,7 +54,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, the outer join is converted to an inner join and the corresponding on expression is added to E. */ - expr= simplify_joins_(join, &nested_join->join_list, + expr= simplify_joins_mcs(join, &nested_join->join_list, expr, FALSE, in_sj || table->sj_on_expr); if (!table->prep_on_expr || expr != table->on_expr) @@ -67,7 +67,7 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, } nested_join->used_tables= (table_map) 0; nested_join->not_null_tables=(table_map) 0; - conds= simplify_joins_(join, &nested_join->join_list, conds, top, + conds= simplify_joins_mcs(join, &nested_join->join_list, conds, top, in_sj || table->sj_on_expr); used_tables= nested_join->used_tables; not_null_tables= nested_join->not_null_tables; @@ -241,5 +241,5 @@ simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, li.replace(repl_list); } } - DBUG_RETURN(conds); + DBUG_RETURN(conds); } diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h index d433c06a5..2b221d1c5 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.h +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -20,7 +20,7 @@ #include "idb_mysql.h" -COND *simplify_joins_(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); +COND *simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); #endif diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 17dd7ee56..95bd1ed0e 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -27,7 +27,7 @@ void disable_indices_for_CEJ(THD *thd_) TABLE_LIST* global_list; for (global_list = thd_->lex->query_tables; global_list; global_list = global_list->next_global) { - // MCOL-652 - doing this with derived tables can cause bad things to happen + // MCOL-652 - doing this with derived tables can cause bad things to happen if (!global_list->derived) { global_list->index_hints= new (thd_->mem_root) List(); @@ -42,7 +42,7 @@ void disable_indices_for_CEJ(THD *thd_) } } -bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) +bool optimize_unflattened_subqueries_mcs(SELECT_LEX *select_lex) { bool result = false; TABLE_LIST *tbl; @@ -52,7 +52,7 @@ bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) if (tbl->is_view_or_derived()) { SELECT_LEX *dsl = tbl->derived->first_select(); - result = optimize_unflattened_subqueries_(dsl); + result = optimize_unflattened_subqueries_mcs(dsl); } } @@ -546,7 +546,7 @@ create_columnstore_derived_handler(THD* thd, TABLE_LIST *derived) //To search for CROSS JOIN-s we use tree invariant //G(V,E) where [V] = [E]+1 List join_preds_list; - TABLE_LIST *tl; + TABLE_LIST *tl; for (tl = sl->get_table_list(); !unsupported_feature && tl; tl = tl->next_local) { Item_cond* where_icp= 0; @@ -796,9 +796,6 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) // Disable SP support in the select_handler for now. if ((thd->lex)->sphead) { - //std::string warnMsg("Select Handler doesn't support SP."); - //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - // CS_WARNING_ID, warnMsg.c_str()); return handler; } @@ -808,9 +805,6 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) if (((thd->lex)->result && !((select_dumpvar *)(thd->lex)->result)->var_list.is_empty())) { - //std::string warnMsg("Select Handler doesn't assign variables values."); - //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - // CS_WARNING_ID, warnMsg.c_str()); return handler; } @@ -830,14 +824,10 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) || (thd->lex)->exchange) { - //std::string warnMsg("Query inserts its result. Select Handler doesn't work."); - //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - // CS_WARNING_ID, warnMsg.c_str()); return handler; } bool unsupported_feature = false; - logging::Message::Args args; // Iterate and traverse through the item list and do not create SH // if the unsupported (set/get_user_var) functions are present. TABLE_LIST* table_ptr = select_lex->get_table_list(); @@ -850,80 +840,57 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) item_check(item, &unsupported_feature); if (unsupported_feature) { - //std::string warnMsg("Select Handler doesn't work b/c of the unsupported item found in item_check()"); - //push_warning(thd, Sql_condition::WARN_LEVEL_WARN, - // CS_WARNING_ID, warnMsg.c_str()); return handler; } } } - // We apply dedicated rewrites from MDB here so the data structures - // becomes dirty and CS has to raise an error in case of any problem. + // We apply dedicated rewrites from MDB here so MDB's data structures + // becomes dirty and CS has to raise an error in case of any problem + // or unsupported feature. + handler= new ha_columnstore_select_handler(thd, select_lex); JOIN *join= select_lex->join; - // Next block tries to execute the query using SH very early to fallback - // if execution fails. - if (!unsupported_feature) { disable_indices_for_CEJ(thd); if (select_lex->handle_derived(thd->lex, DT_MERGE)) { - // early quit b/c of the error in handle_derived - return handler; + unsupported_feature = true; + handler->err_msg.assign("create_columnstore_select_handler(): \ + Internal error occured in SL::handle_derived()"); } - COND *conds = NULL; - conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); + COND *conds = nullptr; + if (!unsupported_feature) + { + conds = simplify_joins_mcs(join, select_lex->join_list, + join->conds, TRUE, FALSE); + } // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add // an equi-JOIN condition. - optimize_unflattened_subqueries_(select_lex); - - if (conds) + if (!unsupported_feature + && optimize_unflattened_subqueries_mcs(select_lex)) + { + unsupported_feature = true; + handler->err_msg.assign("create_columnstore_select_handler(): \ + Internal error occured in optimize_unflattened_subqueries_mcs()"); + } + + if (!unsupported_feature && conds) { #ifdef DEBUG_WALK_COND conds->traverse_cond(cal_impl_if::debug_walk, NULL, Item::POSTFIX); #endif join->conds = conds; } - - // Impossible HAVING or WHERE - // TODO replace with function call - if (unsupported_feature - || select_lex->having_value == Item::COND_FALSE - || select_lex->cond_value == Item::COND_FALSE ) - { - unsupported_feature = true; - restore_optimizer_flags(thd); - } } - // Restore back the saved group_list - if (group_list_ptrs) - { - restore_group_list(select_lex, group_list_ptrs); - } + // We can't raise error now so set an error to raise it later in init_SH. + handler->rewrite_error= unsupported_feature; - if (!unsupported_feature) - { - handler= new ha_columnstore_select_handler(thd, select_lex); - mcs_handler_info mhi= mcs_handler_info(reinterpret_cast(handler), SELECT); - // handler::table is the place for the result set - int rc= 0; - // Skip execution for EXPLAIN queries - if (!thd->lex->describe) - { - rc= ha_cs_impl_pushdown_init(&mhi, handler->table); - } - - // Return SH even if init fails b/c CS changed SELECT_LEX structures - // with simplify_joins_() - if (rc) - unsupported_feature = true; - return handler; - } - - return NULL; + // Return SH even if init fails b/c CS changed SELECT_LEX structures + // with simplify_joins_mcs() + return handler; } /*********************************************************** @@ -937,7 +904,8 @@ ha_columnstore_select_handler::ha_columnstore_select_handler(THD *thd, SELECT_LEX* select_lex) : select_handler(thd, mcs_hton) { - select = select_lex; + select= select_lex; + rewrite_error= false; } /*********************************************************** @@ -961,10 +929,26 @@ int ha_columnstore_select_handler::init_scan() { DBUG_ENTER("ha_columnstore_select_handler::init_scan"); - // Dummy init for SH. Actual init happens in create_SH - // to allow fallback to other handlers if SH fails. int rc = 0; + if (!rewrite_error) + { + // handler::table is the place for the result set + // Skip execution for EXPLAIN queries + if (!thd->lex->describe) + { + mcs_handler_info mhi= mcs_handler_info( + reinterpret_cast(this), SELECT); + rc= ha_cs_impl_pushdown_init(&mhi, this->table); + } + } + else + { + my_printf_error(ER_INTERNAL_ERROR, "%s", MYF(0), err_msg.c_str()); + sql_print_error("%s", err_msg.c_str()); + rc= ER_INTERNAL_ERROR; + } + DBUG_RETURN(rc); } @@ -1005,6 +989,3 @@ int ha_columnstore_select_handler::end_scan() DBUG_RETURN(rc); } - -void ha_columnstore_select_handler::print_error(int, unsigned long) -{} diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index 268d6b5b6..e8d5d23e3 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -80,9 +80,9 @@ class ha_mcs_group_by_handler: public group_by_handler public: ha_mcs_group_by_handler(THD* thd_arg, Query* query); ~ha_mcs_group_by_handler(); - int init_scan(); - int next_row(); - int end_scan(); + int init_scan() override; + int next_row() override; + int end_scan() override; List* select; TABLE_LIST* table_list; @@ -114,9 +114,9 @@ private: public: ha_columnstore_derived_handler(THD* thd_arg, TABLE_LIST *tbl); ~ha_columnstore_derived_handler(); - int init_scan(); - int next_row(); - int end_scan(); + int init_scan() override; + int next_row() override; + int end_scan() override; void print_error(int, unsigned long); }; @@ -139,12 +139,13 @@ private: COLUMNSTORE_SHARE *share; public: + bool rewrite_error; + std::string err_msg; ha_columnstore_select_handler(THD* thd_arg, SELECT_LEX* sel); ~ha_columnstore_select_handler(); - int init_scan(); - int next_row(); - int end_scan(); - void print_error(int, unsigned long); + int init_scan() override; + int next_row() override; + int end_scan() override; }; #endif From 3d5d113a84e4504c61b69d4265b975d936ad4c81 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 26 Mar 2020 11:41:35 -0400 Subject: [PATCH 58/84] Merge pull request #1117 from drrtuy/MCOL-3828_1_4 MCOL-3828 This commit replaces the method that calls JOIN::optimise() --- dbcon/mysql/ha_mcs_impl.cpp | 98 +++++++++++++++-------------- dbcon/mysql/ha_mcs_opt_rewrites.cpp | 89 +++++++++++++++++++++++++- dbcon/mysql/ha_mcs_opt_rewrites.h | 1 + dbcon/mysql/ha_mcs_pushdown.cpp | 44 ++++--------- dbcon/mysql/ha_mcs_pushdown.h | 2 +- 5 files changed, 152 insertions(+), 82 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 3ffa33ccd..5e35c383c 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -1196,7 +1196,7 @@ vector getOnUpdateTimestampColumns(string& schema, string& tableName, in uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -2009,7 +2009,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) else { delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; ci->dmlProc = new MessageQueueClient("DMLProc"); } @@ -2069,7 +2069,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) //cout << "line 1442. received 0 byte from DMLProc and retry = "<< retry << endl; // Seems dmlProc isn't playing. Reset it and try again. delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; isTimeOut = true; //@Bug 4742 } } @@ -2117,7 +2117,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) cout << ex.what() << endl; b = 1; delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; errorMsg = ex.what(); } catch ( ... ) @@ -2125,7 +2125,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) //cout << "... exception while writing to DMLProc" << endl; b = 1; delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; errorMsg = "Unknown error caught"; } @@ -2189,7 +2189,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) errorMsg = "Lost connection to DMLProc"; b = 1; delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; } catch (...) { @@ -2247,7 +2247,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) } delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; return rc; } @@ -2375,7 +2375,7 @@ int ha_mcs_impl_rnd_init(TABLE* table) boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); idbassert(ci != 0); @@ -2581,7 +2581,7 @@ int ha_mcs_impl_rnd_init(TABLE* table) ti = ci->tableMap[table]; ti.msTablePtr = table; - if (ti.tpl_ctx == 0) + if (ti.tpl_ctx == nullptr) { ti.tpl_ctx = new sm::cpsm_tplh_t(); ti.tpl_scan_ctx = sm::sp_cpsm_tplsch_t(new sm::cpsm_tplsch_t()); @@ -2589,7 +2589,7 @@ int ha_mcs_impl_rnd_init(TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { @@ -2707,7 +2707,7 @@ int ha_mcs_impl_rnd_next(uchar* buf, TABLE* table) // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) // return HA_ERR_END_OF_FILE; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); // @bug 3078 @@ -2772,7 +2772,7 @@ int ha_mcs_impl_rnd_end(TABLE* table, bool is_pushdown_hand) { int rc = 0; THD* thd = current_thd; - cal_connection_info* ci = NULL; + cal_connection_info* ci = nullptr; if (get_fe_conn_info_ptr() != NULL) ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -2911,7 +2911,7 @@ int ha_mcs_impl_create(const char* name, TABLE* table_arg, HA_CREATE_INFO* creat { THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -2940,7 +2940,7 @@ int ha_mcs_impl_create(const char* name, TABLE* table_arg, HA_CREATE_INFO* creat int ha_mcs_impl_delete_table(const char* name) { THD* thd = current_thd; - char* dbName = NULL; + char* dbName = nullptr; if (!name) { @@ -2951,7 +2951,7 @@ int ha_mcs_impl_delete_table(const char* name) //if this is an InfiniDB tmp table ('#sql*.frm') just leave... if (!memcmp((uchar*)name, tmp_file_prefix, tmp_file_prefix_length)) return 0; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3010,7 +3010,7 @@ int ha_mcs_impl_write_row(const uchar* buf, TABLE* table) } } - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3052,7 +3052,7 @@ int ha_mcs_impl_write_row(const uchar* buf, TABLE* table) int ha_mcs_impl_update_row() { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3066,7 +3066,7 @@ int ha_mcs_impl_update_row() int ha_mcs_impl_delete_row() { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3082,7 +3082,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) { THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3295,7 +3295,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) SECURITY_ATTRIBUTES saAttr; saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); saAttr.bInheritHandle = TRUE; - saAttr.lpSecurityDescriptor = NULL; + saAttr.lpSecurityDescriptor = nullptr; HANDLE handleList[2]; const char* pSectionMsg; bSuccess = true; @@ -3315,7 +3315,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) } // Launch cpimport - LPPROC_THREAD_ATTRIBUTE_LIST lpAttributeList = NULL; + LPPROC_THREAD_ATTRIBUTE_LIST lpAttributeList = nullptr; SIZE_T attrSize = 0; STARTUPINFOEX siStartInfo; @@ -3333,7 +3333,7 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) pSectionMsg = "HeapAlloc for AttrList"; lpAttributeList = reinterpret_cast (HeapAlloc(GetProcessHeap(), 0, attrSize)); - bSuccess = lpAttributeList != NULL; + bSuccess = lpAttributeList != nullptr; } if (bSuccess) @@ -3368,8 +3368,8 @@ void ha_mcs_impl_start_bulk_insert(ha_rows rows, TABLE* table) memset(&siStartInfo, 0, sizeof(STARTUPINFOEX)); siStartInfo.StartupInfo.cb = sizeof(STARTUPINFOEX); siStartInfo.lpAttributeList = lpAttributeList; - siStartInfo.StartupInfo.hStdError = NULL; - siStartInfo.StartupInfo.hStdOutput = NULL; + siStartInfo.StartupInfo.hStdError = nullptr; + siStartInfo.StartupInfo.hStdOutput = nullptr; siStartInfo.StartupInfo.hStdInput = ci->cpimport_stdin_Rd; siStartInfo.StartupInfo.dwFlags |= STARTF_USESTDHANDLES; // Create the child process. @@ -3598,7 +3598,7 @@ int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table) std::string aTmpDir(startup::StartUp::tmpDir()); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3826,7 +3826,7 @@ int ha_mcs_impl_end_bulk_insert(bool abort, TABLE* table) int ha_mcs_impl_commit (handlerton* hton, THD* thd, bool all) { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3858,7 +3858,7 @@ int ha_mcs_impl_commit (handlerton* hton, THD* thd, bool all) int ha_mcs_impl_rollback (handlerton* hton, THD* thd, bool all) { - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3892,7 +3892,7 @@ int ha_mcs_impl_close_connection (handlerton* hton, THD* thd) // from vtable(lock_type = 2) // An ugly way. I will use ha_data w/o external_lock. // This in MCOL-2178 - cal_connection_info* ci = NULL; + cal_connection_info* ci = nullptr; if(thd_get_ha_data(thd, hton) != (void*)0x42) // 0x42 is the magic CS sets when setup hton { ci = reinterpret_cast(thd_get_ha_data(thd, hton)); @@ -3906,7 +3906,7 @@ int ha_mcs_impl_close_connection (handlerton* hton, THD* thd) { rc = ha_mcs_impl_close_connection_(hton, thd, *ci); delete ci->dmlProc; - ci->dmlProc = NULL; + ci->dmlProc = nullptr; } if (ci->cal_conn_hndl) @@ -3915,7 +3915,7 @@ int ha_mcs_impl_close_connection (handlerton* hton, THD* thd) ci->cal_conn_hndl = 0; } - thd_set_ha_data(thd, hton, NULL); + thd_set_ha_data(thd, hton, nullptr); return rc; } @@ -3924,7 +3924,7 @@ int ha_mcs_impl_rename_table(const char* from, const char* to) { IDEBUG( cout << "ha_mcs_impl_rename_table: " << from << " => " << to << endl ); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -3967,7 +3967,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table) alias.assign(table->alias.ptr(), table->alias.length()); IDEBUG( cout << "ha_mcs_impl_cond_push: " << alias << endl ); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4006,7 +4006,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table) if (ti.condInfo) { delete ti.condInfo; - ti.condInfo = 0; + ti.condInfo = nullptr; ci->tableMap[table] = ti; } @@ -4019,7 +4019,7 @@ COND* ha_mcs_impl_cond_push(COND* cond, TABLE* table) } else { - return NULL; + return nullptr; } } @@ -4042,7 +4042,7 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) alias.assign(table->alias.ptr(), table->alias.length()); IDEBUG( cout << "external_lock for " << alias << endl ); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4059,7 +4059,9 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) CalTableMap::iterator mapiter = ci->tableMap.find(table); // make sure this is a release lock (2nd) call called in // the table mode. - if (mapiter != ci->tableMap.end() && mapiter->second.csep && lock_type == 2) + if (mapiter != ci->tableMap.end() + && (mapiter->second.condInfo || mapiter->second.csep) + && lock_type == 2) { // CS ends up processing query with handlers // table mode @@ -4072,13 +4074,13 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) ci->extendedStats = mapiter->second.conn_hndl->extendedStats; ci->miniStats = mapiter->second.conn_hndl->miniStats; sm::sm_cleanup(mapiter->second.conn_hndl); - mapiter->second.conn_hndl = 0; + mapiter->second.conn_hndl = nullptr; } if (mapiter->second.condInfo) { delete mapiter->second.condInfo; - mapiter->second.condInfo = 0; + mapiter->second.condInfo = nullptr; } // MCOL-2178 Check for tableMap size to set this only once. @@ -4129,7 +4131,7 @@ int ha_mcs_impl_external_lock(THD* thd, TABLE* table, int lock_type) if (tme.second.condInfo) { delete tme.second.condInfo; - tme.second.condInfo= 0; + tme.second.condInfo= nullptr; } } ci->tableMap.clear(); @@ -4197,7 +4199,7 @@ int ha_mcs_impl_group_by_init(mcs_handler_info *handler_info, TABLE* table) boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); csc->identity(CalpontSystemCatalog::FE); - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4516,7 +4518,7 @@ int ha_mcs_impl_group_by_init(mcs_handler_info *handler_info, TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { @@ -4618,7 +4620,7 @@ int ha_mcs_impl_group_by_next(TABLE* table) // if (MIGR::infinidb_vtable.impossibleWhereOnUnion) // return HA_ERR_END_OF_FILE; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -4697,7 +4699,7 @@ int ha_mcs_impl_group_by_end(TABLE* table) { int rc = 0; THD* thd = current_thd; - cal_connection_info* ci = NULL; + cal_connection_info* ci = nullptr; if (thd->slave_thread && !get_replication_slave(thd) && ( thd->lex->sql_command == SQLCOM_INSERT || @@ -4959,8 +4961,8 @@ int ha_cs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) sm::cpsm_conhdl_t* hndl; SCSEP csep; // Declare handlers ptrs in this scope for future use. - select_handler* sh = NULL; - derived_handler* dh = NULL; + select_handler* sh = nullptr; + derived_handler* dh = nullptr; // update traceFlags according to the autoswitch state. ci->traceFlags = (ci->traceFlags | CalpontSelectExecutionPlan::TRACE_TUPLE_OFF)^ @@ -5234,7 +5236,7 @@ int ha_cs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { @@ -5312,7 +5314,7 @@ int ha_cs_impl_select_next(uchar* buf, TABLE* table) int rc = HA_ERR_END_OF_FILE; THD* thd = current_thd; - if (get_fe_conn_info_ptr() == NULL) + if (get_fe_conn_info_ptr() == nullptr) set_fe_conn_info_ptr((void*)new cal_connection_info()); cal_connection_info* ci = reinterpret_cast(get_fe_conn_info_ptr()); @@ -5363,7 +5365,7 @@ int ha_cs_impl_select_next(uchar* buf, TABLE* table) // make sure rowgroup is null so the new meta data can be taken. This is for some case mysql // call rnd_init for a table more than once. - ti.tpl_scan_ctx->rowGroup = NULL; + ti.tpl_scan_ctx->rowGroup = nullptr; try { diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.cpp b/dbcon/mysql/ha_mcs_opt_rewrites.cpp index 62c4c057a..a29366fed 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.cpp +++ b/dbcon/mysql/ha_mcs_opt_rewrites.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 MariaDB Corporation +/* Copyright (C) 2019-20 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -14,6 +14,9 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ + +#include + #include "ha_mcs_opt_rewrites.h" // Search simplify_joins() function in the server's code for detail @@ -243,3 +246,87 @@ simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool to } DBUG_RETURN(conds); } + +/*@brief in_subselect_rewrite_walk - Rewrites Item_in_subselect*/ +/************************************************************ +* DESCRIPTION: +* It traverses filter predicates searching for +* Item_in_subselect and rewrites it adding equi-join predicate +* to finalise IN_2_EXISTS rewrite. +* PARAMETERS: +* item_arg - Item to check. +* arg - bool to early return if predicate injection fails. +* RETURN: +***********************************************************/ +void in_subselect_rewrite_walk(const Item* item_arg, void* arg) +{ + bool* result= reinterpret_cast(arg); + if (*result) return; + + Item* item= const_cast(item_arg); + + JOIN* join= nullptr; + if (typeid(*item) == typeid(Item_in_subselect)) + { + Item_in_subselect* sub= reinterpret_cast(item); + // MCS 1.4.3 doesn't support IN + subquery with UNION so + // we safe to take this JOIN. + join= sub->unit->first_select()->join; + // Inject equi-JOIN predicates if needed. + *result= sub->create_in_to_exists_cond(join); + *result= (*result) ? *result : + sub->inject_in_to_exists_cond(join); + } + else if (typeid(*item) == typeid(Item_singlerow_subselect)) + { + Item_singlerow_subselect* sub= + reinterpret_cast(item); + // MCS 1.4.3 doesn't support IN + subquery with UNION so + // we safe to take this JOIN. + join= sub->unit->first_select()->join; + } + else + { + // Exit for any but dedicated Items. + return; + } + + // Walk recursively to process nested IN ops. + if (join->conds) + { + join->conds->traverse_cond(in_subselect_rewrite_walk, + arg, Item::POSTFIX); + } +} + +/*@brief in_subselect_rewrite - Rewrites Item_in_subselect*/ +/************************************************************ +* DESCRIPTION: +* It traverses TABLE_LISTs running in_subselect_rewrite_walk +* PARAMETERS: +* select_lex +* RETURN: +* bool to to indicate predicate injection failures. +***********************************************************/ +bool in_subselect_rewrite(SELECT_LEX *select_lex) +{ + bool result = false; + TABLE_LIST *tbl; + List_iterator_fast li(select_lex->leaf_tables); + while (!result && (tbl= li++)) + { + if (tbl->is_view_or_derived()) + { + SELECT_LEX *dsl = tbl->derived->first_select(); + result = in_subselect_rewrite(dsl); + } + } + + if (select_lex->join && select_lex->join->conds) + { + select_lex->join->conds->traverse_cond(in_subselect_rewrite_walk, &result, + Item::POSTFIX); + } + + return result; +} diff --git a/dbcon/mysql/ha_mcs_opt_rewrites.h b/dbcon/mysql/ha_mcs_opt_rewrites.h index 2b221d1c5..3cf985b23 100644 --- a/dbcon/mysql/ha_mcs_opt_rewrites.h +++ b/dbcon/mysql/ha_mcs_opt_rewrites.h @@ -21,6 +21,7 @@ #include "idb_mysql.h" COND *simplify_joins_mcs(JOIN *join, List *join_list, COND *conds, bool top, bool in_sj); +bool in_subselect_rewrite(SELECT_LEX *select_lex); #endif diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 95bd1ed0e..3b0764a66 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -42,26 +42,6 @@ void disable_indices_for_CEJ(THD *thd_) } } -bool optimize_unflattened_subqueries_mcs(SELECT_LEX *select_lex) -{ - bool result = false; - TABLE_LIST *tbl; - List_iterator_fast li(select_lex->leaf_tables); - while (!result && (tbl= li++)) - { - if (tbl->is_view_or_derived()) - { - SELECT_LEX *dsl = tbl->derived->first_select(); - result = optimize_unflattened_subqueries_mcs(dsl); - } - } - - result = (!result) ? - select_lex->optimize_unflattened_subqueries(false) : true; - - return result; -} - void mutate_optimizer_flags(THD *thd_) { // MCOL-2178 Disable all optimizer flags as it was in the fork. @@ -272,7 +252,6 @@ void save_join_predicates(const Item* item, void* arg) } } - /*@brief check_walk - It traverses filter conditions */ /************************************************************ * DESCRIPTION: @@ -863,18 +842,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) COND *conds = nullptr; if (!unsupported_feature) { - conds = simplify_joins_mcs(join, select_lex->join_list, + conds= simplify_joins_mcs(join, select_lex->join_list, join->conds, TRUE, FALSE); } - // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add - // an equi-JOIN condition. - if (!unsupported_feature - && optimize_unflattened_subqueries_mcs(select_lex)) - { - unsupported_feature = true; - handler->err_msg.assign("create_columnstore_select_handler(): \ - Internal error occured in optimize_unflattened_subqueries_mcs()"); - } if (!unsupported_feature && conds) { @@ -883,9 +853,19 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) #endif join->conds = conds; } + + // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add + // an equi-JOIN condition. + if (!unsupported_feature && in_subselect_rewrite(select_lex)) + { + unsupported_feature = true; + handler->err_msg.assign("create_columnstore_select_handler(): \ + Internal error occured in in_subselect_rewrite()"); + } + } - // We can't raise error now so set an error to raise it later in init_SH. + // We shouldn't raise error now so set an error to raise it later in init_SH. handler->rewrite_error= unsupported_feature; // Return SH even if init fails b/c CS changed SELECT_LEX structures diff --git a/dbcon/mysql/ha_mcs_pushdown.h b/dbcon/mysql/ha_mcs_pushdown.h index e8d5d23e3..4492f0470 100644 --- a/dbcon/mysql/ha_mcs_pushdown.h +++ b/dbcon/mysql/ha_mcs_pushdown.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019 MariaDB + Copyright (c) 2019-20 MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by From 6d9544a15e0d5e2e0cea1c66ce3212e79c54e25c Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 30 Mar 2020 17:59:11 -0400 Subject: [PATCH 59/84] Merge pull request #1126 from drrtuy/MCOL-3903_1_4 MCOL-3903 Enable Select Handler to run query part of INSERT..SELECT. Conflicts: dbcon/mysql/ha_mcs_pushdown.cpp --- dbcon/mysql/ha_exists_sub.cpp | 2 +- dbcon/mysql/ha_from_sub.cpp | 5 +- dbcon/mysql/ha_in_sub.cpp | 2 +- dbcon/mysql/ha_mcs_execplan.cpp | 493 +++++++++++------------------- dbcon/mysql/ha_mcs_impl.cpp | 2 +- dbcon/mysql/ha_mcs_impl_if.h | 8 +- dbcon/mysql/ha_mcs_pushdown.cpp | 16 +- dbcon/mysql/ha_scalar_sub.cpp | 2 +- dbcon/mysql/ha_select_sub.cpp | 2 +- dbcon/mysql/ha_subquery.h | 2 +- utils/loggingcpp/ErrorMessage.txt | 2 +- 11 files changed, 201 insertions(+), 335 deletions(-) diff --git a/dbcon/mysql/ha_exists_sub.cpp b/dbcon/mysql/ha_exists_sub.cpp index 9db9294c9..4c6ebd424 100644 --- a/dbcon/mysql/ha_exists_sub.cpp +++ b/dbcon/mysql/ha_exists_sub.cpp @@ -112,7 +112,7 @@ execplan::ParseTree* ExistsSub::transform() return NULL; } - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_from_sub.cpp b/dbcon/mysql/ha_from_sub.cpp index 12b2b24de..d34bc8c2e 100644 --- a/dbcon/mysql/ha_from_sub.cpp +++ b/dbcon/mysql/ha_from_sub.cpp @@ -323,8 +323,7 @@ FromSubQuery::FromSubQuery(gp_walk_info& gwip) : SubQuery(gwip) {} FromSubQuery::FromSubQuery(gp_walk_info& gwip, - SELECT_LEX* sub, - bool isPushdownHandler) : + SELECT_LEX* sub) : SubQuery(gwip), fFromSub(sub) {} @@ -348,7 +347,7 @@ SCSEP FromSubQuery::transform() csep->derivedTbAlias(fAlias); // always lower case csep->derivedTbView(fGwip.viewName.alias); - if (getSelectPlan(gwi, *fFromSub, csep, false, true) != 0) + if (getSelectPlan(gwi, *fFromSub, csep, false) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_in_sub.cpp b/dbcon/mysql/ha_in_sub.cpp index 4d1e828ce..5ae5e18f2 100644 --- a/dbcon/mysql/ha_in_sub.cpp +++ b/dbcon/mysql/ha_in_sub.cpp @@ -148,7 +148,7 @@ execplan::ParseTree* InSub::transform() gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) { fGwip.fatalParseError = true; diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 1d11be0eb..da10645f0 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3117,7 +3117,10 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) return ct; } -ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand, bool isRefItem) +ReturnedColumn* buildReturnedColumn( + Item* item, gp_walk_info& gwi, + bool& nonSupport, + bool isRefItem) { ReturnedColumn* rc = NULL; @@ -3275,9 +3278,9 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp } if (func_name == "+" || func_name == "-" || func_name == "*" || func_name == "/" ) - return buildArithmeticColumn(ifp, gwi, nonSupport, pushdownHand); + return buildArithmeticColumn(ifp, gwi, nonSupport); else - return buildFunctionColumn(ifp, gwi, nonSupport, pushdownHand); + return buildFunctionColumn(ifp, gwi, nonSupport); } case Item::SUM_FUNC_ITEM: @@ -3410,8 +3413,7 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp ArithmeticColumn* buildArithmeticColumn( Item_func* item, gp_walk_info& gwi, - bool& nonSupport, - bool pushdownHand) + bool& nonSupport) { if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -3435,7 +3437,7 @@ ArithmeticColumn* buildArithmeticColumn( { if (gwi.clauseType == SELECT || /*gwi.clauseType == HAVING || */gwi.clauseType == GROUP_BY || gwi.clauseType == FROM) // select list { - lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport, pushdownHand)); + lhs = new ParseTree(buildReturnedColumn(sfitempp[0], gwi, nonSupport)); if (!lhs->data() && (sfitempp[0]->type() == Item::FUNC_ITEM)) { @@ -3443,7 +3445,7 @@ ArithmeticColumn* buildArithmeticColumn( Item_func* ifp = (Item_func*)sfitempp[0]; lhs = buildParseTree(ifp, gwi, nonSupport); } - else if(pushdownHand && !lhs->data() && (sfitempp[0]->type() == Item::REF_ITEM)) + else if(!lhs->data() && (sfitempp[0]->type() == Item::REF_ITEM)) { // There must be an aggregation column in extended SELECT // list so find the corresponding column. @@ -3454,7 +3456,7 @@ ArithmeticColumn* buildArithmeticColumn( if(rc) lhs = new ParseTree(rc); } - rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport, pushdownHand)); + rhs = new ParseTree(buildReturnedColumn(sfitempp[1], gwi, nonSupport)); if (!rhs->data() && (sfitempp[1]->type() == Item::FUNC_ITEM)) { @@ -3462,7 +3464,7 @@ ArithmeticColumn* buildArithmeticColumn( Item_func* ifp = (Item_func*)sfitempp[1]; rhs = buildParseTree(ifp, gwi, nonSupport); } - else if(pushdownHand && !rhs->data() && (sfitempp[1]->type() == Item::REF_ITEM)) + else if(!rhs->data() && (sfitempp[1]->type() == Item::REF_ITEM)) { // There must be an aggregation column in extended SELECT // list so find the corresponding column. @@ -3638,7 +3640,6 @@ ReturnedColumn* buildFunctionColumn( Item_func* ifp, gp_walk_info& gwi, bool& nonSupport, - bool pushdownHand, bool selectBetweenIn) { if (get_fe_conn_info_ptr() == NULL) @@ -3680,7 +3681,7 @@ ReturnedColumn* buildFunctionColumn( // Arithmetic exp if (funcName == "+" || funcName == "-" || funcName == "*" || funcName == "/" ) { - ArithmeticColumn* ac = buildArithmeticColumn(ifp, gwi, nonSupport, pushdownHand); + ArithmeticColumn* ac = buildArithmeticColumn(ifp, gwi, nonSupport); return ac; } @@ -3834,11 +3835,10 @@ ReturnedColumn* buildFunctionColumn( return NULL; } - ReturnedColumn* rc = buildReturnedColumn(ifp->arguments()[i], gwi, nonSupport, pushdownHand); + ReturnedColumn* rc = buildReturnedColumn(ifp->arguments()[i], gwi, nonSupport); // MCOL-1510 It must be a temp table field, so find the corresponding column. - if (!rc && pushdownHand - && ifp->arguments()[i]->type() == Item::REF_ITEM) + if (!rc && ifp->arguments()[i]->type() == Item::REF_ITEM) { gwi.fatalParseError = false; rc = buildAggFrmTempField(ifp->arguments()[i], gwi); @@ -5652,7 +5652,7 @@ void gp_walk(const Item* item, void* arg) if (col->type() != Item::COND_ITEM) { - rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, false, true); + rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, false); if ( col->type() == Item::FIELD_ITEM ) gwip->fatalParseError = false; @@ -6191,7 +6191,7 @@ int processFrom(bool &isUnion, if (table_ptr->derived) { SELECT_LEX* select_cursor = table_ptr->derived->first_select(); - FromSubQuery fromSub(gwi, select_cursor, true); + FromSubQuery fromSub(gwi, select_cursor); string alias(table_ptr->alias.str); fromSub.alias(lower(alias)); @@ -6303,7 +6303,7 @@ int processFrom(bool &isUnion, union_gwi.thd = gwi.thd; uint32_t err = 0; - if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel, true)) != 0) + if ((err = getSelectPlan(union_gwi, *sl, plan, unionSel)) != 0) return err; unionVec.push_back(SCEP(plan)); @@ -6543,25 +6543,176 @@ int processWhere(SELECT_LEX &select_lex, return 0; } +/*@brief Process LIMIT part of a query or sub-query */ +/*********************************************************** + * DESCRIPTION: + * Processes LIMIT and OFFSET parts + * RETURNS + * error id as an int + ***********************************************************/ +int processLimitAndOffset( + SELECT_LEX& select_lex, + gp_walk_info& gwi, + SCSEP& csep, + bool unionSel, + bool isUnion, + bool isSelectHandlerTop +) +{ + // LIMIT processing part + uint64_t limitNum = std::numeric_limits::max(); + + // non-MAIN union branch + if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + { + /* Consider the following query: + "select a from t1 where exists (select b from t2 where a=b);" + CS first builds a hash table for t2, then pushes down the hash to + PrimProc for a distributed hash join execution, with t1 being the + large-side table. However, the server applies an optimization in + Item_exists_subselect::fix_length_and_dec in sql/item_subselect.cc + (see server commit ae476868a5394041a00e75a29c7d45917e8dfae8) + where it sets explicit_limit to true, which causes csep->limitNum set to 1. + This causes the hash table for t2 to only contain a single record for the + hash join, giving less number of rows in the output result set than expected. + We therefore do not allow limit set to 1 here for such queries. + */ + if (gwi.subSelectType != CalpontSelectExecutionPlan::IN_SUBS + && gwi.subSelectType != CalpontSelectExecutionPlan::EXISTS_SUBS + && select_lex.master_unit()->global_parameters()->explicit_limit) + { + if (select_lex.master_unit()->global_parameters()->offset_limit) + { + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + csep->limitStart(offset->val_int()); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + csep->limitNum(select->val_int()); + // MCOL-894 Activate parallel ORDER BY + csep->orderByThreads(get_orderby_threads(gwi.thd)); + } + } + } + // union with explicit select at the top level + else if (isUnion && select_lex.explicit_limit) + { + if (select_lex.braces) + { + if (select_lex.offset_limit) + csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); + + if (select_lex.select_limit) + csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); + } + } + // other types of queries that have explicit LIMIT + else if (select_lex.explicit_limit) + { + uint32_t limitOffset = 0; + + if (select_lex.join) + { + JOIN* join = select_lex.join; +#if MYSQL_VERSION_ID >= 50172 + + // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer + // (not null though) and will cause seg fault. Prefer checking + // select_lex->offset_limit if not null. + if (join->select_lex && + join->select_lex->offset_limit && + join->select_lex->offset_limit->is_fixed() && + join->select_lex->select_limit && + join->select_lex->select_limit->is_fixed()) + { + limitOffset = join->select_lex->offset_limit->val_int(); + limitNum = join->select_lex->select_limit->val_int(); + } + else if (join->unit) + { + limitOffset = join->unit->offset_limit_cnt; + limitNum = join->unit->select_limit_cnt - limitOffset; + } + +#else + limitOffset = (join->unit)->offset_limit_cnt; + limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; +#endif + } + else + { + if (select_lex.master_unit()->global_parameters()->offset_limit) + { + Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; + limitOffset = offset->val_int(); + } + + if (select_lex.master_unit()->global_parameters()->select_limit) + { + Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; + limitNum = select->val_int(); + } + } + + csep->limitStart(limitOffset); + csep->limitNum(limitNum); + } + // If an explicit limit is not specified, use the system variable value + else + { + csep->limitNum(gwi.thd->variables.select_limit); + } + + // We don't currently support limit with correlated subquery + if (gwi.subQuery && !gwi.correlatedTbNameVec.empty() && csep->hasOrderBy()) + { + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); + setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); + return ER_CHECK_NOT_IMPLEMENTED; + } + + // MDB applies OFFSET on its own for SH processing. + // See related MDEV-16327 for details. + if (isSelectHandlerTop && csep->limitStart()) + { + if (std::numeric_limits::max()-csep->limitStart() + < csep->limitNum()) + { + csep->limitNum(std::numeric_limits::max()); + csep->limitStart(0); + } + else + { + csep->limitNum(csep->limitNum()+csep->limitStart()); + csep->limitStart(0); + } + } + + return 0; +} + /*@brief Translates SELECT_LEX into CSEP */ /*********************************************************** * DESCRIPTION: * This function takes SELECT_LEX and tries to produce * a corresponding CSEP out of it. It is made of parts that * process parts of the query, e.g. FROM, WHERE, SELECT, - * HAVING, GROUP BY, ORDER BY. FROM and WHERE are processed - * by processFrom(), processWhere(). CS calls getSelectPlan() + * HAVING, GROUP BY, ORDER BY. FROM, WHERE, LIMIT are processed + * by corresponding methods. CS calls getSelectPlan() * recursively to process subqueries. * ARGS * isUnion if true CS processes UNION unit now - * isPushdownHand legacy to be removed + * isSelectHandlerTop removes offset at the top of SH query. * RETURNS * error id as an int ***********************************************************/ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, bool isUnion, - bool isPushdownHand) + bool isSelectHandlerTop) { #ifdef DEBUG_WALK_COND cerr << "getSelectPlan()" << endl; @@ -6769,7 +6920,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, ReturnedColumn* rc; if (funcName == "in" || funcName == " IN " || funcName == "between") { - rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, false, true); + rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, false); } else { @@ -6801,15 +6952,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, parse_item(ifp, funcFieldVec, hasNonSupportItem, parseInfo, &gwi); uint32_t after_size = funcFieldVec.size(); - // group by func and func in subquery can not be post processed // pushdown handler projection functions // @bug3881. set_user_var can not be treated as constant function // @bug5716. Try to avoid post process function for union query. - if ((gwi.subQuery || select_lex.group_list.elements != 0 || - !csep->unionVec().empty() || isUnion || isPushdownHand ) && - !hasNonSupportItem && (after_size - before_size) == 0 && - !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT) - ) + if (!hasNonSupportItem && (after_size - before_size) == 0 && + !(parseInfo & AGG_BIT) && !(parseInfo & SUB_BIT)) { String val, *str = ifp->val_str(&val); string valStr; @@ -7490,6 +7637,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, } } + // ORDER BY processing { SQL_I_List order_list = select_lex.order_list; ORDER* ordercol = reinterpret_cast(order_list.first); @@ -7503,9 +7651,9 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // MCOL-2166 Looking for this sorting item in GROUP_BY items list. // Shouldn't look into this if query doesn't have GROUP BY or // aggregations - if(isPushdownHand - && select_lex.agg_func_used() && select_lex.group_list.first - && !sortItemIsInGrouping(*ordercol->item, select_lex.group_list.first)) + if(select_lex.agg_func_used() && select_lex.group_list.first + && !sortItemIsInGrouping(*ordercol->item, + select_lex.group_list.first)) { std::ostringstream ostream; std::ostringstream& osr = ostream; @@ -7522,10 +7670,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, // re-visit the first of ordercol list ordercol = reinterpret_cast(order_list.first); - // for subquery or pushdown query, order+limit by will be supported in CS - // union order by and limit are supported - if (gwi.hasWindowFunc || isPushdownHand || ( isUnion && ordercol ) - || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT ) { for (; ordercol; ordercol = ordercol->next) { @@ -7601,158 +7745,6 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, gwi.orderByCols.push_back(SRCP(rc)); } } - // DRRTUY The whole block is marked for removal in 1.4.1 -#if 0 - else if (!isUnion) - { - vector fieldVec; - - // the following order by is just for redo phase - if (!unionSel) - { - for (; ordercol; ordercol = ordercol->next) - { - Item* ord_item = *(ordercol->item); - - // @bug5993. Could be nested ref. - while (ord_item->type() == Item::REF_ITEM) - ord_item = (*((Item_ref*)ord_item)->ref); - - //ReturnedColumn* rc = 0; - // check if this order by column is on the select list - //Item_func* ifp = (Item_func*)(*(ordercol->item)); - //rc = buildFunctionColumn(ifp, gwi, gwi.fatalParseError); - - if (ord_item->type() == Item::FUNC_ITEM) - { - //FunctionColumn* fc = dynamic_cast(rc); - } - else if (ord_item->type() == Item::SUBSELECT_ITEM) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - else if (ord_item->type() == Item::SUM_FUNC_ITEM) - { - ReturnedColumn* ac = 0; - - Item_sum* ifp = (Item_sum*)(*(ordercol->item)); - // @bug3477. add aggregate column to the select list of the create phase. - ac = buildAggregateColumn(ifp, gwi); - - if (!ac) - { - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, - IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY), gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - // check if this order by column is on the select list - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - AggregateColumn* ret = dynamic_cast(gwi.returnedCols[i].get()); - - if (!ret) - continue; - - } - - if (ac || !gwi.groupByCols.empty()) - { - SRCP srcp(ac); - gwi.returnedCols.push_back(srcp); - - continue; - } - } - else if (ord_item->name.length && ord_item->type() == Item::FIELD_ITEM) - { - Item_field* field = reinterpret_cast(ord_item); - ReturnedColumn* rc = buildSimpleColumn(field, gwi); - - for (uint32_t i = 0; i < gwi.returnedCols.size(); i++) - { - SimpleColumn* sc = dynamic_cast(gwi.returnedCols[i].get()); - - if (sc && ((Item_field*)ord_item)->cached_table && - (strcasecmp(getViewName(((Item_field*)ord_item)->cached_table).c_str(), sc->viewName().c_str()) != 0)) - { - continue; - } - - if (sc && sc->sameColumn(rc)) - { - break; - } - } - } - - { - // @bug 2719. Error out order by not on the distinct select list. - if (select_lex.options & SELECT_DISTINCT) - { - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_ORDERBY_NOT_IN_DISTINCT); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - bool hasNonSupportItem = false; - uint16_t parseInfo = 0; - parse_item(ord_item, fieldVec, hasNonSupportItem, parseInfo, &gwi); - - if (hasNonSupportItem) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - } - } - } - - // populate string to be added to the select list for order by - for (uint32_t i = 0; i < fieldVec.size(); i++) - { - SimpleColumn* sc = buildSimpleColumn(fieldVec[i], gwi); - - if (!sc) - { - string emsg = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_ORDER_BY); - setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, emsg, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - - String str; - fieldVec[i]->print(&str, QT_ORDINARY); - sc->alias(string(str.c_ptr())); - SRCP srcp(sc); - uint32_t j = 0; - - for (; j < gwi.returnedCols.size(); j++) - { - if (sc->sameColumn(gwi.returnedCols[j].get())) - { - SimpleColumn* field = dynamic_cast(gwi.returnedCols[j].get()); - - if (field && field->alias() == sc->alias()) - break; - } - } - - if (j == gwi.returnedCols.size()) - { - gwi.returnedCols.push_back(srcp); - gwi.columnMap.insert(CalpontSelectExecutionPlan::ColumnMap::value_type(string(fieldVec[i]->field_name.str), srcp)); - TABLE_LIST* tmp = (fieldVec[i]->cached_table ? fieldVec[i]->cached_table : 0); - gwi.tableMap[make_aliastable(sc->schemaName(), sc->tableName(), sc->tableAlias(), sc->isColumnStore())] = - make_pair(1, tmp); - } - } - } -#endif - // make sure columnmap, returnedcols and count(*) arg_list are not empty TableMap::iterator tb_iter = gwi.tableMap.begin(); @@ -7878,129 +7870,16 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { csep->hasOrderBy(true); // To activate LimitedOrderBy - if(isPushdownHand) - { - csep->orderByThreads(get_orderby_threads(gwi.thd)); - csep->specHandlerProcessed(true); - } + csep->orderByThreads(get_orderby_threads(gwi.thd)); + csep->specHandlerProcessed(true); } } - // LIMIT processing part - uint64_t limitNum = std::numeric_limits::max(); - - // non-MAIN union branch - if (unionSel || gwi.subSelectType != CalpontSelectExecutionPlan::MAIN_SELECT) + if ((rc = processLimitAndOffset(select_lex, gwi, csep, unionSel, isUnion, isSelectHandlerTop))) { - /* Consider the following query: - "select a from t1 where exists (select b from t2 where a=b);" - CS first builds a hash table for t2, then pushes down the hash to - PrimProc for a distributed hash join execution, with t1 being the - large-side table. However, the server applies an optimization in - Item_exists_subselect::fix_length_and_dec in sql/item_subselect.cc - (see server commit ae476868a5394041a00e75a29c7d45917e8dfae8) - where it sets explicit_limit to true, which causes csep->limitNum set to 1. - This causes the hash table for t2 to only contain a single record for the - hash join, giving less number of rows in the output result set than expected. - We therefore do not allow limit set to 1 here for such queries. - */ - if (gwi.subSelectType != CalpontSelectExecutionPlan::IN_SUBS - && gwi.subSelectType != CalpontSelectExecutionPlan::EXISTS_SUBS - && select_lex.master_unit()->global_parameters()->explicit_limit) - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - csep->limitStart(offset->val_int()); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - csep->limitNum(select->val_int()); - // MCOL-894 Activate parallel ORDER BY - csep->orderByThreads(get_orderby_threads(gwi.thd)); - } - } + return rc; } - // union with explicit select at the top level - else if (isUnion && select_lex.explicit_limit) - { - if (select_lex.braces) - { - if (select_lex.offset_limit) - csep->limitStart(((Item_int*)select_lex.offset_limit)->val_int()); - - if (select_lex.select_limit) - csep->limitNum(((Item_int*)select_lex.select_limit)->val_int()); - } - } - // other types of queries that have explicit LIMIT - else if (select_lex.explicit_limit) - { - uint32_t limitOffset = 0; - - if (select_lex.join) - { - JOIN* join = select_lex.join; -#if MYSQL_VERSION_ID >= 50172 - - // @bug5729. After upgrade, join->unit sometimes is uninitialized pointer - // (not null though) and will cause seg fault. Prefer checking - // select_lex->offset_limit if not null. - if (join->select_lex && - join->select_lex->offset_limit && - join->select_lex->offset_limit->is_fixed() && - join->select_lex->select_limit && - join->select_lex->select_limit->is_fixed()) - { - limitOffset = join->select_lex->offset_limit->val_int(); - limitNum = join->select_lex->select_limit->val_int(); - } - else if (join->unit) - { - limitOffset = join->unit->offset_limit_cnt; - limitNum = join->unit->select_limit_cnt - limitOffset; - } - -#else - limitOffset = (join->unit)->offset_limit_cnt; - limitNum = (join->unit)->select_limit_cnt - (join->unit)->offset_limit_cnt; -#endif - } - else - { - if (select_lex.master_unit()->global_parameters()->offset_limit) - { - Item_int* offset = (Item_int*)select_lex.master_unit()->global_parameters()->offset_limit; - limitOffset = offset->val_int(); - } - - if (select_lex.master_unit()->global_parameters()->select_limit) - { - Item_int* select = (Item_int*)select_lex.master_unit()->global_parameters()->select_limit; - limitNum = select->val_int(); - } - } - - csep->limitStart(limitOffset); - csep->limitNum(limitNum); - } - // If an explicit limit is not specified, use the system variable value - else - { - csep->limitNum(gwi.thd->variables.select_limit); - } - - // We don't currently support limit with correlated subquery - if (gwi.subQuery && !gwi.correlatedTbNameVec.empty() && csep->hasOrderBy()) - { - gwi.fatalParseError = true; - gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_LIMIT_SUB); - setError(gwi.thd, ER_INTERNAL_ERROR, gwi.parseErrorText, gwi); - return ER_CHECK_NOT_IMPLEMENTED; - } - } // LIMIT processing finishes here + } // ORDER BY end if (select_lex.options & SELECT_DISTINCT) csep->distinct(true); @@ -8214,7 +8093,7 @@ int cp_get_group_plan(THD* thd, SCSEP& csep, cal_impl_if::cal_group_info& gi) int cs_get_derived_plan(derived_handler* handler, THD* thd, SCSEP& csep, gp_walk_info& gwi) { SELECT_LEX select_lex = *handler->select; - int status = getSelectPlan(gwi, select_lex, csep, false, true); + int status = getSelectPlan(gwi, select_lex, csep, false); if (status > 0) return ER_INTERNAL_ERROR; @@ -8412,7 +8291,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro SELECT_LEX* select_cursor = table_ptr->derived->first_select(); // Use Pushdown handler for subquery processing - FromSubQuery fromSub(gwi, select_cursor, true); + FromSubQuery fromSub(gwi, select_cursor); string alias(table_ptr->alias.str); fromSub.alias(lower(alias)); diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 5e35c383c..e93275be6 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -1780,7 +1780,7 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi) gwi.clauseType = WHERE; - if (getSelectPlan(gwi, select_lex, updateCP, false, true) != 0) //@Bug 3030 Modify the error message for unsupported functions + if (getSelectPlan(gwi, select_lex, updateCP, false) != 0) //@Bug 3030 Modify the error message for unsupported functions { if (gwi.cs_vtable_is_update_with_derive) { diff --git a/dbcon/mysql/ha_mcs_impl_if.h b/dbcon/mysql/ha_mcs_impl_if.h index 3c1d2ada6..9975605b7 100644 --- a/dbcon/mysql/ha_mcs_impl_if.h +++ b/dbcon/mysql/ha_mcs_impl_if.h @@ -342,7 +342,7 @@ int cp_get_table_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_table_in int cp_get_group_plan(THD* thd, execplan::SCSEP& csep, cal_impl_if::cal_group_info& gi); int cs_get_derived_plan(derived_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); int cs_get_select_plan(select_handler* handler, THD* thd, execplan::SCSEP& csep, gp_walk_info& gwi); -int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false, bool isPushdownHand = false); +int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, bool isUnion = false, bool isSelectHandlerTop = false); int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, execplan::SCSEP& csep, cal_group_info& gi, bool isUnion = false); void setError(THD* thd, uint32_t errcode, const std::string errmsg, gp_walk_info* gwi); void setError(THD* thd, uint32_t errcode, const std::string errmsg); @@ -352,9 +352,9 @@ const std::string bestTableName(const Item_field* ifp); bool isMCSTable(TABLE* table_ptr); // execution plan util functions prototypes -execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand = false, bool isRefItem = false); -execplan::ReturnedColumn* buildFunctionColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand = false, bool selectBetweenIn = false); -execplan::ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool pushdownHand = false); +execplan::ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupport, bool isRefItem = false); +execplan::ReturnedColumn* buildFunctionColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport, bool selectBetweenIn = false); +execplan::ArithmeticColumn* buildArithmeticColumn(Item_func* item, gp_walk_info& gwi, bool& nonSupport); execplan::ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi); execplan::SimpleColumn* buildSimpleColumn(Item_field* item, gp_walk_info& gwi); execplan::FunctionColumn* buildCaseFunction(Item_func* item, gp_walk_info& gwi, bool& nonSupport); diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index 3b0764a66..64ad79eae 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -787,21 +787,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) return handler; } - // Remove this in 1.4.3 - // Save the original group_list as it can be mutated by the - // optimizer which calls the remove_const() function - Group_list_ptrs *group_list_ptrs = NULL; - if (save_group_list(thd, select_lex, &group_list_ptrs)) - { - return handler; - } - - // Select_handler use the short-cut that effectively disables - // INSERT..SELECT, LDI, SELECT..INTO OUTFILE + // Select_handler couldn't properly process UPSERT..SELECT if ((thd->lex)->sql_command == SQLCOM_INSERT_SELECT - || (thd->lex)->sql_command == SQLCOM_CREATE_TABLE - || (thd->lex)->exchange) - + && thd->lex->duplicates == DUP_UPDATE) { return handler; } diff --git a/dbcon/mysql/ha_scalar_sub.cpp b/dbcon/mysql/ha_scalar_sub.cpp index f41482ba2..7d6d2dd11 100644 --- a/dbcon/mysql/ha_scalar_sub.cpp +++ b/dbcon/mysql/ha_scalar_sub.cpp @@ -253,7 +253,7 @@ execplan::ParseTree* ScalarSub::buildParseTree(PredicateOperator* op) gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSub->get_select_lex()), csep, false) != 0) { //@todo more in error handling if (!gwi.fatalParseError) diff --git a/dbcon/mysql/ha_select_sub.cpp b/dbcon/mysql/ha_select_sub.cpp index 708d2cca8..6d5dfd611 100644 --- a/dbcon/mysql/ha_select_sub.cpp +++ b/dbcon/mysql/ha_select_sub.cpp @@ -77,7 +77,7 @@ SCSEP SelectSubQuery::transform() gwi.tbList.insert(gwi.tbList.begin(), fGwip.tbList.begin(), fGwip.tbList.end()); gwi.derivedTbList.insert(gwi.derivedTbList.begin(), fGwip.derivedTbList.begin(), fGwip.derivedTbList.end()); - if (getSelectPlan(gwi, *(fSelSub->get_select_lex()), csep, false, true) != 0) + if (getSelectPlan(gwi, *(fSelSub->get_select_lex()), csep, false) != 0) { if (!gwi.fatalParseError) { diff --git a/dbcon/mysql/ha_subquery.h b/dbcon/mysql/ha_subquery.h index d14c95131..506cdf0eb 100644 --- a/dbcon/mysql/ha_subquery.h +++ b/dbcon/mysql/ha_subquery.h @@ -181,7 +181,7 @@ class FromSubQuery : public SubQuery { public: FromSubQuery(gp_walk_info&); - FromSubQuery(gp_walk_info&, SELECT_LEX* fromSub, bool isPushdownHand=false); + FromSubQuery(gp_walk_info&, SELECT_LEX* fromSub); ~FromSubQuery(); const SELECT_LEX* fromSub() const { diff --git a/utils/loggingcpp/ErrorMessage.txt b/utils/loggingcpp/ErrorMessage.txt index 3b066fc0f..08233bbf6 100755 --- a/utils/loggingcpp/ErrorMessage.txt +++ b/utils/loggingcpp/ErrorMessage.txt @@ -17,7 +17,7 @@ # Non support errors 1000 ~ 2000. # The query will go through the optimizer again with some optimization turned off 1000 ERR_MISS_JOIN %1% not joined. -1001 ERR_NON_SUPPORTED_FUNCTION Function '%1%' can only be used in the outermost select or order by clause and cannot be used in conjunction with an aggregate function. +1001 ERR_NON_SUPPORTED_FUNCTION Function '%1%' isn't supported. 1002 ERR_INCOMPATIBLE_JOIN %1% incompatible column type specified for join condition. 1003 ERR_CIRCULAR_JOIN Circular joins are not supported. 1004 ERR_MIX_JOIN Mixed %1% JOIN is not supported. From 57595a44ad65276332069b967a92bdc7cdbc9d4d Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Wed, 1 Apr 2020 12:26:07 -0500 Subject: [PATCH 60/84] Merge pull request #1132 from drrtuy/MCOL-3903_1_4_5 MCOL-3903 Fix the fail in expBetween(BETWEEN in projection wasn't --- dbcon/mysql/ha_mcs_execplan.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index da10645f0..99380fdd4 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -5652,7 +5652,7 @@ void gp_walk(const Item* item, void* arg) if (col->type() != Item::COND_ITEM) { - rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, false); + rc = buildReturnedColumn(col, *gwip, gwip->fatalParseError, true); if ( col->type() == Item::FIELD_ITEM ) gwip->fatalParseError = false; @@ -6920,7 +6920,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, ReturnedColumn* rc; if (funcName == "in" || funcName == " IN " || funcName == "between") { - rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, false); + rc = buildFunctionColumn(ifp, gwi, hasNonSupportItem, true); } else { From 4fca8fe7ee3df9052a7fcaf4b098360d18bfbba3 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Wed, 1 Apr 2020 13:47:27 -0400 Subject: [PATCH 61/84] Bump version to 1.4.3-5. --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 11078936f..b6ca065ff 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=4 COLUMNSTORE_VERSION_PATCH=3 -COLUMNSTORE_VERSION_RELEASE=4 +COLUMNSTORE_VERSION_RELEASE=5 From cc7191f75fd6c96a678f810e45fda50fbde28936 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 3 Apr 2020 12:07:23 -0500 Subject: [PATCH 62/84] MCOL-3842: logging cleanup --- oam/oamcpp/liboamcpp.cpp | 14 ++++++------- procmgr/processmanager.cpp | 42 ++++---------------------------------- procmon/processmonitor.cpp | 20 ------------------ 3 files changed, 11 insertions(+), 65 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index a23039f61..8b3454f33 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -5996,12 +5996,12 @@ bool Oam::autoMovePmDbroot(std::string residePM) if ( ret != 0 ) { - writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); + writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) +" ret: " + itoa(ret), LOG_TYPE_ERROR ); } } catch (...) { - writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); + writeLog("EXCEPTION FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); } // check if a copy is available when residePM returns @@ -7448,15 +7448,15 @@ void Oam::removeDbroot(DBRootConfigList& dbrootlist) catch (exception& e) { cout << endl << "**** glusterctl API exception: " << e.what() << endl; - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl; - writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); + cerr << "FAILURE: Error deleting gluster dbroot# " + itoa(dbrootID) << endl; + writeLog("FAILURE: Error deleting gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); exceptionControl("removeDbroot", API_FAILURE); } catch (...) { cout << endl << "**** glusterctl API exception: UNKNOWN" << endl; - cerr << "FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID) << endl; - writeLog("FAILURE: Error assigning gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); + cerr << "FAILURE: Error deleting gluster dbroot# " + itoa(dbrootID) << endl; + writeLog("FAILURE: Error deleting gluster dbroot# " + itoa(dbrootID), LOG_TYPE_ERROR ); exceptionControl("removeDbroot", API_FAILURE); } } @@ -9151,7 +9151,7 @@ int Oam::glusterctl(GLUSTER_COMMANDS command, std::string argument1, std::string string dbr = sysConfig->getConfig("SystemModuleConfig", ModuleDBRootID); string command = "" + DataRedundancyConfigs[pm].pmIpAddr + ":/dbroot" + dbr + " /var/lib/columnstore/data" + dbr + - " glusterfs defaults,direct-io-mode=enable 00"; + " glusterfs defaults,direct-io-mode=enable 0 0"; string toPM = "pm" + itoa(pm + 1); distributeFstabUpdates(command, toPM); } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index faffb62fd..73bdd8d66 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3806,8 +3806,6 @@ int ProcessManager::disableModule(string target, bool manualFlag) //set Columnstore.xml enable state setEnableState( target, SnewState); - log.writeLog(__LINE__, "disableModule - setEnableState", LOG_TYPE_DEBUG); - //sleep a bit to give time for the state change to apply sleep(1); @@ -3816,8 +3814,6 @@ int ProcessManager::disableModule(string target, bool manualFlag) { if ( updatePMSconfig() != API_SUCCESS ) return API_FAILURE; - - log.writeLog(__LINE__, "disableModule - Updated PM server Count", LOG_TYPE_DEBUG); } //Update DBRM section of Columnstore.xml @@ -6514,12 +6510,12 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ catch (SocketClosed& ex) { string error = ex.what(); -// log.writeLog(__LINE__, "EXCEPTION ERROR on mqRequest.read, module " + module + " : " + error, LOG_TYPE_ERROR); + log.writeLog(__LINE__, "EXCEPTION ERROR on mqRequest.read, module " + module + " : " + error, LOG_TYPE_ERROR); return returnStatus; } catch (...) { -// log.writeLog(__LINE__, "EXCEPTION ERROR on mqRequest.read: Caught unknown exception! module " + module, LOG_TYPE_ERROR); + log.writeLog(__LINE__, "EXCEPTION ERROR on mqRequest.read: Caught unknown exception! module " + module, LOG_TYPE_ERROR); return returnStatus; } @@ -6567,11 +6563,11 @@ int ProcessManager::sendMsgProcMon( std::string module, ByteStream msg, int requ catch (exception& ex) { string error = ex.what(); -// log.writeLog(__LINE__, "EXCEPTION ERROR on MessageQueueClient: " + error, LOG_TYPE_ERROR); + log.writeLog(__LINE__, "EXCEPTION ERROR on MessageQueueClient: " + error, LOG_TYPE_ERROR); } catch (...) { -// log.writeLog(__LINE__, "EXCEPTION ERROR on MessageQueueClient: Caught unknown exception!", LOG_TYPE_ERROR); + log.writeLog(__LINE__, "EXCEPTION ERROR on MessageQueueClient: Caught unknown exception!", LOG_TYPE_ERROR); } return returnStatus; @@ -8005,8 +8001,6 @@ int ProcessManager::updatePMSconfig( bool check ) vector IpAddrs; vector nicIDs; - log.writeLog(__LINE__, "updatePMSconfig Started", LOG_TYPE_DEBUG); - pthread_mutex_lock(&THREAD_LOCK); ModuleTypeConfig moduletypeconfig; @@ -8195,8 +8189,6 @@ int ProcessManager::updateWorkerNodeconfig() vector module; vector ipadr; - log.writeLog(__LINE__, "updateWorkerNodeconfig Started", LOG_TYPE_DEBUG); - pthread_mutex_lock(&THREAD_LOCK); //setup current module as work-node #1 by entering it in first @@ -8309,8 +8301,6 @@ int ProcessManager::updateWorkerNodeconfig() sysConfig3->write(); pthread_mutex_unlock(&THREAD_LOCK); - log.writeLog(__LINE__, "updateWorkerNodeconfig completed", LOG_TYPE_DEBUG); - return API_SUCCESS; } @@ -8817,7 +8807,6 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa // StorageManager: Need to make these existence checks use an idbfilesystem op if we // decide to put the BRM-managed files in cloud storage string currentDbrmFile; - log.writeLog(__LINE__, "I declare that I am ProcMgr, and I am running getDBRMData!", LOG_TYPE_DEBUG); IDBFileSystem &fs = IDBPolicy::getFs(currentFileName.c_str()); boost::scoped_ptr oldFile(IDBDataFile::open(IDBPolicy::getType(currentFileName.c_str(), IDBPolicy::WRITEENG), @@ -10098,12 +10087,6 @@ int ProcessManager::OAMParentModuleChange() } else { -// processManager.restartProcessType("mysql", localModule); -// processManager.restartProcessType("ExeMgr", localModule); -// processManager.restartProcessType("WriteEngineServer", localModule); - -// processManager.reinitProcessType("DBRMWorkerNode"); - //send message to start new Standby Process-Manager, if needed newStandbyModule = getStandbyModule(); @@ -10184,15 +10167,6 @@ int ProcessManager::OAMParentModuleChange() } } - //restart DDLProc/DMLProc to perform any rollbacks, if needed - //dont rollback in amazon, wait until down pm recovers -// if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) -// && !amazon ) { -// processManager.restartProcessType("DDLProc", config.moduleName()); -// sleep(1); -// processManager.restartProcessType("DMLProc", config.moduleName()); -// } - if ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) { //change master MySQL Replication setup @@ -10301,8 +10275,6 @@ std::string ProcessManager::getStandbyModule() string backupStandbyModule = "NONE"; string newStandbyModule = "NONE"; - log.writeLog(__LINE__, "getStandbyModule called", LOG_TYPE_DEBUG); - //check if gluster, if so then find PMs that have copies of DBROOT #1 string pmList = ""; @@ -10431,8 +10403,6 @@ bool ProcessManager::setStandbyModule(std::string newStandbyModule, bool send) { Oam oam; - log.writeLog(__LINE__, "setStandbyModule called", LOG_TYPE_DEBUG); - if ( newStandbyModule.empty() ) return true; @@ -10502,8 +10472,6 @@ bool ProcessManager::clearStandbyModule() { Oam oam; - log.writeLog(__LINE__, "clearStandbyModule called", LOG_TYPE_DEBUG); - pthread_mutex_lock(&THREAD_LOCK); Configuration config; @@ -10793,8 +10761,6 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist { Oam oam; - log.writeLog(__LINE__, "setMySQLReplication called", LOG_TYPE_DEBUG); - string MySQLRep; try diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index e2da321ee..b07ab1c27 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -4117,8 +4117,6 @@ int ProcessMonitor::createDataDirs(std::string cloud) MonitorLog log; Oam oam; - log.writeLog(__LINE__, "createDataDirs called", LOG_TYPE_DEBUG); - if ( config.moduleType() == "um" && ( cloud == "amazon-ec2" || cloud == "amazon-vpc") ) { @@ -4278,8 +4276,6 @@ int ProcessMonitor::getDBRMdata(string *path) Oam oam; ByteStream msg; - log.writeLog(__LINE__, "getDBRMdata called", LOG_TYPE_DEBUG); - int returnStatus = API_FAILURE; msg << (ByteStream::byte) GETDBRMDATA; @@ -4964,8 +4960,6 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master { Oam oam; - log.writeLog(__LINE__, "runMasterRep function called", LOG_TYPE_DEBUG); - SystemModuleTypeConfig systemModuleTypeConfig; try @@ -5140,8 +5134,6 @@ int ProcessMonitor::runSlaveRep(std::string& masterLogFile, std::string& masterL { Oam oam; - log.writeLog(__LINE__, "runSlaveRep function called", LOG_TYPE_DEBUG); - // get master replicaion module IP Address string PrimaryUMModuleName; oam.getSystemConfig("PrimaryUMModuleName", PrimaryUMModuleName); @@ -5224,8 +5216,6 @@ int ProcessMonitor::runDisableRep() { Oam oam; - log.writeLog(__LINE__, "runDisableRep function called", LOG_TYPE_DEBUG); - // mysql port number string MySQLPort; @@ -5279,8 +5269,6 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul { Oam oam; - log.writeLog(__LINE__, "runMasterDist function called", LOG_TYPE_DEBUG); - SystemModuleTypeConfig systemModuleTypeConfig; try @@ -5405,8 +5393,6 @@ bool ProcessMonitor::amazonIPCheck() MonitorLog log; Oam oam; - log.writeLog(__LINE__, "amazonIPCheck function called", LOG_TYPE_DEBUG); - // delete description file so it will create a new one string tmpLog = tmpLogDir + "/describeInstance.log"; unlink(tmpLog.c_str()); @@ -5700,8 +5686,6 @@ void ProcessMonitor::unmountExtraDBroots() ModuleConfig moduleconfig; Oam oam; - log.writeLog(__LINE__, "unmountExtraDBroots called ", LOG_TYPE_DEBUG); - string DBRootStorageType = "internal"; try @@ -5797,8 +5781,6 @@ int ProcessMonitor::checkDataMount() //check/update the pmMount files - log.writeLog(__LINE__, "checkDataMount called ", LOG_TYPE_DEBUG); - string DBRootStorageType = "internal"; vector dbrootList; @@ -6031,8 +6013,6 @@ void ProcessMonitor::calTotalUmMemory() //check/update the pmMount files - log.writeLog(__LINE__, "calTotalUmMemory called ", LOG_TYPE_DEBUG); - try { sysinfo(&myinfo); From a88e930f1683ee53fab14ab22179acd0d948e89d Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 3 Apr 2020 12:15:26 -0500 Subject: [PATCH 63/84] MCOL-3842: make sure fuser -k command completes and message thread has started fully before continuing. --- procmgr/main.cpp | 9 ++++++--- procmgr/processmanager.cpp | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index e2a5dd10e..aa11af949 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -47,6 +47,7 @@ using namespace threadpool; using namespace config; bool runStandby = false; +bool MsgThreadActive = false; bool runCold = false; string systemName = "system"; string iface_name; @@ -412,6 +413,8 @@ int main(int argc, char** argv) log.writeLog(__LINE__, "ERROR: makeConfig failed", LOG_TYPE_ERROR); } + + // This will never work..... try { oam.distributeConfigFile(); @@ -474,8 +477,6 @@ static void messageThread(Configuration config) sleep (1); } - log.writeLog(__LINE__, "Message Thread started ..", LOG_TYPE_DEBUG); - //read and cleanup port before trying to use try { @@ -489,6 +490,8 @@ static void messageThread(Configuration config) { } + log.writeLog(__LINE__, "Message Thread started ..", LOG_TYPE_DEBUG); + // //waiting for request // @@ -499,7 +502,7 @@ static void messageThread(Configuration config) try { MessageQueueServer procmgr("ProcMgr"); - + MsgThreadActive = true; for (;;) { try diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 73bdd8d66..030d7b7ca 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -53,6 +53,7 @@ pthread_mutex_t THREAD_LOCK; extern string cloud; extern bool amazon; extern bool runStandby; +extern bool MsgThreadActive; extern string iface_name; extern string PMInstanceType; extern string UMInstanceType; @@ -9187,6 +9188,14 @@ int ProcessManager::switchParentOAMModule(std::string newActiveModuleName) //clear run standby flag; runStandby = false; + int retryCount = 0; + //sleep, give time for message thread to startup + while (!MsgThreadActive && retryCount < 5) + { + log.writeLog(__LINE__, "Waiting for Message Thread...", LOG_TYPE_DEBUG); + sleep(5); + ++retryCount; + } int moduleID = atoi(newActiveModuleName.substr(MAX_MODULE_TYPE_SIZE, MAX_MODULE_ID_SIZE).c_str()); @@ -9922,9 +9931,14 @@ int ProcessManager::OAMParentModuleChange() //clear run standby flag; runStandby = false; - + int retryCount = 0; //sleep, give time for message thread to startup - sleep(5); + while (!MsgThreadActive && retryCount < 5) + { + log.writeLog(__LINE__, "Waiting for Message Thread...", LOG_TYPE_DEBUG); + sleep(5); + ++retryCount; + } try { From accfe3b0709c38e40d13c141e51c0ba2cb392ea0 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 3 Apr 2020 12:17:18 -0500 Subject: [PATCH 64/84] MCOL-3842: we need to delete instances not just release them when a failover happens because the worknodes have changed in configuration files. --- versioning/BRM/masterdbrmnode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioning/BRM/masterdbrmnode.cpp b/versioning/BRM/masterdbrmnode.cpp index 0fa5a2a51..68add1c8b 100644 --- a/versioning/BRM/masterdbrmnode.cpp +++ b/versioning/BRM/masterdbrmnode.cpp @@ -1278,7 +1278,7 @@ void MasterDBRMNode::doReload(messageqcpp::IOSocket* sock) for (i = 0; i < (int) slaves.size(); i++) { - MessageQueueClientPool::releaseInstance(slaves[i]); + MessageQueueClientPool::deleteInstance(slaves[i]); slaves[i] = NULL; } From 89e3f6121be66c6edbecaaf4c513feed860baea3 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 3 Apr 2020 12:22:23 -0500 Subject: [PATCH 65/84] MCOL-3842: Need to exit here if theres already a standby module so gluster doesn't assign a second node. --- procmgr/processmanager.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 030d7b7ca..64d89a4c5 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -10292,6 +10292,26 @@ std::string ProcessManager::getStandbyModule() //check if gluster, if so then find PMs that have copies of DBROOT #1 string pmList = ""; + try + { + oam.getProcessStatus(systemprocessstatus); + for ( unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) + { + if ( systemprocessstatus.processstatus[i].ProcessName == "ProcessManager" && + systemprocessstatus.processstatus[i].ProcessOpState == oam::STANDBY ) + //already have a hot-standby + return ""; + } + } + catch (exception& ex) + { + log.writeLog(__LINE__, "EXCEPTION ERROR on getProcessStatus: " + string(ex.what()), LOG_TYPE_ERROR); + } + catch (...) + { + log.writeLog(__LINE__, "EXCEPTION ERROR on getProcessStatus: Caught unknown exception!", LOG_TYPE_ERROR); + } + if (DataRedundancyConfig == "y") { From 7c65d7fda069e35defee6ce6a99a693dbfcdb3a8 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 3 Apr 2020 12:35:11 -0500 Subject: [PATCH 66/84] MCOL-3842: Fix how processes are restarted in failover event. --- procmgr/main.cpp | 29 ++++++++++++++---------- procmgr/processmanager.cpp | 45 ++++++++++++++++++++++---------------- procmgr/processmanager.h | 4 ++++ 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index aa11af949..e3d177763 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -2095,11 +2095,6 @@ void pingDeviceThread() oam.dbrmctl("halt"); log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); - processManager.setSystemState(oam::BUSY_INIT); - - //string cmd = "/etc/init.d/glusterd restart > /dev/null 2>&1"; - //system(cmd.c_str()); - //send notification oam.sendDeviceNotification(moduleName, MODULE_DOWN); @@ -2112,9 +2107,7 @@ void pingDeviceThread() //set module to disable state processManager.disableModule(moduleName, false); - //call dbrm control - oam.dbrmctl("reload"); - log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); + // if pm, move dbroots to other pms if ( ( moduleName.find("pm") == 0 && !amazon && ( DBRootStorageType != "internal") ) || @@ -2145,6 +2138,9 @@ void pingDeviceThread() { processManager.setModuleState(moduleName, oam::AUTO_DISABLED); + //call dbrm control + oam.dbrmctl("reload"); + log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); // resume the dbrm oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); @@ -2163,6 +2159,9 @@ void pingDeviceThread() { if ( moduleName.find("um") == 0 ) { + //call dbrm control + oam.dbrmctl("reload"); + log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); // resume the dbrm oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); @@ -2353,8 +2352,8 @@ void pingDeviceThread() } } - //set recycle process - processManager.recycleProcess(moduleName); + //set reinit process + processManager.reinitProcesses(); //set query system state ready processManager.setQuerySystemState(true); @@ -2368,6 +2367,9 @@ void pingDeviceThread() ( opState != oam::AUTO_DISABLED ) ) { + //call dbrm control + oam.dbrmctl("reload"); + log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); // resume the dbrm oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); @@ -2378,13 +2380,16 @@ void pingDeviceThread() } else { + //call dbrm control + oam.dbrmctl("reload"); + log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); // non-amazon // resume the dbrm oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - //set recycle process - processManager.recycleProcess(moduleName); + //set reinit process + processManager.reinitProcesses(); //set query system state ready processManager.setQuerySystemState(true); diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 64d89a4c5..d74cf8c7b 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -921,8 +921,6 @@ void processMSG(messageqcpp::IOSocket* cfIos) status = processManager.disableModule(moduleName, true); log.writeLog(__LINE__, "Disable Module Completed on " + moduleName, LOG_TYPE_INFO); - processManager.recycleProcess(moduleName); - //check for SIMPLEX Processes on mate might need to be started processManager.checkSimplexModule(moduleName); @@ -3739,12 +3737,6 @@ int ProcessManager::disableModule(string target, bool manualFlag) bool degraded; oam.getModuleStatus(target, opState, degraded); - if (opState == newState || opState == oam::MAN_DISABLED) - { - pthread_mutex_unlock(&THREAD_LOCK); - return API_SUCCESS; - } - // if current state is AUTO_DISABLED and new state is MAN_DISABLED // update state to MAN_DISABLED @@ -3822,7 +3814,6 @@ int ProcessManager::disableModule(string target, bool manualFlag) { return API_FAILURE; } - processManager.recycleProcess(target); //check for SIMPLEX Processes on mate might need to be started processManager.checkSimplexModule(target); @@ -3830,11 +3821,31 @@ int ProcessManager::disableModule(string target, bool manualFlag) //distribute config file distributeConfigFile("system"); + processManager.reinitProcesses(); + log.writeLog(__LINE__, "disableModule successfully complete for " + target, LOG_TYPE_DEBUG); return API_SUCCESS; } +void ProcessManager::reinitProcesses() +{ + log.writeLog(__LINE__, "reinitProcesses... ", LOG_TYPE_DEBUG); + + restartProcessType("DBRMControllerNode"); + reinitProcessType("ExeMgr"); + reinitProcessType("DBRMWorkerNode"); + restartProcessType("WriteEngineServer"); + sleep(1); + startProcessType("DDLProc"); + sleep(1); + startProcessType("DMLProc"); + reinitProcessType("DDLProc"); + reinitProcessType("DMLProc"); + + log.writeLog(__LINE__, "reinitProcesses complete", LOG_TYPE_DEBUG); +} + /****************************************************************************************** * @brief recycleProcess * @@ -10040,8 +10051,12 @@ int ProcessManager::OAMParentModuleChange() for ( ; pt1 != downModuleList.end() ; pt1++) { - disableModule(*pt1, false); - processManager.setProcessStates(*pt1, oam::AUTO_OFFLINE); + // Don't do this again for downOAMParentName we just did it 3 lines ago + if (*pt1 != downOAMParentName) + { + disableModule(*pt1, false); + processManager.setProcessStates(*pt1, oam::AUTO_OFFLINE); + } } //distribute config file @@ -10090,9 +10105,6 @@ int ProcessManager::OAMParentModuleChange() sleep(2); } - //set recycle process - processManager.recycleProcess(downOAMParentName); - //restart/reinit processes to force their release of the controller node port if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) @@ -10189,9 +10201,6 @@ int ProcessManager::OAMParentModuleChange() processManager.setMySQLReplication(devicenetworklist, config.moduleName()); } - //set query system state not ready - processManager.setQuerySystemState(true); - // clear alarm aManager.sendAlarmReport(config.moduleName().c_str(), MODULE_SWITCH_ACTIVE, CLEAR); @@ -10363,8 +10372,6 @@ std::string ProcessManager::getStandbyModule() //not gluster, check by status try { - oam.getProcessStatus(systemprocessstatus); - for ( unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) { if ( systemprocessstatus.processstatus[i].ProcessName == "ProcessManager" && diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 9c8e1378d..038b8ab40 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -299,6 +299,10 @@ public: */ int disableModule(std::string target, bool manualFlag); + /** + *@brief reinit Processes trying to replace recycleProcess + */ + void reinitProcesses(); /** *@brief recycle Processes */ From 9ab09f584fdd747c76bc962e21e1ef743decef82 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 3 Apr 2020 12:40:03 -0500 Subject: [PATCH 67/84] MCOL-3900: this causes 3 node config to have split brain for primary UM and primary OAM if 3rd node fails. --- procmgr/processmanager.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index d74cf8c7b..e18baf156 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3815,9 +3815,6 @@ int ProcessManager::disableModule(string target, bool manualFlag) return API_FAILURE; } - //check for SIMPLEX Processes on mate might need to be started - processManager.checkSimplexModule(target); - //distribute config file distributeConfigFile("system"); From 5c977f2c850121148b6485f5e2a05538aedfcae5 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 3 Apr 2020 12:41:06 -0500 Subject: [PATCH 68/84] MCOL-3905: slavecomm should use the path provided to the function for openning journal file with the change that added UUID to path for downloaded files during getDBRMData. --- versioning/BRM/slavecomm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index eef66bea4..097cf3959 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -2195,7 +2195,7 @@ int SlaveComm::replayJournal(string prefix) fName = prefix + "_journal"; } - const char* filename = journalName.c_str(); + const char* filename = fName.c_str(); IDBDataFile* journalf = IDBDataFile::open( IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0); From 78117414de4e27c8ca3c40c56b5fa2bcd6d705ed Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 6 Apr 2020 14:06:57 -0500 Subject: [PATCH 69/84] MCOL-3842: Fix sequence of reinit / reload failover code fix. Cleanup logging. --- procmgr/main.cpp | 12 ++++++++---- procmgr/processmanager.cpp | 38 +++++++++++++++++--------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index e3d177763..190c72da4 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -2380,17 +2380,21 @@ void pingDeviceThread() } else { + processManager.distributeConfigFile("system"); + + processManager.reinitProcesses(); + + // non-amazon //call dbrm control oam.dbrmctl("reload"); log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); - // non-amazon // resume the dbrm oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); - //set reinit process - processManager.reinitProcesses(); - + //set recycle process + //processManager.recycleProcess(moduleName); + //processManager.reinitProcesses(); //set query system state ready processManager.setQuerySystemState(true); } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index e18baf156..aad0623dc 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3737,6 +3737,12 @@ int ProcessManager::disableModule(string target, bool manualFlag) bool degraded; oam.getModuleStatus(target, opState, degraded); + if (opState == newState || opState == oam::MAN_DISABLED) + { + pthread_mutex_unlock(&THREAD_LOCK); + return API_SUCCESS; + } + // if current state is AUTO_DISABLED and new state is MAN_DISABLED // update state to MAN_DISABLED @@ -3818,28 +3824,27 @@ int ProcessManager::disableModule(string target, bool manualFlag) //distribute config file distributeConfigFile("system"); - processManager.reinitProcesses(); + processManager.reinitProcesses(); log.writeLog(__LINE__, "disableModule successfully complete for " + target, LOG_TYPE_DEBUG); return API_SUCCESS; } + void ProcessManager::reinitProcesses() { log.writeLog(__LINE__, "reinitProcesses... ", LOG_TYPE_DEBUG); - restartProcessType("DBRMControllerNode"); reinitProcessType("ExeMgr"); reinitProcessType("DBRMWorkerNode"); - restartProcessType("WriteEngineServer"); + reinitProcessType("WriteEngineServer"); sleep(1); - startProcessType("DDLProc"); - sleep(1); - startProcessType("DMLProc"); reinitProcessType("DDLProc"); + sleep(1); reinitProcessType("DMLProc"); + log.writeLog(__LINE__, "reinitProcesses complete", LOG_TYPE_DEBUG); } @@ -3900,7 +3905,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule) startProcessType("mysqld"); - return; + return; } /****************************************************************************************** @@ -6739,8 +6744,6 @@ void ProcessManager::setQuerySystemState(bool set) Oam oam; BRM::DBRM dbrm; - log.writeLog(__LINE__, "setQuerySystemState called = " + oam.itoa(set), LOG_TYPE_DEBUG); - try { dbrm.setSystemQueryReady(set); @@ -8169,8 +8172,6 @@ int ProcessManager::updatePMSconfig( bool check ) sysConfig1->write(); pthread_mutex_unlock(&THREAD_LOCK); - log.writeLog(__LINE__, "updatePMSconfig completed", LOG_TYPE_DEBUG); - return API_SUCCESS; } catch (...) @@ -8510,8 +8511,6 @@ int ProcessManager::setPMProcIPs( std::string moduleName, std::string processNam Oam oam; ModuleConfig moduleconfig; - log.writeLog(__LINE__, "setPMProcIPs called for " + moduleName, LOG_TYPE_DEBUG); - pthread_mutex_lock(&THREAD_LOCK); if ( processName == oam::UnassignedName || processName == "DDLProc") @@ -8630,8 +8629,6 @@ int ProcessManager::distributeConfigFile(std::string name, std::string file) Oam oam; int returnStatus = oam::API_SUCCESS; - log.writeLog(__LINE__, "distributeConfigFile called for " + name + " file = " + file, LOG_TYPE_DEBUG); - string dirName = std::string(MCSSYSCONFDIR) + "/columnstore/"; string fileName = dirName + file; @@ -9941,7 +9938,7 @@ int ProcessManager::OAMParentModuleChange() runStandby = false; int retryCount = 0; //sleep, give time for message thread to startup - while (!MsgThreadActive && retryCount < 5) + while (!MsgThreadActive && retryCount < 10) { log.writeLog(__LINE__, "Waiting for Message Thread...", LOG_TYPE_DEBUG); sleep(5); @@ -10043,7 +10040,7 @@ int ProcessManager::OAMParentModuleChange() //do it here to get current processes active faster to process queries faster processManager.setProcessStates(downOAMParentName, oam::AUTO_OFFLINE); - //set other down modules to disable state + //set OTHER down modules to disable state vector::iterator pt1 = downModuleList.begin(); for ( ; pt1 != downModuleList.end() ; pt1++) @@ -10059,7 +10056,7 @@ int ProcessManager::OAMParentModuleChange() //distribute config file distributeConfigFile("system"); - //restart local module + //restart local module WHY?? processManager.stopModule(config.moduleName(), oam::FORCEFUL, true); string localModule = config.moduleName(); @@ -10102,11 +10099,12 @@ int ProcessManager::OAMParentModuleChange() sleep(2); } + //restart/reinit processes to force their release of the controller node port if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) { - status = 0; + int status = 0; } else { @@ -10326,8 +10324,6 @@ std::string ProcessManager::getStandbyModule() string errmsg; oam.glusterctl(oam::GLUSTER_WHOHAS, "1", pmList, errmsg); - log.writeLog(__LINE__, "GLUSTER_WHOHAS called:" + pmList, LOG_TYPE_DEBUG); - boost::char_separator sep(" "); boost::tokenizer< boost::char_separator > tokens(pmList, sep); From 8458ed0a662c79de132177febe23f59783cb6fd6 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 6 Apr 2020 15:25:02 -0500 Subject: [PATCH 70/84] MCOL-3842: Fix logging and retry count for MessageThread startup waiting. --- procmgr/main.cpp | 3 ++- procmgr/processmanager.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 190c72da4..6aae4cfb6 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -414,7 +414,8 @@ int main(int argc, char** argv) } - // This will never work..... + // TODO: This is called before MessageThread is created. + // Doesn't break anything but can be removed as it's done after MessageThread creation. try { oam.distributeConfigFile(); diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index aad0623dc..81887d25c 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -9195,7 +9195,7 @@ int ProcessManager::switchParentOAMModule(std::string newActiveModuleName) runStandby = false; int retryCount = 0; //sleep, give time for message thread to startup - while (!MsgThreadActive && retryCount < 5) + while (!MsgThreadActive && retryCount < 10) { log.writeLog(__LINE__, "Waiting for Message Thread...", LOG_TYPE_DEBUG); sleep(5); @@ -10104,7 +10104,7 @@ int ProcessManager::OAMParentModuleChange() if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM) && ( moduleNameList.size() <= 0 && config.moduleType() == "pm") ) { - int status = 0; + // Do Nothing } else { From 699b55d8dd527e9dedf5536a0b152ac78b632c22 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 6 Apr 2020 17:03:08 -0400 Subject: [PATCH 71/84] Bumped version to 1.4.3-6. --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b6ca065ff..2b6978986 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=4 COLUMNSTORE_VERSION_PATCH=3 -COLUMNSTORE_VERSION_RELEASE=5 +COLUMNSTORE_VERSION_RELEASE=6 From 4bd3ceb24d13546a6f92c69e2f9915f036426bbf Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 8 Apr 2020 18:58:57 -0500 Subject: [PATCH 72/84] Fix the _journal file path for getDBRMData if it was not received because it is 0 length. remove extra log lines. --- procmon/processmonitor.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index b07ab1c27..b3c9ba1d8 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -4437,9 +4437,9 @@ int ProcessMonitor::getDBRMdata(string *path) //create journal file if none come across if ( !journalFile) { - string journalFilename = "/var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves_journal"; - IDBDataFile *idbJournalFile = IDBDataFile::open(IDBPolicy::getType(journalFilename.c_str(), - IDBPolicy::WRITEENG), journalFilename.c_str(), "w", 0); + bf::path pJournalFilename(pTmp / "BRM_saves_journal"); + IDBDataFile *idbJournalFile = IDBDataFile::open(IDBPolicy::getType(pJournalFilename.string().c_str(), + IDBPolicy::WRITEENG), pJournalFilename.string().c_str(), "w", 0); delete idbJournalFile; //string cmd = "touch " + startup::StartUp::installDir() + "/data1/systemFiles/dbrm/BRM_saves_journal"; //system(cmd.c_str()); @@ -6138,7 +6138,7 @@ int ProcessMonitor::glusterAssign(std::string dbrootID) if ( WEXITSTATUS(ret) != 0 ) { - log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); + //log.writeLog(__LINE__, "glusterAssign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); ifstream in(tmpLog.c_str()); in.seekg(0, std::ios::end); @@ -6183,7 +6183,7 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) if ( WEXITSTATUS(ret) != 0 ) { - log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); + //log.writeLog(__LINE__, "glusterUnassign mount failure: dbroot: " + dbrootID + " error: " + oam.itoa(WEXITSTATUS(ret)), LOG_TYPE_ERROR); ifstream in(tmpLog.c_str()); in.seekg(0, std::ios::end); From 2bdb40ed646d92e70ef71ce892a2cc18367ae176 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 10 Apr 2020 17:43:21 -0500 Subject: [PATCH 73/84] add logic for ha_mcs_impl_direct_update_delete_rows to handle delete statements with replication enabled. --- dbcon/mysql/ha_mcs_impl.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index e93275be6..00987e353 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2291,6 +2291,17 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows *affected_rows) cal_impl_if::gp_walk_info gwi; gwi.thd = thd; + if (thd->slave_thread && !get_replication_slave(thd) && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + if (execute) { rc = doUpdateDelete(thd, gwi); From e7caeb872292f3cd492b50e6006088c33f315010 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 13 Apr 2020 14:02:06 -0500 Subject: [PATCH 74/84] Another check for Update/Delete with replication. --- dbcon/mysql/ha_mcs_impl.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 00987e353..9ed7d9d53 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2300,7 +2300,11 @@ int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows *affected_rows) thd->lex->sql_command == SQLCOM_DELETE_MULTI || thd->lex->sql_command == SQLCOM_TRUNCATE || thd->lex->sql_command == SQLCOM_LOAD)) + { + if (affected_rows) + *affected_rows = 0; return 0; + } if (execute) { @@ -4927,6 +4931,17 @@ int ha_cs_impl_pushdown_init(mcs_handler_info* handler_info, TABLE* table) return 0; } + if (thd->slave_thread && !get_replication_slave(thd) && ( + thd->lex->sql_command == SQLCOM_INSERT || + thd->lex->sql_command == SQLCOM_INSERT_SELECT || + thd->lex->sql_command == SQLCOM_UPDATE || + thd->lex->sql_command == SQLCOM_UPDATE_MULTI || + thd->lex->sql_command == SQLCOM_DELETE || + thd->lex->sql_command == SQLCOM_DELETE_MULTI || + thd->lex->sql_command == SQLCOM_TRUNCATE || + thd->lex->sql_command == SQLCOM_LOAD)) + return 0; + //Update and delete code if ( ((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) return doUpdateDelete(thd, gwi); From 11154bfdd57432b7a31ce23fd45b71b416163782 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 14 Apr 2020 12:13:21 -0500 Subject: [PATCH 75/84] MCOL-3945: retry was not being incremented in this loop. --- oam/oamcpp/oamcache.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/oam/oamcpp/oamcache.cpp b/oam/oamcpp/oamcache.cpp index bd2957764..aae381b18 100644 --- a/oam/oamcpp/oamcache.cpp +++ b/oam/oamcpp/oamcache.cpp @@ -155,6 +155,7 @@ void OamCache::checkReload() } sleep(1); + ++retry; //cout << "pm " << *it << " -> connection " << (i-1) << endl; } From 06774251399b2add6eac6ecdb80bac9b651a3d75 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 14 Apr 2020 12:15:57 -0500 Subject: [PATCH 76/84] MCOL-3945: move execution of saveBRM to after autoMovePmDbroot on OAMParentModuleChange. --- procmgr/processmanager.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 81887d25c..24bbfbc07 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -7450,6 +7450,7 @@ void startModuleThread(string module) int opState = oam::ACTIVE; bool degraded; oam.getModuleStatus(moduleName, opState, degraded); + log.writeLog(__LINE__, "startModuleThread on " + moduleName + " opstate = " + oam.itoa(opState), LOG_TYPE_DEBUG); if ( opState == oam::AUTO_OFFLINE || opState == oam::AUTO_INIT) startType = oam::AUTO_OFFLINE; @@ -9886,9 +9887,6 @@ int ProcessManager::OAMParentModuleChange() log.writeLog(__LINE__, " ", LOG_TYPE_DEBUG); log.writeLog(__LINE__, "*** OAMParentModule outage, OAM Parent Module change-over started ***", LOG_TYPE_DEBUG); - //run save.brm script - processManager.saveBRM(true, false); - gdownActiveOAMModule = downOAMParentName; // update Columnstore.xml entries @@ -9954,6 +9952,10 @@ int ProcessManager::OAMParentModuleChange() log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR); } + //run save.brm script + //MCOL-3945 move saveBRM after autoMovePmDbroot as this will potentially mount the dbrm directory from dbroot1 + processManager.saveBRM(true, false); + //distribute config file distributeConfigFile("system"); From 18d4d7eae7152a3a98b9e4a89275002ab42c0386 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 14 Apr 2020 14:49:32 -0500 Subject: [PATCH 77/84] Remove extra logging for debug. --- procmgr/processmanager.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 24bbfbc07..bf9bd3206 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -7450,7 +7450,6 @@ void startModuleThread(string module) int opState = oam::ACTIVE; bool degraded; oam.getModuleStatus(moduleName, opState, degraded); - log.writeLog(__LINE__, "startModuleThread on " + moduleName + " opstate = " + oam.itoa(opState), LOG_TYPE_DEBUG); if ( opState == oam::AUTO_OFFLINE || opState == oam::AUTO_INIT) startType = oam::AUTO_OFFLINE; From 6765b71fd57dc633fd92ff9ef897965c7b83432d Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 16 Apr 2020 14:42:20 -0500 Subject: [PATCH 78/84] Restart ExeMgr / DMLProc / DDLProc on failovers for RM config update. Also make sure ExeMgr restart happens on primary failover. --- procmgr/processmanager.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index bf9bd3206..fdb4f33ce 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3836,13 +3836,13 @@ void ProcessManager::reinitProcesses() { log.writeLog(__LINE__, "reinitProcesses... ", LOG_TYPE_DEBUG); - reinitProcessType("ExeMgr"); reinitProcessType("DBRMWorkerNode"); reinitProcessType("WriteEngineServer"); + restartProcessType("ExeMgr"); sleep(1); - reinitProcessType("DDLProc"); + restartProcessType("DDLProc"); sleep(1); - reinitProcessType("DMLProc"); + restartProcessType("DMLProc"); log.writeLog(__LINE__, "reinitProcesses complete", LOG_TYPE_DEBUG); @@ -10197,6 +10197,8 @@ int ProcessManager::OAMParentModuleChange() processManager.setMySQLReplication(devicenetworklist, config.moduleName()); } + processManager.reinitProcesses(); + // clear alarm aManager.sendAlarmReport(config.moduleName().c_str(), MODULE_SWITCH_ACTIVE, CLEAR); From de99a2767315ff0ccdb254defccb3d2247865e59 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 17 Apr 2020 03:51:45 -0500 Subject: [PATCH 79/84] Restart DBRMController on failover --- procmgr/processmanager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index fdb4f33ce..289604d8a 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3836,6 +3836,7 @@ void ProcessManager::reinitProcesses() { log.writeLog(__LINE__, "reinitProcesses... ", LOG_TYPE_DEBUG); + restartProcessType("DBRMControllerNode"); reinitProcessType("DBRMWorkerNode"); reinitProcessType("WriteEngineServer"); restartProcessType("ExeMgr"); From 8dab0fc397a2b67099efa81d0c9ab131cd4875df Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Mon, 20 Apr 2020 03:14:48 -0500 Subject: [PATCH 80/84] Have process reinits wait for DMLProc to not be in BUSY_INIT state after starting. Restart ControllerNode on Primary failover only. --- procmgr/processmanager.cpp | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 289604d8a..35d3a6a0a 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3834,9 +3834,10 @@ int ProcessManager::disableModule(string target, bool manualFlag) void ProcessManager::reinitProcesses() { + Oam oam; + log.writeLog(__LINE__, "reinitProcesses... ", LOG_TYPE_DEBUG); - restartProcessType("DBRMControllerNode"); reinitProcessType("DBRMWorkerNode"); reinitProcessType("WriteEngineServer"); restartProcessType("ExeMgr"); @@ -3844,7 +3845,30 @@ void ProcessManager::reinitProcesses() restartProcessType("DDLProc"); sleep(1); restartProcessType("DMLProc"); + sleep(1); + // waiting until dml are ACTIVE + while (true) + { + ProcessStatus DMLprocessstatus; + + try + { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch (...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); + else + break; + + // wait some more + sleep(2); + } log.writeLog(__LINE__, "reinitProcesses complete", LOG_TYPE_DEBUG); } @@ -10198,6 +10222,8 @@ int ProcessManager::OAMParentModuleChange() processManager.setMySQLReplication(devicenetworklist, config.moduleName()); } + processManager.restartProcessType("DBRMControllerNode"); + processManager.reinitProcesses(); // clear alarm From 1dff484c4197c11eba97ace92f7ae072c9e402bc Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 23 Apr 2020 04:14:59 -0500 Subject: [PATCH 81/84] Fix for config to update global flah. --- procmon/main.cpp | 3 ++- procmon/processmonitor.cpp | 30 ++++++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/procmon/main.cpp b/procmon/main.cpp index 533c69d92..1ac9b4510 100644 --- a/procmon/main.cpp +++ b/procmon/main.cpp @@ -3210,6 +3210,7 @@ void processStatusMSG(messageqcpp::IOSocket* cfIos) oam::DeviceNetworkConfig devicenetworkconfig; oam::DeviceNetworkList devicenetworklist; string value; + MonitorConfig currentConfig; *msg >> moduleCount; @@ -3223,7 +3224,7 @@ void processStatusMSG(messageqcpp::IOSocket* cfIos) log.writeLog(__LINE__, "statusControl: REQUEST RECEIVED: Add Module"); string moduleType = devicenetworkconfig.DeviceName.substr(0, MAX_MODULE_TYPE_SIZE); - string OAMParentModuleType = config.OAMParentName().substr(0, 2); + string OAMParentModuleType = currentConfig.OAMParentName().substr(0, 2); // add to module status shared memory DeviceNetworkList::iterator pt = devicenetworklist.begin(); diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index b3c9ba1d8..468ce8891 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -460,6 +460,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO { Oam oam; ByteStream ackMsg; + MonitorConfig currentConfig; ByteStream::byte messageType; ByteStream::byte requestID; @@ -809,6 +810,24 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO if ( processName == "cpimport" ) { system("pkill -sighup cpimport"); + for (int i=0; i < 10; i++) + { + //get pid + char buf[512]; + FILE *cmd_pipe = popen("pidof -s cpimport", "r"); + + fgets(buf, 512, cmd_pipe); + pid_t pid = strtoul(buf, NULL, 10); + + pclose( cmd_pipe ); + + if (pid) + sleep(2); + else + break; + } + // kill other processes + system("pkill -9 cpimport.bin"); } else { @@ -2217,6 +2236,7 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName, char* argList[MAXARGUMENTS]; unsigned int i = 0; MonitorLog log; + MonitorConfig currentConfig; unsigned int numAugs = 0; Oam oam; SystemProcessStatus systemprocessstatus; @@ -3736,7 +3756,7 @@ int ProcessMonitor::updateConfig() { //ProcMon log file MonitorLog log; -// MonitorConfig config; + MonitorConfig currentConfig; // ProcessMonitor aMonitor(config, log); Oam oam; @@ -3758,7 +3778,7 @@ int ProcessMonitor::updateConfig() } //Update a map for application launch ID for this Process-Monitor - string OAMParentModuleType = config.OAMParentName().substr(0, MAX_MODULE_TYPE_SIZE); + string OAMParentModuleType = currentConfig.OAMParentName().substr(0, MAX_MODULE_TYPE_SIZE); string systemModuleType = config.moduleName().substr(0, MAX_MODULE_TYPE_SIZE); for ( unsigned int i = 0 ; i < systemprocessconfig.processconfig.size(); i++) @@ -5476,9 +5496,11 @@ bool ProcessMonitor::amazonIPCheck() // get all ips if parent oam // get just parent and local if not parent oam - if ( config.moduleName() == config.OAMParentName() || + MonitorConfig currentConfig; + + if ( config.moduleName() == currentConfig.OAMParentName() || moduleName == config.moduleName() || - moduleName == config.OAMParentName() ) + moduleName == currentConfig.OAMParentName() ) { HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); From ee5ed53e7774b698d3e34b1e58efb0c907fb1b05 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 23 Apr 2020 05:08:59 -0500 Subject: [PATCH 82/84] Fixes for cpimport primary node failover and locking issues. --- procmgr/main.cpp | 119 ++++++++++++++++++++++++++++++++----- procmgr/processmanager.cpp | 97 +++++++++++++++++++----------- procmgr/processmanager.h | 2 +- 3 files changed, 168 insertions(+), 50 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 6aae4cfb6..7de3a3d9f 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1649,17 +1649,17 @@ void pingDeviceThread() if (busy) break; - //set query system state not ready - processManager.setQuerySystemState(false); - - processManager.setSystemState(oam::BUSY_INIT); - processManager.reinitProcessType("cpimport"); // halt the dbrm oam.dbrmctl("halt"); log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); + //set query system state not ready + processManager.setQuerySystemState(false); + + processManager.setSystemState(oam::BUSY_INIT); + aManager.sendAlarmReport(moduleName.c_str(), MODULE_DOWN_AUTO, CLEAR); //send notification @@ -1724,6 +1724,36 @@ void pingDeviceThread() //set query system state ready processManager.setQuerySystemState(true); + // waiting until dml are ACTIVE + // disableModule is going to trigger DMLProc to restart wait for it + int retry = 0; + while (retry < 30) + { + ProcessStatus DMLprocessstatus; + + try + { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch (...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); + + if (DMLprocessstatus.ProcessOpState == oam::ACTIVE) + break; + + if (DMLprocessstatus.ProcessOpState == oam::FAILED) + break; + + // wait some more + sleep(2); + ++retry; + } + goto break_case; } } @@ -2016,6 +2046,7 @@ void pingDeviceThread() log.writeLog(__LINE__, "Module failed to auto start: " + moduleName, LOG_TYPE_CRITICAL); + if ( amazon ) processManager.setSystemState(oam::FAILED); else @@ -2024,6 +2055,35 @@ void pingDeviceThread() //set query system state ready processManager.setQuerySystemState(true); + // waiting until dml are ACTIVE + // disableModule is going to trigger DMLProc to restart wait for it + int retry = 0; + while (retry < 30) + { + ProcessStatus DMLprocessstatus; + + try + { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch (...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); + + if (DMLprocessstatus.ProcessOpState == oam::ACTIVE) + break; + + if (DMLprocessstatus.ProcessOpState == oam::FAILED) + break; + + // wait some more + sleep(2); + ++retry; + } //clear count moduleInfoList[moduleName] = 0; } @@ -2085,17 +2145,22 @@ void pingDeviceThread() Configuration config; log.writeLog(__LINE__, "*** module is down: " + moduleName, LOG_TYPE_CRITICAL); - //set query system state not ready - processManager.setQuerySystemState(false); - - processManager.setSystemState(oam::BUSY_INIT); - processManager.reinitProcessType("cpimport"); // halt the dbrm oam.dbrmctl("halt"); log.writeLog(__LINE__, "'dbrmctl halt' done", LOG_TYPE_DEBUG); + //set query system state not ready + processManager.setQuerySystemState(false); + + processManager.setSystemState(oam::BUSY_INIT); + + //call dbrm control + oam.dbrmctl("reload"); + log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); + + //send notification oam.sendDeviceNotification(moduleName, MODULE_DOWN); @@ -2149,6 +2214,36 @@ void pingDeviceThread() //set query system state ready processManager.setQuerySystemState(true); + // waiting until dml are ACTIVE + // disableModule is going to trigger DMLProc to restart wait for it + int retry = 0; + while (retry < 30) + { + ProcessStatus DMLprocessstatus; + + try + { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch (...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); + + if (DMLprocessstatus.ProcessOpState == oam::ACTIVE) + break; + + if (DMLprocessstatus.ProcessOpState == oam::FAILED) + break; + + // wait some more + sleep(2); + ++retry; + } + break; } } @@ -2381,10 +2476,6 @@ void pingDeviceThread() } else { - processManager.distributeConfigFile("system"); - - processManager.reinitProcesses(); - // non-amazon //call dbrm control oam.dbrmctl("reload"); diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 35d3a6a0a..4dd6ac9f9 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3832,7 +3832,7 @@ int ProcessManager::disableModule(string target, bool manualFlag) } -void ProcessManager::reinitProcesses() +void ProcessManager::reinitProcesses(std::string skipModule) { Oam oam; @@ -3840,35 +3840,12 @@ void ProcessManager::reinitProcesses() reinitProcessType("DBRMWorkerNode"); reinitProcessType("WriteEngineServer"); - restartProcessType("ExeMgr"); + restartProcessType("ExeMgr",skipModule); sleep(1); - restartProcessType("DDLProc"); + restartProcessType("DDLProc",skipModule); sleep(1); - restartProcessType("DMLProc"); - sleep(1); - - // waiting until dml are ACTIVE - while (true) - { - ProcessStatus DMLprocessstatus; - - try - { - oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); - } - catch (exception& ex) - {} - catch (...) - {} - - if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) - log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); - else - break; - - // wait some more - sleep(2); - } + restartProcessType("DMLProc",skipModule); + sleep(3); log.writeLog(__LINE__, "reinitProcesses complete", LOG_TYPE_DEBUG); } @@ -4921,6 +4898,7 @@ int ProcessManager::reinitProcessType( std::string processName ) if ( systemprocessstatus.processstatus[i].ProcessName == "ServerMonitor" ) { // found one, request reinit of it + log.writeLog(__LINE__, "reinitProcessType: cpimport" + systemprocessstatus.processstatus[i].Module, LOG_TYPE_DEBUG); retStatus = processManager.reinitProcess(systemprocessstatus.processstatus[i].Module, "cpimport"); log.writeLog(__LINE__, "reinitProcessType: ACK received from Process-Monitor, return status = " + oam.itoa(retStatus), LOG_TYPE_DEBUG); @@ -9967,6 +9945,10 @@ int ProcessManager::OAMParentModuleChange() ++retryCount; } + //run save.brm script + //Nope turns out this has to be done first... + + processManager.saveBRM(false); try { oam.autoMovePmDbroot(downOAMParentName); @@ -9976,10 +9958,6 @@ int ProcessManager::OAMParentModuleChange() log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR); } - //run save.brm script - //MCOL-3945 move saveBRM after autoMovePmDbroot as this will potentially mount the dbrm directory from dbroot1 - processManager.saveBRM(true, false); - //distribute config file distributeConfigFile("system"); @@ -10098,8 +10076,11 @@ int ProcessManager::OAMParentModuleChange() status = startsystemthreadStatus; } + reinitProcessType("cpimport"); + // waiting until dml are ACTIVE - while (true) + int retry = 0; + while (retry < 30) { ProcessStatus DMLprocessstatus; @@ -10123,6 +10104,7 @@ int ProcessManager::OAMParentModuleChange() // wait some more sleep(2); + ++retry; } @@ -10226,6 +10208,35 @@ int ProcessManager::OAMParentModuleChange() processManager.reinitProcesses(); + // waiting until dml are ACTIVE + retry = 0; + while (retry < 30) + { + ProcessStatus DMLprocessstatus; + + try + { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch (...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); + + if (DMLprocessstatus.ProcessOpState == oam::ACTIVE) + break; + + if (DMLprocessstatus.ProcessOpState == oam::FAILED) + break; + + // wait some more + sleep(2); + ++retry; + } + // clear alarm aManager.sendAlarmReport(config.moduleName().c_str(), MODULE_SWITCH_ACTIVE, CLEAR); @@ -11151,7 +11162,15 @@ int ProcessManager::glusterAssign(std::string moduleName, std::string dbroot) msg << dbroot; int returnStatus = sendMsgProcMon( moduleName, msg, requestID, 30 ); - + int retry = 0; + // Try this for a minute because in failover the node returning to service may not be listening yet + while(returnStatus != API_SUCCESS && retry < 60) + { + log.writeLog(__LINE__, "glusterAssign retrying...", LOG_TYPE_DEBUG); + returnStatus = sendMsgProcMon( moduleName, msg, requestID, 30 ); + sleep(1); + ++retry; + } if ( returnStatus == API_SUCCESS) { //log the success event @@ -11181,7 +11200,15 @@ int ProcessManager::glusterUnassign(std::string moduleName, std::string dbroot) msg << dbroot; int returnStatus = sendMsgProcMon( moduleName, msg, requestID, 30 ); - + int retry = 0; + // Try this for a minute because in failover the node returning to service may not be listening yet + while(returnStatus != API_SUCCESS && retry < 60) + { + log.writeLog(__LINE__, "glusterUnassign retrying...", LOG_TYPE_DEBUG); + returnStatus = sendMsgProcMon( moduleName, msg, requestID, 30 ); + sleep(1); + ++retry; + } if ( returnStatus == API_SUCCESS) { //log the success event diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 038b8ab40..76e473488 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -302,7 +302,7 @@ public: /** *@brief reinit Processes trying to replace recycleProcess */ - void reinitProcesses(); + void reinitProcesses(std::string skipModule = "none"); /** *@brief recycle Processes */ From d88fd2b6d2fe431ebbaffcd2c70b8046000006f4 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 23 Apr 2020 10:42:37 -0500 Subject: [PATCH 83/84] Add comment describing why non-primary failover does a dbrm reload at start. --- procmgr/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 7de3a3d9f..dabf0c1c4 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -2156,7 +2156,8 @@ void pingDeviceThread() processManager.setSystemState(oam::BUSY_INIT); - //call dbrm control + // call for a reload in case cpimport was running and + // some cleanup is needed on dbrmcontroller thats active before continuing oam.dbrmctl("reload"); log.writeLog(__LINE__, "'dbrmctl reload' done", LOG_TYPE_DEBUG); From b131709e54cef3f4150406699c39c16f40feded8 Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Fri, 24 Apr 2020 13:33:13 -0500 Subject: [PATCH 84/84] restart DMLProc again after failover to fix locking issues on rollback --- procmgr/main.cpp | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index dabf0c1c4..f1f7bfb47 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -2486,8 +2486,37 @@ void pingDeviceThread() log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); //set recycle process - //processManager.recycleProcess(moduleName); - //processManager.reinitProcesses(); + // waiting until dml are ACTIVE + // disableModule is going to trigger DMLProc to restart wait for it + int retry = 0; + while (retry < 30) + { + ProcessStatus DMLprocessstatus; + + try + { + oam.getProcessStatus("DMLProc", config.moduleName(), DMLprocessstatus); + } + catch (exception& ex) + {} + catch (...) + {} + + if (DMLprocessstatus.ProcessOpState == oam::BUSY_INIT) + log.writeLog(__LINE__, "Waiting for DMLProc to finish rollback", LOG_TYPE_DEBUG); + + if (DMLprocessstatus.ProcessOpState == oam::ACTIVE) + break; + + if (DMLprocessstatus.ProcessOpState == oam::FAILED) + break; + + // wait some more + sleep(2); + ++retry; + } + // restart DMLProc again to retrigger rollback with all dbroots connected + processManager.restartProcessType("DMLProc"); //set query system state ready processManager.setQuerySystemState(true); }