From 5ff131e7264089b177d7dc9aaec4102efedae404 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 17 Jan 2020 10:31:05 -0800 Subject: [PATCH 01/22] Merge pull request #1004 from pleblanc1976/runtime-disable-oam Runtime disable oam --- cmake/configureEngine.cmake | 4 -- exemgr/main.cpp | 14 +++--- mcsconfig.h.in | 3 -- oam/oamcpp/liboamcpp.cpp | 15 +++--- oam/oamcpp/oamcache.cpp | 12 ++--- oamapps/mcsadmin/mcsadmin.cpp | 32 ++++++------- oamapps/postConfigure/installer.cpp | 23 ++++++--- oamapps/postConfigure/postConfigure.cpp | 8 ++-- .../we_redistributecontrolthread.cpp | 47 +++++++++---------- writeengine/splitter/we_sdhandler.cpp | 5 +- 10 files changed, 81 insertions(+), 82 deletions(-) diff --git a/cmake/configureEngine.cmake b/cmake/configureEngine.cmake index 0814ca454..7626f9edc 100644 --- a/cmake/configureEngine.cmake +++ b/cmake/configureEngine.cmake @@ -716,10 +716,6 @@ IF (NOT INLINE) SET (inline "") ENDIF() -IF($ENV{SKIP_OAM_INIT}) - set(SKIP_OAM_INIT 1 CACHE BOOL "Skip OAM initialization" FORCE) -ENDIF() - EXECUTE_PROCESS( COMMAND rm -f conftest.data conftest.file conftest.sym WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/exemgr/main.cpp b/exemgr/main.cpp index 94b9aabb7..3fa3a1c79 100644 --- a/exemgr/main.cpp +++ b/exemgr/main.cpp @@ -76,10 +76,7 @@ #include #include -#if defined(SKIP_OAM_INIT) #include "dbrm.h" -#endif - namespace { @@ -1646,11 +1643,12 @@ int main(int argc, char* argv[]) { } } -#if defined(SKIP_OAM_INIT) - BRM::DBRM *dbrm = new BRM::DBRM(); - dbrm->setSystemQueryReady(true); - delete dbrm; -#endif + if (getenv("SKIP_OAM_INIT")) + { + BRM::DBRM *dbrm = new BRM::DBRM(); + dbrm->setSystemQueryReady(true); + delete dbrm; + } threadpool::ThreadPool exeMgrThreadPool(serverThreads, 0); exeMgrThreadPool.setName("ExeMgrServer"); diff --git a/mcsconfig.h.in b/mcsconfig.h.in index 4ee8e9ee6..76f17917a 100644 --- a/mcsconfig.h.in +++ b/mcsconfig.h.in @@ -2,9 +2,6 @@ #ifndef TEST_MCSCONFIG_H #define TEST_MCSCONFIG_H -/* Define to 1 to let the system come up without using OAM */ -#cmakedefine SKIP_OAM_INIT 1 - /* Define to 1 if you have the `alarm' function. */ #cmakedefine HAVE_ALARM 1 diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index ca5132718..5fb72f387 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1255,7 +1255,7 @@ void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string passw bool storeHostnames) { // build and send msg - int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, + int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, password, mysqlpw); if (returnStatus != API_SUCCESS) @@ -1953,9 +1953,9 @@ void Oam::getProcessConfig(SystemProcessConfig& systemprocessconfig) Config* proConfig = Config::makeConfig(ProcessConfigFile.c_str()); Config *csConfig = Config::makeConfig(); string strStorageManagerEnabled = csConfig->getConfig("StorageManager", "Enabled"); - bool storageManagerEnabled = !strStorageManagerEnabled.empty() && (strStorageManagerEnabled[0] == 'Y' || + bool storageManagerEnabled = !strStorageManagerEnabled.empty() && (strStorageManagerEnabled[0] == 'Y' || strStorageManagerEnabled[0] == 'y' || strStorageManagerEnabled[0] == 'T' || strStorageManagerEnabled[0] == 't'); - + for (int processID = 1; processID < MAX_PROCESS + 1; processID++) { ProcessConfig processconfig; @@ -1969,7 +1969,7 @@ void Oam::getProcessConfig(SystemProcessConfig& systemprocessconfig) processconfig ); // hide StorageManager from everything else if it is disabled - if (processconfig.ProcessName.empty() || + if (processconfig.ProcessName.empty() || (!storageManagerEnabled && processconfig.ProcessName == "StorageManager")) continue; @@ -2370,7 +2370,8 @@ void Oam::setProcessStatus(const std::string process, const std::string module, void Oam::processInitComplete(std::string processName, int state) { //This method takes too long on Windows and doesn't do anything there anyway... -#if !defined(_MSC_VER) && !defined(SKIP_OAM_INIT) + if (getenv("SKIP_OAM_INIT") != NULL) + return; // get current Module name string moduleName; oamModuleInfo_t st; @@ -2414,7 +2415,6 @@ void Oam::processInitComplete(std::string processName, int state) writeLog("processInitComplete: Status update failed", LOG_TYPE_ERROR ); exceptionControl("processInitComplete", API_FAILURE); -#endif } /******************************************************************** @@ -10716,7 +10716,7 @@ void Oam::mountDBRoot(dbrootList dbrootConfigList, bool mount) // nothing to do here if (DBRootStorageType == "storagemanager") return; - + string DataRedundancyConfig = "n"; try @@ -11197,4 +11197,3 @@ namespace procheartbeat */ } // end of namespace // vim:ts=4 sw=4: - diff --git a/oam/oamcpp/oamcache.cpp b/oam/oamcpp/oamcache.cpp index ed5097fda..bd2957764 100644 --- a/oam/oamcpp/oamcache.cpp +++ b/oam/oamcpp/oamcache.cpp @@ -111,7 +111,7 @@ void OamCache::checkReload() // Restore for Windows when we support multiple PMs while (it != uniquePids.end()) { -#if !defined(SKIP_OAM_INIT) + if (getenv("SKIP_OAM_INIT") == NULL) { try { @@ -180,10 +180,11 @@ void OamCache::checkReload() break; } } -#else - pmToConnectionMap[*it] = i++; - moduleIds.push_back(*it); -#endif + else + { + pmToConnectionMap[*it] = i++; + moduleIds.push_back(*it); + } it++; } @@ -346,4 +347,3 @@ string OamCache::getModuleName() } } /* namespace oam */ - diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 0b9e500cc..e2970769e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -174,11 +174,12 @@ void setupSignalHandlers() int main(int argc, char* argv[]) { -// print a warning if this is a developer build -#ifdef SKIP_OAM_INIT - cout << "SKIP_OAM_INIT is set" << endl; - sleep(2); -#endif + // print a warning if this is a developer build + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set" << endl; + sleep(2); + } #ifndef _MSC_VER setuid(0); // set effective ID to root; ignore return status @@ -246,7 +247,7 @@ int main(int argc, char* argv[]) if (user != 0) rootUser = false; - + // create/open command log file if not created logFile.open(DEFAULT_LOG_FILE.c_str(), ios::app); @@ -2622,7 +2623,7 @@ int processCommand(string* arguments) if ( DBRootStorageType == "hdfs") { string logFile = tmpDir + "/cc-stop.pdsh"; - + cmd = "pdsh -a 'columnstore stop' > " + logFile + " 2>&1"; system(cmd.c_str()); @@ -4932,7 +4933,7 @@ int processCommand(string* arguments) case 39: // getSystemDirectories { cout << endl << "System Installation and Temporary File Directories" << endl << endl; - + cout << "System Temporary File Directory = " << tmpDir << endl << endl; } break; @@ -5695,7 +5696,7 @@ int processCommand(string* arguments) umStorageNames umstoragenames; int hostArg; int dbrootPerPM = 0; - + //check if module type or module name was entered if ( arguments[1].size() == 2 ) { @@ -5711,7 +5712,7 @@ int processCommand(string* arguments) moduleType = arguments[1]; moduleCount = atoi(arguments[2].c_str()); hostArg = 4; - + // MCOL-1607. Check whether we should store host names or IP addresses. if (arguments[3] != "" && (arguments[3][0] == 'y' || arguments[3][0] == 'Y')) storeHostnames = true; @@ -5746,7 +5747,7 @@ int processCommand(string* arguments) // MCOL-1607. Check whether we should store host names or IP addresses. if (arguments[2] != "" && (arguments[2][0] == 'y' || arguments[2][0] == 'Y')) storeHostnames = true; - + if (arguments[4] != "") password = arguments[4]; else @@ -6024,7 +6025,7 @@ int processCommand(string* arguments) // prompt for IP Address string prompt = "IP Address of " + hostName + " not found, enter IP Address or enter 'abort'"; IPAddress = dataPrompt(prompt); - + if ( IPAddress == "abort" || !oam.isValidIP(IPAddress) ) return 1; } @@ -6065,7 +6066,7 @@ int processCommand(string* arguments) DataRedundancyHostname = dataPrompt(prompt); if (storeHostnames) DataRedundancyIPAddress = DataRedundancyHostname; - else + else { DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); @@ -6074,7 +6075,7 @@ int processCommand(string* arguments) // prompt for IP Address string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; DataRedundancyIPAddress = dataPrompt(prompt); - + if (!oam.isValidIP(DataRedundancyIPAddress)) return 1; } @@ -7896,7 +7897,7 @@ int processCommand(string* arguments) gracefulTemp = oam::FORCEFUL; int returnStatus = oam.restartSystem(gracefulTemp, ackTemp); switch (returnStatus) - { + { case API_SUCCESS: if ( waitForActive() ) cout << endl << " Successful restart of System " << endl << endl; @@ -10034,4 +10035,3 @@ bool SendToWES(Oam& oam, ByteStream bs) return true; } // vim:ts=4 sw=4: - diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 3266bac8d..9a2c73b1c 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -183,14 +183,14 @@ int main(int argc, char* argv[]) if (p && *p) USER = p; - + string tmpDir = startup::StartUp::tmpDir(); // setup to start on reboot, for non-root amazon installs if ( !rootUser ) { system("sed -i -e 's/#runuser/runuser/g' /etc/rc.d/rc.local >/dev/null 2>&1"); - } + } //copy Columnstore.xml.rpmsave if upgrade option is selected if ( installType == "upgrade" ) @@ -694,7 +694,7 @@ int main(int argc, char* argv[]) cmd = "chmod 755 -R /var/lib/columnstore/data1/systemFiles/dbrm > /dev/null 2>&1"; system(cmd.c_str()); } - + string idbstartcmd = "columnstore start"; { @@ -711,6 +711,12 @@ int main(int argc, char* argv[]) mysqlSetup(); sleep(5); + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + exit(0); + } + //start on local module int rtnCode = system(idbstartcmd.c_str()); @@ -732,6 +738,12 @@ int main(int argc, char* argv[]) mysqlSetup(); sleep(5); + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + exit(0); + } + //startup mysqld and infinidb processes cout << endl; cmd = "clearShm > /dev/null 2>&1"; @@ -778,7 +790,7 @@ int main(int argc, char* argv[]) exit (1); } } - + cout << endl << "MariaDB ColumnStore Install Successfully Completed, System is Active" << endl << endl; cout << "Enter the following command to define MariaDB ColumnStore Alias Commands" << endl << endl; @@ -990,7 +1002,7 @@ bool updateProcessConfig(int serverTypeInstall) */ bool makeRClocal(string moduleName, int IserverTypeInstall) { - + return true; string moduleType = moduleName.substr(0, MAX_MODULE_TYPE_SIZE); @@ -1175,4 +1187,3 @@ bool uncommentCalpontXml( string entry) } // vim:ts=4 sw=4: - diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1cb97e199..b807876c3 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3730,9 +3730,11 @@ int main(int argc, char* argv[]) // startup MariaDB ColumnStore // - #ifdef SKIP_OAM_INIT - exit(0); - #endif + if (getenv("SKIP_OAM_INIT")) + { + cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + exit(0); + } if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || pmNumber > 1 ) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 1ce786a75..f67bf9430 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -618,35 +618,35 @@ int RedistributeControlThread::executeRedistributePlan() try { // skip system status check in case no OAM -#if !defined(_MSC_VER) && !defined(SKIP_OAM_INIT) - // make sure system is in active state - bool isActive = false; - - while (!isActive) + if (getenv("SKIP_OAM_INIT") == NULL) { - bool noExcept = true; - SystemStatus systemstatus; + // make sure system is in active state + bool isActive = false; - try + while (!isActive) { - fControl->fOam->getSystemStatus(systemstatus); - } - catch (const std::exception& ex) - { - fErrorMsg += ex.what(); - noExcept = false; - } - catch (...) - { - noExcept = false; - } + bool noExcept = true; + SystemStatus systemstatus; - if (noExcept && ((isActive = (systemstatus.SystemOpState == oam::ACTIVE)) == false)) - sleep(1);; + try + { + fControl->fOam->getSystemStatus(systemstatus); + } + catch (const std::exception& ex) + { + fErrorMsg += ex.what(); + noExcept = false; + } + catch (...) + { + noExcept = false; + } + + if (noExcept && ((isActive = (systemstatus.SystemOpState == oam::ACTIVE)) == false)) + sleep(1);; + } } -#endif - if (fStopAction) return RED_EC_USER_STOP; @@ -851,4 +851,3 @@ void RedistributeControlThread::doStopAction() } // namespace // vim:ts=4 sw=4: - diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index 47916d877..b23f61120 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -547,7 +547,7 @@ void WESDHandler::setup() } // getModuleStatus will take too long. Also to test in development -#if !defined(_MSC_VER) && !defined(SKIP_OAM_INIT) + if (getenv("SKIP_OAM_INIT") == NULL) { vector& aVec = fRef.fCmdArgs.getPmVec(); @@ -578,8 +578,6 @@ void WESDHandler::setup() } } } -#endif - int rtn = fDbrm.getSystemReady(); @@ -2997,4 +2995,3 @@ void WESDHandler::setDisconnectFailure(bool Flag) //------------------------------------------------------------------------------ } /* namespace WriteEngine */ - From 9cc1e1c4f895c886db9a9d788015f84775cb29c6 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Fri, 24 Jan 2020 09:26:13 -0600 Subject: [PATCH 02/22] Merge pull request #1006 from LinuxJedi/change-engine MCOL-128 Support ALTER TABLE...ENGINE=Columnstore --- dbcon/mysql/ha_mcs_ddl.cpp | 36 +++++++++++++++++++++++------------- dbcon/mysql/ha_mcs_impl.cpp | 5 ++++- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 43671055e..8454b8cce 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -2400,14 +2400,10 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea return 1; } - // @bug 3908. error out primary key for now. + // Send notice if primary key specified that it is not supported if (table_arg->key_info && table_arg->key_info->name.length && string(table_arg->key_info->name.str) == "PRIMARY") { - string emsg = logging::IDBErrorInfo::instance()->errorMsg(ERR_CONSTRAINTS); - setError(thd, ER_CHECK_NOT_IMPLEMENTED, emsg); - ci.alterTableState = cal_connection_info::NOT_ALTER; - ci.isAlter = false; - return 1; + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "INDEXES"); } int compressiontype = get_compression_type(thd); @@ -2452,22 +2448,25 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea return 1; } - // - // Check if this is a "CREATE TABLE ... LIKE " statement. + // Check if this is one of + // * "CREATE TABLE ... LIKE " + // * "ALTER TABLE ... ENGINE=Columnstore" + // * "CREATE TABLE ... AS ..." // If so generate a full create table statement using the properties of // the source table. Note that source table has to be a columnstore table and // we only check for currently supported options. // - if (thd->lex->create_info.like()) + if ((thd->lex->sql_command == SQLCOM_CREATE_TABLE && thd->lex->used_tables) || + (thd->lex->sql_command == SQLCOM_ALTER_TABLE && create_info->used_fields & HA_CREATE_USED_ENGINE) || + (thd->lex->create_info.like())) { TABLE_SHARE *share = table_arg->s; my_bitmap_map *old_map; // To save the read_set char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH]; String datatype, def_value; ostringstream oss; - string tbl_name (name+2); - std::replace(tbl_name.begin(), tbl_name.end(), '/', '.'); + string tbl_name = string(share->db.str) + "." + string(share->table_name.str); // Save the current read_set map and mark it for read old_map= tmp_use_all_columns(table_arg, table_arg->read_set); @@ -2505,10 +2504,18 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea // Process table level options + /* TODO: uncomment when we support AUTO_INCREMENT if (create_info->auto_increment_value > 1) { oss << " AUTO_INCREMENT=" << create_info->auto_increment_value; } + */ + + if (create_info->auto_increment_value > 1) + { + push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "AUTO INCREMENT"); + } + if (share->table_charset) { @@ -2539,7 +2546,7 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea oss << ";"; stmt = oss.str(); - + tmp_restore_column_map(table_arg->read_set, old_map); } @@ -2600,7 +2607,7 @@ int ha_mcs_impl_delete_table_(const char* db, const char* name, cal_connection_i string emsg; - if (thd->lex->sql_command == SQLCOM_DROP_DB) + if ((thd->lex->sql_command == SQLCOM_DROP_DB) || (thd->lex->sql_command == SQLCOM_ALTER_TABLE)) { std::string tableName(name); tableName.erase(0, tableName.rfind("/") + 1); @@ -2636,6 +2643,9 @@ int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_connection_i decode_file_path(to, decodedDbTo, decodedTblTo); string stmt; + // This case is already handled + if (thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE) + return 0; if (thd->slave_thread && !get_replication_slave(thd)) return 0; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index b89edddb7..e01d1be73 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2361,8 +2361,11 @@ int ha_mcs_impl_rnd_init(TABLE* table) } #endif - if ( (thd->lex)->sql_command == SQLCOM_ALTER_TABLE ) + // If ALTER TABLE and not ENGINE= we don't need rnd_init (gets us in a bad state) + if ((thd->lex->sql_command == SQLCOM_ALTER_TABLE) && !(thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE)) + { return 0; + } //Update and delete code if ( ((thd->lex)->sql_command == SQLCOM_UPDATE) || ((thd->lex)->sql_command == SQLCOM_DELETE) || ((thd->lex)->sql_command == SQLCOM_DELETE_MULTI) || ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI)) From ea3ced83afd0a35d8dc2e06c53d0ccf588152092 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 27 Jan 2020 12:56:04 +0000 Subject: [PATCH 03/22] Merge pull request #1011 from pleblanc1976/minor-fixes Minor fixes --- oamapps/postConfigure/installer.cpp | 8 ++++++-- oamapps/postConfigure/postConfigure.cpp | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 9a2c73b1c..cb94d7c23 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -713,7 +713,9 @@ int main(int argc, char* argv[]) if (getenv("SKIP_OAM_INIT")) { - cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + cout << "(2) SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + sysConfig->setConfig("Installation", "MySQLRep", "n"); + sysConfig->write(); exit(0); } @@ -740,7 +742,9 @@ int main(int argc, char* argv[]) if (getenv("SKIP_OAM_INIT")) { - cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + cout << "(3) SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + sysConfig->setConfig("Installation", "MySQLRep", "n"); + sysConfig->write(); exit(0); } diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index b807876c3..1ec9b023e 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -3732,7 +3732,9 @@ int main(int argc, char* argv[]) if (getenv("SKIP_OAM_INIT")) { - cout << "SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + cout << "(1) SKIP_OAM_INIT is set, so will not start ColumnStore or init the system catalog" << endl; + sysConfig->setConfig("Installation", "MySQLRep", "n"); + sysConfig->write(); exit(0); } From b72a9784ba3ec6b8b5d9367bd948dc682ecc8bf1 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 28 Jan 2020 15:24:27 +0000 Subject: [PATCH 04/22] Merge pull request #1014 from tntnatbry/MCOL-3675 MCOL-3675 and MCOL-3676 Detect in postConfigure if columnstore-post-install is run or not --- oamapps/postConfigure/postConfigure.cpp | 34 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 1ec9b023e..5f30a37eb 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -58,6 +58,7 @@ #include /* for strncpy */ #include +#include #include #include #include @@ -498,6 +499,31 @@ int main(int argc, char* argv[]) cout << "IMPORTANT: This tool requires to run on the Performance Module #1" << endl; cout << endl; + // MCOL-3675 + struct stat dir_info; + if (stat(tmpDir.c_str(), &dir_info) != 0) + { + cerr<getConfig(InstallSection, "ProfileFile"); + } + catch (...) + {} + + // MCOL-3676 + if (ProfileFile.empty()) + { + cerr<getConfig(InstallSection, "ProfileFile"); - } - catch (...) - {} - if ( waitForActive() ) { cout << " DONE" << endl; From 9e588039d52b9ef57e944f4334ec09a0bdc90905 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Tue, 28 Jan 2020 15:26:05 +0000 Subject: [PATCH 05/22] Merge pull request #1010 from tntnatbry/MCOL-3680 MCOL-3680 mysqld will abort if Columnstore.xml is missing. Conflicts: dbcon/mysql/ha_mcs.cpp --- dbcon/mysql/ha_mcs.cpp | 317 ++++++++++++++++++++++----- dbcon/mysql/ha_mcs.h | 2 +- dbcon/mysql/is_columnstore_files.cpp | 14 +- utils/configcpp/configcpp.cpp | 16 +- 4 files changed, 278 insertions(+), 71 deletions(-) diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index 58dba85c6..67b683e21 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -30,14 +30,14 @@ #define COLUMNSTORE_MATURITY MariaDB_PLUGIN_MATURITY_STABLE #endif -static handler* calpont_create_handler(handlerton* hton, - TABLE_SHARE* table, - MEM_ROOT* mem_root); +static handler* mcs_create_handler(handlerton* hton, + TABLE_SHARE* table, + MEM_ROOT* mem_root); -static int calpont_commit(handlerton* hton, THD* thd, bool all); +static int mcs_commit(handlerton* hton, THD* thd, bool all); -static int calpont_rollback(handlerton* hton, THD* thd, bool all); -static int calpont_close_connection ( handlerton* hton, THD* thd ); +static int mcs_rollback(handlerton* hton, THD* thd, bool all); +static int mcs_close_connection(handlerton* hton, THD* thd ); handlerton* mcs_hton; char cs_version[25]; char cs_commit_hash[41]; // a commit hash is 40 characters @@ -59,11 +59,11 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* sel); Hash used to track the number of open tables; variable for example share methods */ -static HASH calpont_open_tables; +static HASH mcs_open_tables; #ifndef _MSC_VER /* The mutex used to init the hash; variable for example share methods */ -pthread_mutex_t calpont_mutex; +pthread_mutex_t mcs_mutex; #endif #ifdef DEBUG_ENTER @@ -79,20 +79,20 @@ pthread_mutex_t calpont_mutex; Function we use in the creation of our hash to get key. */ -static uchar* calpont_get_key(COLUMNSTORE_SHARE* share, size_t* length, - my_bool not_used __attribute__((unused))) +static uchar* mcs_get_key(COLUMNSTORE_SHARE* share, size_t* length, + my_bool not_used __attribute__((unused))) { *length = share->table_name_length; return (uchar*) share->table_name; } // This one is unused -int calpont_discover(handlerton* hton, THD* thd, TABLE_SHARE* share) +int mcs_discover(handlerton* hton, THD* thd, TABLE_SHARE* share) { - DBUG_ENTER("calpont_discover"); - DBUG_PRINT("calpont_discover", ("db: '%s' name: '%s'", share->db.str, share->table_name.str)); + DBUG_ENTER("mcs_discover"); + DBUG_PRINT("mcs_discover", ("db: '%s' name: '%s'", share->db.str, share->table_name.str)); #ifdef INFINIDB_DEBUG - fprintf(stderr, "calpont_discover()\n"); + fprintf(stderr, "mcs_discover()\n"); #endif uchar* frm_data = NULL; @@ -114,8 +114,8 @@ int calpont_discover(handlerton* hton, THD* thd, TABLE_SHARE* share) } // This f() is also unused -int calpont_discover_existence(handlerton* hton, const char* db, - const char* table_name) +int mcs_discover_existence(handlerton* hton, const char* db, + const char* table_name) { return ha_mcs_impl_discover_existence(db, table_name); } @@ -144,18 +144,19 @@ static int columnstore_init_func(void* p) mcs_hton = (handlerton*)p; #ifndef _MSC_VER - (void) pthread_mutex_init(&calpont_mutex, MY_MUTEX_INIT_FAST); + (void) pthread_mutex_init(&mcs_mutex, MY_MUTEX_INIT_FAST); #endif - (void) my_hash_init(&calpont_open_tables, system_charset_info, 32, 0, 0, - (my_hash_get_key) calpont_get_key, 0, 0); + (void) my_hash_init(&mcs_open_tables, system_charset_info, 32, 0, 0, + (my_hash_get_key) mcs_get_key, 0, 0); - mcs_hton->create = calpont_create_handler; + mcs_hton->state = SHOW_OPTION_YES; + mcs_hton->create = mcs_create_handler; mcs_hton->flags = HTON_CAN_RECREATE; -// mcs_hton->discover_table= calpont_discover; -// mcs_hton->discover_table_existence= calpont_discover_existence; - mcs_hton->commit = calpont_commit; - mcs_hton->rollback = calpont_rollback; - mcs_hton->close_connection = calpont_close_connection; +// mcs_hton->discover_table = mcs_discover; +// mcs_hton->discover_table_existence = mcs_discover_existence; + mcs_hton->commit = mcs_commit; + mcs_hton->rollback = mcs_rollback; + mcs_hton->close_connection = mcs_close_connection; mcs_hton->create_group_by = create_columnstore_group_by_handler; mcs_hton->create_derived = create_columnstore_derived_handler; mcs_hton->create_select = create_columnstore_select_handler; @@ -165,37 +166,64 @@ static int columnstore_init_func(void* p) static int columnstore_done_func(void* p) { - DBUG_ENTER("calpont_done_func"); + DBUG_ENTER("columnstore_done_func"); - my_hash_free(&calpont_open_tables); + my_hash_free(&mcs_open_tables); #ifndef _MSC_VER - pthread_mutex_destroy(&calpont_mutex); + pthread_mutex_destroy(&mcs_mutex); #endif DBUG_RETURN(0); } -static handler* calpont_create_handler(handlerton* hton, - TABLE_SHARE* table, - MEM_ROOT* mem_root) +static handler* mcs_create_handler(handlerton* hton, + TABLE_SHARE* table, + MEM_ROOT* mem_root) { return new (mem_root) ha_mcs(hton, table); } -static int calpont_commit(handlerton* hton, THD* thd, bool all) +static int mcs_commit(handlerton* hton, THD* thd, bool all) { - int rc = ha_mcs_impl_commit( hton, thd, all); + int rc; + try + { + rc = ha_mcs_impl_commit(hton, thd, all); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } return rc; } -static int calpont_rollback(handlerton* hton, THD* thd, bool all) +static int mcs_rollback(handlerton* hton, THD* thd, bool all) { - int rc = ha_mcs_impl_rollback( hton, thd, all); + int rc; + try + { + rc = ha_mcs_impl_rollback(hton, thd, all); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } return rc; } -static int calpont_close_connection ( handlerton* hton, THD* thd ) +static int mcs_close_connection(handlerton* hton, THD* thd) { - int rc = ha_mcs_impl_close_connection( hton, thd); + int rc; + try + { + rc = ha_mcs_impl_close_connection(hton, thd); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } return rc; } @@ -256,7 +284,16 @@ int ha_mcs::open(const char* name, int mode, uint32_t test_if_locked) { DBUG_ENTER("ha_mcs::open"); - int rc = ha_mcs_impl_open(name, mode, test_if_locked); + int rc; + try + { + rc = ha_mcs_impl_open(name, mode, test_if_locked); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -282,7 +319,16 @@ int ha_mcs::close(void) { DBUG_ENTER("ha_mcs::close"); - int rc = ha_mcs_impl_close(); + int rc; + try + { + rc = ha_mcs_impl_close(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -303,7 +349,16 @@ int ha_mcs::close(void) int ha_mcs::write_row(const uchar* buf) { DBUG_ENTER("ha_mcs::write_row"); - int rc = ha_mcs_impl_write_row(buf, table); + int rc; + try + { + rc = ha_mcs_impl_write_row(buf, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -311,14 +366,30 @@ int ha_mcs::write_row(const uchar* buf) void ha_mcs::start_bulk_insert(ha_rows rows, uint flags) { DBUG_ENTER("ha_mcs::start_bulk_insert"); - ha_mcs_impl_start_bulk_insert(rows, table); + try + { + ha_mcs_impl_start_bulk_insert(rows, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + } DBUG_VOID_RETURN; } int ha_mcs::end_bulk_insert() { DBUG_ENTER("ha_mcs::end_bulk_insert"); - int rc = ha_mcs_impl_end_bulk_insert(false, table); + int rc; + try + { + rc = ha_mcs_impl_end_bulk_insert(false, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -339,7 +410,16 @@ int ha_mcs::update_row(const uchar* old_data, uchar* new_data) { DBUG_ENTER("ha_mcs::update_row"); - int rc = ha_mcs_impl_update_row(); + int rc; + try + { + rc = ha_mcs_impl_update_row(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -365,15 +445,33 @@ int ha_mcs::direct_update_rows_init(List *update_fields) int ha_mcs::direct_update_rows(ha_rows *update_rows) { DBUG_ENTER("ha_mcs::direct_update_rows"); - int rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } int ha_mcs::direct_update_rows(ha_rows *update_rows, ha_rows *found_rows) { DBUG_ENTER("ha_mcs::direct_update_rows"); - int rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); - *found_rows = *update_rows; + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(false, update_rows); + *found_rows = *update_rows; + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -386,7 +484,16 @@ int ha_mcs::direct_delete_rows_init() int ha_mcs::direct_delete_rows(ha_rows *deleted_rows) { DBUG_ENTER("ha_mcs::direct_delete_rows"); - int rc = ha_mcs_impl_direct_update_delete_rows(true, deleted_rows); + int rc; + try + { + rc = ha_mcs_impl_direct_update_delete_rows(true, deleted_rows); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } /** @@ -412,7 +519,16 @@ int ha_mcs::direct_delete_rows(ha_rows *deleted_rows) int ha_mcs::delete_row(const uchar* buf) { DBUG_ENTER("ha_mcs::delete_row"); - int rc = ha_mcs_impl_delete_row(); + int rc; + try + { + rc = ha_mcs_impl_delete_row(); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -511,7 +627,15 @@ int ha_mcs::rnd_init(bool scan) int rc = 0; if(scan) { - rc = ha_mcs_impl_rnd_init(table); + try + { + rc = ha_mcs_impl_rnd_init(table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } } DBUG_RETURN(rc); @@ -521,7 +645,16 @@ int ha_mcs::rnd_end() { DBUG_ENTER("ha_mcs::rnd_end"); - int rc = ha_mcs_impl_rnd_end(table); + int rc; + try + { + rc = ha_mcs_impl_rnd_end(table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -545,7 +678,16 @@ int ha_mcs::rnd_next(uchar* buf) { DBUG_ENTER("ha_mcs::rnd_next"); - int rc = ha_mcs_impl_rnd_next(buf, table); + int rc; + try + { + rc = ha_mcs_impl_rnd_next(buf, table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -601,7 +743,16 @@ void ha_mcs::position(const uchar* record) int ha_mcs::rnd_pos(uchar* buf, uchar* pos) { DBUG_ENTER("ha_mcs::rnd_pos"); - int rc = ha_mcs_impl_rnd_pos(buf, pos); + int rc; + try + { + rc = ha_mcs_impl_rnd_pos(buf, pos); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -754,11 +905,20 @@ int ha_mcs::external_lock(THD* thd, int lock_type) { DBUG_ENTER("ha_mcs::external_lock"); - //@Bug 2526 Only register the transaction when autocommit is off - if ((thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) - trans_register_ha( thd, true, mcs_hton); + int rc; + try + { + //@Bug 2526 Only register the transaction when autocommit is off + if ((thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) + trans_register_ha( thd, true, mcs_hton); - int rc = ha_mcs_impl_external_lock(thd, table, lock_type); + rc = ha_mcs_impl_external_lock(thd, table, lock_type); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -839,7 +999,17 @@ int ha_mcs::delete_table(const char* name) DBUG_ENTER("ha_mcs::delete_table"); /* This is not implemented but we want someone to be able that it works. */ - int rc = ha_mcs_impl_delete_table(name); + int rc; + + try + { + rc = ha_mcs_impl_delete_table(name); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -862,7 +1032,16 @@ int ha_mcs::delete_table(const char* name) int ha_mcs::rename_table(const char* from, const char* to) { DBUG_ENTER("ha_mcs::rename_table "); - int rc = ha_mcs_impl_rename_table(from, to); + int rc; + try + { + rc = ha_mcs_impl_rename_table(from, to); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } @@ -912,14 +1091,32 @@ int ha_mcs::create(const char* name, TABLE* table_arg, { DBUG_ENTER("ha_mcs::create"); - int rc = ha_mcs_impl_create(name, table_arg, create_info); + int rc; + try + { + rc = ha_mcs_impl_create(name, table_arg, create_info); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + rc = ER_INTERNAL_ERROR; + } DBUG_RETURN(rc); } const COND* ha_mcs::cond_push(const COND* cond) { DBUG_ENTER("ha_mcs::cond_push"); - DBUG_RETURN(ha_mcs_impl_cond_push(const_cast(cond), table)); + COND* ret_cond = NULL; + try + { + ret_cond = ha_mcs_impl_cond_push(const_cast(cond), table); + } + catch (std::runtime_error& e) + { + current_thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + } + DBUG_RETURN(ret_cond); } diff --git a/dbcon/mysql/ha_mcs.h b/dbcon/mysql/ha_mcs.h index cd29a165b..8baf31e7e 100644 --- a/dbcon/mysql/ha_mcs.h +++ b/dbcon/mysql/ha_mcs.h @@ -28,7 +28,7 @@ extern handlerton* mcs_hton; COLUMNSTORE_SHARE is a structure that will be shared among all open handlers. This example implements the minimum of what you will probably need. */ -typedef struct st_calpont_share +typedef struct st_mcs_share { char* table_name; uint32_t table_name_length, use_count; diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index 020583459..054d9b7d7 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -93,7 +93,19 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th char oidDirName[WriteEngine::FILE_NAME_SIZE]; char fullFileName[WriteEngine::FILE_NAME_SIZE]; char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE]; - config::Config* config = config::Config::makeConfig(); + + config::Config* config; + + try + { + config = config::Config::makeConfig(); + } + catch (std::runtime_error& e) + { + thd->raise_error_printf(ER_INTERNAL_ERROR, e.what()); + return ER_INTERNAL_ERROR; + } + WriteEngine::Config we_config; off_t fileSize = 0; off_t compressedFileSize = 0; diff --git a/utils/configcpp/configcpp.cpp b/utils/configcpp/configcpp.cpp index 1bea5c09b..823e6b2a6 100644 --- a/utils/configcpp/configcpp.cpp +++ b/utils/configcpp/configcpp.cpp @@ -114,19 +114,17 @@ Config* Config::makeConfig(const char* cf) Config::Config(const string& configFile) : fDoc(0), fConfigFile(configFile), fMtime(0), fParser() { - for ( int i = 0 ; i < 20 ; i++ ) + int i = 0; + for ( ; i < 2 ; i++ ) { - if (access(fConfigFile.c_str(), R_OK) != 0) - { - if ( i >= 15 ) - throw runtime_error("Config::Config: error accessing config file " + fConfigFile); - - sleep (1); - } - else + if (access(fConfigFile.c_str(), R_OK) == 0) break; + sleep (1); } + if ( i == 2 ) + throw runtime_error("Config::Config: error accessing config file " + fConfigFile); + struct stat statbuf; if (stat(configFile.c_str(), &statbuf) == 0) From 95f0caaf6356ea54b7ef6aeed138f97d03ae949d Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Tue, 28 Jan 2020 12:32:05 -0600 Subject: [PATCH 06/22] Merge pull request #1007 from LinuxJedi/MCOL-3707 MCOL-3707 Packaging pre/post functions --- CMakeLists.txt | 4 ++++ build/preUn_platform.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9794324ea..cab7bd282 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -406,6 +406,10 @@ IF (INSTALL_LAYOUT) else () SETA(CPACK_RPM_columnstore-platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "MariaDB-columnstore-libs" "snappy" "jemalloc" "net-tools" PARENT_SCOPE) endif() + + SET(CPACK_RPM_columnstore-platform_POST_INSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/build/postInstall_platform.sh PARENT_SCOPE) + SET(CPACK_RPM_columnstore-platform_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/build/preUn_platform.sh PARENT_SCOPE) + ENDIF () ELSE () # MariaDB has its own packaging routines diff --git a/build/preUn_platform.sh b/build/preUn_platform.sh index 3d5c2d200..a3cf9a653 100644 --- a/build/preUn_platform.sh +++ b/build/preUn_platform.sh @@ -1,3 +1,5 @@ + +mcsadmin shutdown y rpmmode=upgrade if [ "$1" -eq "$1" 2> /dev/null ]; then if [ $1 -ne 1 ]; then From 7a16877aae603e6321af281b1fcdfdab84cc474c Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Thu, 30 Jan 2020 14:29:30 -0600 Subject: [PATCH 07/22] Merge pull request #1016 from dhall-MariaDB/MCOL-3744 MCOL-3744 mcssystemready to work with SKIP_OAM_INIT --- dbcon/mysql/ha_mcs_client_udfs.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/dbcon/mysql/ha_mcs_client_udfs.cpp b/dbcon/mysql/ha_mcs_client_udfs.cpp index 51ce2917f..8395b364b 100644 --- a/dbcon/mysql/ha_mcs_client_udfs.cpp +++ b/dbcon/mysql/ha_mcs_client_udfs.cpp @@ -294,13 +294,24 @@ extern "C" try { - oam.getSystemStatus(systemstatus); - - if (systemstatus.SystemOpState == ACTIVE - && dbrm.getSystemReady() - && dbrm.getSystemQueryReady()) + if (getenv("SKIP_OAM_INIT")) { - return 1; + if (dbrm.getSystemReady() + && dbrm.getSystemQueryReady()) + { + return 1; + } + } + else + { + oam.getSystemStatus(systemstatus); + + if (systemstatus.SystemOpState == ACTIVE + && dbrm.getSystemReady() + && dbrm.getSystemQueryReady()) + { + return 1; + } } } catch (...) From 42adfb60e5307a6afc47768e5ab6238997f6fbd7 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 31 Jan 2020 12:30:17 -0500 Subject: [PATCH 08/22] Merge pull request #1017 from dhall-MariaDB/MCOL-3632 MCOL-3632 Restore error message for ref item type --- dbcon/mysql/ha_mcs_execplan.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 5b6118028..b64e85e42 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3306,6 +3306,11 @@ ReturnedColumn* buildReturnedColumn(Item* item, gp_walk_info& gwi, bool& nonSupp case Item::WINDOW_FUNC_ITEM: return buildWindowFunctionColumn(*(ref->ref), gwi, nonSupport); + case Item::SUBSELECT_ITEM: + gwi.fatalParseError = true; + gwi.parseErrorText = IDBErrorInfo::instance()->errorMsg(ERR_NON_SUPPORT_SELECT_SUB); + break; + default: gwi.fatalParseError = true; gwi.parseErrorText = "Unknown REF item"; From f27cb8a21f1c3aec4b5bdebb8d2fbb06fde7fef4 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 31 Jan 2020 13:43:28 -0500 Subject: [PATCH 09/22] Merge pull request #1012 from dhall-MariaDB/MCOL-3662 MCOL-3662 Restore error message for multi parameter aggregates save for UDAnF and GROUP_CONCAT --- dbcon/mysql/ha_mcs_execplan.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index b64e85e42..227b80a3b 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4564,10 +4564,9 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) if (gwi.clauseType == SELECT) gwi.aggOnSelect = true; - // N.B. argument_count() is the # of formal parms to the agg fcn. InifniDB only supports 1 argument - // TODO: Support more than one parm -#if 0 - + // Argument_count() is the # of formal parms to the agg fcn. Columnstore + // only supports 1 argument except UDAnF and GROUP_CONCAT + // TODO: Support more than one parm for COUNT(DISTINCT) if (isp->argument_count() != 1 && isp->sum_func() != Item_sum::GROUP_CONCAT_FUNC && isp->sum_func() != Item_sum::UDF_SUM_FUNC) { @@ -4576,7 +4575,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) return NULL; } -#endif AggregateColumn* ac = NULL; if (isp->sum_func() == Item_sum::GROUP_CONCAT_FUNC) From 0db52396f9f99be08b883f9b9c393c9710b28296 Mon Sep 17 00:00:00 2001 From: "David.Hall" Date: Tue, 4 Feb 2020 09:41:12 -0600 Subject: [PATCH 10/22] Merge pull request #1025 from mariadb-corporation/MCOL-3747 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MCOL-3747 This patch properly enables queries with IN + uncorrelated … --- dbcon/mysql/ha_mcs_pushdown.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index c0578ba52..efb319cd6 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -42,6 +42,26 @@ void disable_indices_for_CEJ(THD *thd_) } } +bool optimize_unflattened_subqueries_(SELECT_LEX *select_lex) +{ + bool result = false; + TABLE_LIST *tbl; + List_iterator_fast li(select_lex->leaf_tables); + while (!result && (tbl= li++)) + { + if (tbl->is_view_or_derived()) + { + SELECT_LEX *dsl = tbl->derived->first_select(); + result = optimize_unflattened_subqueries_(dsl); + } + } + + result = (!result) ? + select_lex->optimize_unflattened_subqueries(false) : true; + + return result; +} + void mutate_optimizer_flags(THD *thd_) { // MCOL-2178 Disable all optimizer flags as it was in the fork. @@ -775,7 +795,9 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) } COND *conds = simplify_joins_(join, select_lex->join_list, join->conds, TRUE, FALSE); - select_lex->optimize_unflattened_subqueries(false); + // MCOL-3747 IN-TO-EXISTS rewrite inside MDB didn't add + // an equi-JOIN condition. + optimize_unflattened_subqueries_(select_lex); if (conds) { From d6ef3cad3d05e9c3af8b79b4474c730478e36669 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Thu, 13 Feb 2020 18:02:03 -0600 Subject: [PATCH 11/22] Merge pull request #1049 from pleblanc1976/mcol-3776 Mcol 3776 - shared but unsync'd timezone var --- utils/funcexp/func_add_time.cpp | 2 +- utils/funcexp/func_between.cpp | 2 +- utils/funcexp/func_bitand.cpp | 2 +- utils/funcexp/func_bitwise.cpp | 22 +++++++++++----------- utils/funcexp/func_cast.cpp | 18 +++++++++--------- utils/funcexp/func_ceil.cpp | 4 ++-- utils/funcexp/func_char_length.cpp | 2 +- utils/funcexp/func_date.cpp | 2 +- utils/funcexp/func_date_add.cpp | 2 +- utils/funcexp/func_date_format.cpp | 4 ++-- utils/funcexp/func_day.cpp | 2 +- utils/funcexp/func_dayname.cpp | 2 +- utils/funcexp/func_dayofweek.cpp | 2 +- utils/funcexp/func_dayofyear.cpp | 2 +- utils/funcexp/func_extract.cpp | 2 +- utils/funcexp/func_floor.cpp | 4 ++-- utils/funcexp/func_hour.cpp | 2 +- utils/funcexp/func_if.cpp | 18 +++++++++--------- utils/funcexp/func_last_day.cpp | 2 +- utils/funcexp/func_math.cpp | 2 +- utils/funcexp/func_minute.cpp | 2 +- utils/funcexp/func_month.cpp | 2 +- utils/funcexp/func_monthname.cpp | 2 +- utils/funcexp/func_nullif.cpp | 2 +- utils/funcexp/func_quarter.cpp | 2 +- utils/funcexp/func_regexp.cpp | 2 +- utils/funcexp/func_round.cpp | 2 +- utils/funcexp/func_second.cpp | 2 +- utils/funcexp/func_str_to_date.cpp | 14 +++++++------- utils/funcexp/func_time.cpp | 2 +- utils/funcexp/func_time_format.cpp | 2 +- utils/funcexp/func_time_to_sec.cpp | 2 +- utils/funcexp/func_timediff.cpp | 6 +++--- utils/funcexp/func_timestampdiff.cpp | 4 ++-- utils/funcexp/func_to_days.cpp | 2 +- utils/funcexp/func_truncate.cpp | 2 +- utils/funcexp/func_week.cpp | 2 +- utils/funcexp/func_weekday.cpp | 2 +- utils/funcexp/func_year.cpp | 2 +- utils/funcexp/func_yearweek.cpp | 2 +- utils/funcexp/functor.cpp | 2 +- utils/funcexp/functor.h | 7 ++++++- 42 files changed, 85 insertions(+), 80 deletions(-) diff --git a/utils/funcexp/func_add_time.cpp b/utils/funcexp/func_add_time.cpp index a95eff53d..f5f501949 100644 --- a/utils/funcexp/func_add_time.cpp +++ b/utils/funcexp/func_add_time.cpp @@ -181,7 +181,7 @@ int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, TimeStamp timestamp(val1); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index 8330c4fc0..d742c2d16 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -344,7 +344,7 @@ CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, Cal if (cc) { Result result = cc->result(); - result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, fTimeZone); + result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone()); cc->result(result); } } diff --git a/utils/funcexp/func_bitand.cpp b/utils/funcexp/func_bitand.cpp index 64fa4078b..2863eadf8 100644 --- a/utils/funcexp/func_bitand.cpp +++ b/utils/funcexp/func_bitand.cpp @@ -162,7 +162,7 @@ int64_t Func_bitand::getIntVal(Row& row, TimeStamp timestamp(parm[i]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); values.push_back((m_time.month * 100000000000000) + (m_time.day * 1000000000000) + (m_time.hour * 10000000000) + (m_time.minute * 100000000) + (m_time.second * 1000000) + timestamp.msecond); } diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 028aedc1c..2154f125b 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -195,8 +195,8 @@ int64_t Func_bitand::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "bitand: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -231,8 +231,8 @@ int64_t Func_leftshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "leftshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -267,8 +267,8 @@ int64_t Func_rightshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "rightshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -303,8 +303,8 @@ int64_t Func_bitor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "bitor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -347,8 +347,8 @@ int64_t Func_bitxor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, fTimeZone) || - !getUIntValFromParm(row, parm[1], val2, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) { std::ostringstream oss; oss << "bitxor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); @@ -382,7 +382,7 @@ int64_t Func_bit_count::getIntVal(Row& row, uint64_t val = 0; - if (!getUIntValFromParm(row, parm[0], val, isNull, fTimeZone)) + if (!getUIntValFromParm(row, parm[0], val, isNull, timeZone())) { std::ostringstream oss; oss << "bit_count: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index d335ddc81..ce3d6a531 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -218,7 +218,7 @@ int64_t Func_cast_signed::getIntVal(Row& row, int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(fTimeZone); + return dt.convertToMySQLint(timeZone()); } break; @@ -379,7 +379,7 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, int64_t time = parm[0]->data()->getTimestampIntVal(row, isNull); TimeStamp dt(time); - return dt.convertToMySQLint(fTimeZone); + return dt.convertToMySQLint(timeZone()); } break; @@ -509,7 +509,7 @@ string Func_cast_char::getStrVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { - return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone).substr(0, length); + return dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()).substr(0, length); } break; @@ -676,7 +676,7 @@ int32_t Func_cast_date::getDateIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - string value = dataconvert::DataConvert::timestampToString(val1, fTimeZone); + string value = dataconvert::DataConvert::timestampToString(val1, timeZone()); value = value.substr(0, 10); return dataconvert::DataConvert::stringToDate(value); } @@ -803,7 +803,7 @@ int64_t Func_cast_date::getDatetimeIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -985,7 +985,7 @@ int64_t Func_cast_datetime::getDatetimeIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; @@ -1099,7 +1099,7 @@ int64_t Func_cast_datetime::getTimeIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); Time time; time.hour = m_time.hour; time.minute = m_time.minute; @@ -1478,7 +1478,7 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, { int32_t s = 0; - string value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + string value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); //strip off micro seconds string date = value.substr(0, 14); @@ -1599,7 +1599,7 @@ double Func_cast_double::getDoubleVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); // strip off micro seconds str = str.substr(0, 14); diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index b1f886166..f9f9c1891 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -164,7 +164,7 @@ int64_t Func_ceil::getIntVal(Row& row, TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); if (!isNull) - ret = dt.convertToMySQLint(fTimeZone); + ret = dt.convertToMySQLint(timeZone()); } break; @@ -269,7 +269,7 @@ uint64_t Func_ceil::getUintVal(Row& row, TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); if (!isNull) - ret = dt.convertToMySQLint(fTimeZone); + ret = dt.convertToMySQLint(timeZone()); } break; diff --git a/utils/funcexp/func_char_length.cpp b/utils/funcexp/func_char_length.cpp index c7c382c94..72bdfdfda 100644 --- a/utils/funcexp/func_char_length.cpp +++ b/utils/funcexp/func_char_length.cpp @@ -103,7 +103,7 @@ int64_t Func_char_length::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { - string date = dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + string date = dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); return (int64_t)date.size(); } diff --git a/utils/funcexp/func_date.cpp b/utils/funcexp/func_date.cpp index 7280e838a..76a50a90f 100644 --- a/utils/funcexp/func_date.cpp +++ b/utils/funcexp/func_date.cpp @@ -78,7 +78,7 @@ int64_t Func_date::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull); - value = dataconvert::DataConvert::timestampToString(val1, fTimeZone); + value = dataconvert::DataConvert::timestampToString(val1, timeZone()); value = value.substr(0, 10); break; } diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 5ed920e44..5004f5521 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -782,7 +782,7 @@ int64_t Func_date_add::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); DateTime dt; dt.year = m_time.year; dt.month = m_time.month; diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index e62bc3952..105069c59 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -275,7 +275,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, TimeStamp timestamp(val); int64_t seconds = timestamp.second; MySQLTime time; - gmtSecToMySQLTime(seconds, time, fTimeZone); + gmtSecToMySQLTime(seconds, time, timeZone()); dt.year = time.year; dt.month = time.month; dt.day = time.day; @@ -419,7 +419,7 @@ int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), fTimeZone); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); } diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index c8ba97e6c..c414fd5d4 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -67,7 +67,7 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.day; } diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index 02955fcf4..e46bfa46f 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -79,7 +79,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); year = time.year; month = time.month; day = time.day; diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index e66e1d864..3a29f592d 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -77,7 +77,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); year = time.year; month = time.month; day = time.day; diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index 6d4fc9fcf..ee13730c1 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -76,7 +76,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_extract.cpp b/utils/funcexp/func_extract.cpp index b8b1f90bc..f69b8dcbe 100644 --- a/utils/funcexp/func_extract.cpp +++ b/utils/funcexp/func_extract.cpp @@ -252,7 +252,7 @@ int64_t Func_extract::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); dataconvert::DateTime dt; dt.year = m_time.year; dt.month = m_time.month; diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index 50dd3d919..1e0ee7f09 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -161,7 +161,7 @@ int64_t Func_floor::getIntVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); // strip off micro seconds str = str.substr(0, 14); @@ -278,7 +278,7 @@ uint64_t Func_floor::getUintVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { string str = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); // strip off micro seconds str = str.substr(0, 14); diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 6959ce526..666e84ce0 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -114,7 +114,7 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.hour; } diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index b17ad7317..ef2827bbe 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -134,7 +134,7 @@ int64_t Func_if::getIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getIntVal(row, isNull); } @@ -151,7 +151,7 @@ string Func_if::getStrVal(Row& row, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getStrVal(row, isNull); } @@ -167,7 +167,7 @@ IDB_Decimal Func_if::getDecimalVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDecimalVal(row, isNull); } @@ -183,7 +183,7 @@ double Func_if::getDoubleVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDoubleVal(row, isNull); } @@ -198,7 +198,7 @@ long double Func_if::getLongDoubleVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getLongDoubleVal(row, isNull); } @@ -214,7 +214,7 @@ int32_t Func_if::getDateIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDateIntVal(row, isNull); } @@ -230,7 +230,7 @@ int64_t Func_if::getDatetimeIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getDatetimeIntVal(row, isNull); } @@ -246,7 +246,7 @@ int64_t Func_if::getTimestampIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getTimestampIntVal(row, isNull); } @@ -262,7 +262,7 @@ int64_t Func_if::getTimeIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, fTimeZone)) + if (boolVal(parm[0], row, timeZone())) { return parm[1]->data()->getTimeIntVal(row, isNull); } diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index c201898db..2099ea71a 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -77,7 +77,7 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 705a13620..13c738217 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1834,7 +1834,7 @@ string Func_format::getStrVal(Row& row, case execplan::CalpontSystemCatalog::TIMESTAMP: { - value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); } break; diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index 10b3f1ef6..6cdd109a6 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -114,7 +114,7 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.minute; } diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index 94708fb1b..ffb3f5fc1 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -66,7 +66,7 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.month; } diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 831d16586..7c27c9581 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -107,7 +107,7 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); return time.month; } diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index a81b439dc..3af8a7d4b 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -978,7 +978,7 @@ execplan::IDB_Decimal Func_nullif::getDecimalVal(rowgroup::Row& row, string value; if (parm[1]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), fTimeZone); + value = DataConvert::timestampToString1(parm[1]->data()->getTimestampIntVal(row, isNull), timeZone()); else value = DataConvert::datetimeToString1(parm[1]->data()->getDatetimeIntVal(row, isNull)); diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index e402444a7..6cc4316c4 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -70,7 +70,7 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); month = m_time.month; break; } diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 809cb239a..120a727e6 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -244,7 +244,7 @@ bool Func_regexp::getBoolVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return getBool(row, pm, isNull, ct, fTimeZone) && !isNull; + return getBool(row, pm, isNull, ct, timeZone()) && !isNull; } diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 293f94021..0064ae8df 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -490,7 +490,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, string value; if (op_ct.colDataType == execplan::CalpontSystemCatalog::TIMESTAMP) - value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + value = dataconvert::DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); else value = dataconvert::DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 4f12cfca8..45cca7e21 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -113,7 +113,7 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.second; } diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index 3432ba100..c4b0226f2 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -187,7 +187,7 @@ string Func_str_to_date::getStrVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); string convertedDate = dataconvert::DataConvert::datetimeToString(*((long long*) &dateTime)); return convertedDate; } @@ -198,7 +198,7 @@ int32_t Func_str_to_date::getDateIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return ((((int32_t)(time >> 32)) & 0xFFFFFFC0) | 0x3E); } @@ -209,7 +209,7 @@ int64_t Func_str_to_date::getDatetimeIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } @@ -220,7 +220,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); dataconvert::TimeStamp timestamp; dataconvert::MySQLTime m_time; m_time.year = dateTime.year; @@ -230,7 +230,7 @@ int64_t Func_str_to_date::getTimestampIntVal(rowgroup::Row& row, m_time.minute = dateTime.minute; m_time.second = dateTime.second; bool isValid = true; - int64_t seconds = mySQLTimeToGmtSec(m_time, fTimeZone, isValid); + int64_t seconds = mySQLTimeToGmtSec(m_time, timeZone(), isValid); if (!isValid) { timestamp = -1; @@ -252,7 +252,7 @@ int64_t Func_str_to_date::getTimeIntVal(rowgroup::Row& row, { dataconvert::DateTime dateTime; dataconvert::Time retTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); retTime.day = 0; retTime.is_neg = false; retTime.hour = dateTime.hour; @@ -269,7 +269,7 @@ int64_t Func_str_to_date::getIntVal(rowgroup::Row& row, CalpontSystemCatalog::ColType& ct) { dataconvert::DateTime dateTime; - dateTime = getDateTime(row, parm, isNull, ct, fTimeZone); + dateTime = getDateTime(row, parm, isNull, ct, timeZone()); int64_t time = *(reinterpret_cast(&dateTime)); return time; } diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index b83c678db..70202ac63 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -123,7 +123,7 @@ string Func_time::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); dataconvert::Time time; time.hour = m_time.hour; time.minute = m_time.minute; diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index 08739b046..5dd2f666a 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -77,7 +77,7 @@ string Func_time_format::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); hour = m_time.hour; min = m_time.minute; sec = m_time.second; diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index f8391d12f..e19ce2f81 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -76,7 +76,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(val); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); hour = time.hour; min = time.minute; sec = time.second; diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 881c04017..5bf16e413 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -152,7 +152,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -239,7 +239,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(temp); int64_t seconds = timestamp.second; dataconvert::MySQLTime time; - dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, time, timeZone()); dataconvert::DateTime dt; dt.year = time.year; dt.month = time.month; @@ -332,7 +332,7 @@ int64_t Func_timediff::getTimestampIntVal(rowgroup::Row& row, bool& isNull, CalpontSystemCatalog::ColType& ct) { - return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), fTimeZone); + return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone()); } int64_t Func_timediff::getTimeIntVal(rowgroup::Row& row, diff --git a/utils/funcexp/func_timestampdiff.cpp b/utils/funcexp/func_timestampdiff.cpp index 60b18cdc7..3972554d3 100644 --- a/utils/funcexp/func_timestampdiff.cpp +++ b/utils/funcexp/func_timestampdiff.cpp @@ -57,7 +57,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); dt1.year = m_time.year; dt1.month = m_time.month; dt1.day = m_time.day; @@ -84,7 +84,7 @@ int64_t Func_timestampdiff::getIntVal(rowgroup::Row& row, TimeStamp timestamp(parm[1]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; MySQLTime m_time; - gmtSecToMySQLTime(seconds, m_time, fTimeZone); + gmtSecToMySQLTime(seconds, m_time, timeZone()); dt2.year = m_time.year; dt2.month = m_time.month; dt2.day = m_time.day; diff --git a/utils/funcexp/func_to_days.cpp b/utils/funcexp/func_to_days.cpp index b51eb5aff..26a993d1a 100644 --- a/utils/funcexp/func_to_days.cpp +++ b/utils/funcexp/func_to_days.cpp @@ -90,7 +90,7 @@ int64_t Func_to_days::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 441047f74..057a1c299 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -497,7 +497,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, int64_t x = 0; string value = - DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), fTimeZone); + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); s = parm[1]->data()->getIntVal(row, isNull); diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index 326f1c13a..af024cfe4 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -80,7 +80,7 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index 9b6cc8c63..04ca938b4 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -76,7 +76,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 94abdd79a..2c5ec41f9 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -66,7 +66,7 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); return m_time.year; } diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 3ae72a7f2..91bb60a8c 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -83,7 +83,7 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; dataconvert::MySQLTime m_time; - dataconvert::gmtSecToMySQLTime(seconds, m_time, fTimeZone); + dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; day = m_time.day; diff --git a/utils/funcexp/functor.cpp b/utils/funcexp/functor.cpp index 06999c9fd..7fb55b47e 100644 --- a/utils/funcexp/functor.cpp +++ b/utils/funcexp/functor.cpp @@ -112,7 +112,7 @@ uint64_t Func::stringToDatetime(const string str) uint64_t Func::stringToTimestamp(const string str) { - int64_t ret = DataConvert::stringToTimestamp(str, fTimeZone); + int64_t ret = DataConvert::stringToTimestamp(str, timeZone()); if (ret == -1) { diff --git a/utils/funcexp/functor.h b/utils/funcexp/functor.h index 357d8639d..12dadc83a 100644 --- a/utils/funcexp/functor.h +++ b/utils/funcexp/functor.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "parsetree.h" #include "exceptclasses.h" @@ -73,10 +74,12 @@ public: const std::string timeZone() const { + std::unique_lock l(tzMutex); return fTimeZone; } void timeZone(const std::string timeZone) { + std::unique_lock l(tzMutex); fTimeZone = timeZone; } @@ -201,7 +204,6 @@ protected: virtual int64_t addTime(dataconvert::Time& dt1, dataconvert::Time& dt2); std::string fFuncName; - std::string fTimeZone; private: //defaults okay @@ -213,6 +215,9 @@ private: float fFloatNullVal; double fDoubleNullVal; long double fLongDoubleNullVal; + + std::string fTimeZone; + mutable std::mutex tzMutex; }; From bfe3ab14c0072ca84e409c84e4225b037a547e65 Mon Sep 17 00:00:00 2001 From: "David.Hall" Date: Thu, 6 Feb 2020 09:32:26 -0600 Subject: [PATCH 12/22] Merge pull request #1030 from LinuxJedi/MCOL-3764 MCOL-3764 Fix RENAME TABLE --- dbcon/ddlpackage/ddl.l | 2 +- dbcon/mysql/ha_mcs_ddl.cpp | 87 ++++++++++++-------------------------- 2 files changed, 27 insertions(+), 62 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index c644bddf0..4cef8182e 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -70,7 +70,7 @@ self [,()\[\].;\:\+\-\*\/\%\^\<\>\=] whitespace ({space}+|{comment}) digit [0-9] -ident_start [A-Za-z\200-\377_0-9] +ident_start [A-Za-z\200-\377_0-9#] ident_cont [A-Za-z\200-\377_0-9\$] identifier {ident_start}{ident_cont}* extended_identifier {ident_start}{extended_ident_cont}* diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 8454b8cce..05a3717cb 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -390,32 +390,6 @@ bool validateNextValue( int type, int64_t value ) return validValue; } -static void decode_objectname(char *buf, const char *path, size_t buf_size) -{ - size_t new_path_len = filename_to_tablename(path, buf, buf_size); - buf[new_path_len] = '\0'; -} - -static void decode_file_path(const char *path, char *decoded_dbname, - char *decoded_tbname) -{ - // The format cont ains './' in the beginning of a path. - char *dbname_start = (char*) path + 2; - char *dbname_end = dbname_start; - while (*dbname_end != '/') - dbname_end++; - - int cnt = dbname_end - dbname_start; - char *dbname = (char *)my_alloca(cnt + 1); - memcpy(dbname, dbname_start, cnt); - dbname[cnt] = '\0'; - decode_objectname(decoded_dbname, dbname, FN_REFLEN); - my_afree(dbname); - - char *tbname_start = dbname_end + 1; - decode_objectname(decoded_tbname, tbname_start, FN_REFLEN); -} - bool anyRowInTable(string& schema, string& tableName, int sessionID) { //find a column in the table @@ -914,18 +888,6 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl { CreateTableStatement* createTable = dynamic_cast ( &stmt ); - //@Bug 5767. To handle key words inside `` for a tablename. - if (!(boost::iequals(schema, createTable->fTableDef->fQualifiedName->fSchema)) || !(boost::iequals(table, createTable->fTableDef->fQualifiedName->fName))) - { - rc = 1; - thd->get_stmt_da()->set_overwrite_status(true); - - thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, (IDBErrorInfo::instance()->errorMsg(ERR_CREATE_DATATYPE_NOT_SUPPORT)).c_str()); - ci->alterTableState = cal_connection_info::NOT_ALTER; - ci->isAlter = false; - return rc; - } - bool matchedCol = false; bool isFirstTimestamp = true; @@ -2466,7 +2428,7 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea char datatype_buf[MAX_FIELD_WIDTH], def_value_buf[MAX_FIELD_WIDTH]; String datatype, def_value; ostringstream oss; - string tbl_name = string(share->db.str) + "." + string(share->table_name.str); + string tbl_name = string(share->db.str) + "." + string(share->table_name.str); // Save the current read_set map and mark it for read old_map= tmp_use_all_columns(table_arg, table_arg->read_set); @@ -2634,22 +2596,28 @@ int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_connection_i THD* thd = current_thd; string emsg; - string dbFrom, tblFrom, dbTo, tblTo; - char decodedDbFrom[FN_REFLEN]; - char decodedTblFrom[FN_REFLEN]; - char decodedDbTo[FN_REFLEN]; - char decodedTblTo[FN_REFLEN]; - decode_file_path(from, decodedDbFrom, decodedTblFrom); - decode_file_path(to, decodedDbTo, decodedTblTo); - string stmt; + string tblFrom (from+2); + size_t pos = tblFrom.find("/"); + std::string dbFrom = tblFrom.substr(0, pos); + tblFrom = tblFrom.erase(0, pos + 1); - // This case is already handled - if (thd->lex->create_info.used_fields & HA_CREATE_USED_ENGINE) - return 0; + string tblTo (to+2); + pos = tblTo.find("/"); + std::string dbTo = tblTo.substr(0, pos); + tblTo = tblTo.erase(0, pos + 1); + + string stmt; if (thd->slave_thread && !get_replication_slave(thd)) return 0; + // This is a temporary table rename, we don't use the temporary table name + // so this is a NULL op + if (tblFrom.compare(0, 4, "#sql") == 0) + { + return 0; + } + //@bug 5660. Error out REAL DDL/DML on slave node. // When the statement gets here, it's NOT SSO or RESTRICT if (ci.isSlaveNode) @@ -2659,24 +2627,21 @@ int ha_mcs_impl_rename_table_(const char* from, const char* to, cal_connection_i return 1; } - // User moves tables b/w namespaces. - size_t tblFromLength= strlen(decodedTblFrom); - if (!memcmp(decodedTblFrom, decodedTblTo, tblFromLength)) - { - return 0; - } - - stmt.assign("alter table `"); - stmt.append(decodedTblFrom); + stmt.assign("alter table `"); + stmt.append(dbFrom); + stmt.append("`.`"); + stmt.append(tblFrom); stmt.append("` rename to `"); - stmt.append(decodedTblTo); + stmt.append(dbTo); + stmt.append("`.`"); + stmt.append(tblTo); stmt.append("`;"); string db; if (thd->db.length) db = thd->db.str; else - db.assign(decodedDbFrom); + db.assign(dbFrom); int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg); From f71158601e6a4758c2f354bc8aea4b16213c216c Mon Sep 17 00:00:00 2001 From: "David.Hall" Date: Mon, 10 Feb 2020 10:08:38 -0600 Subject: [PATCH 13/22] Merge pull request #1032 from drrtuy/MCOL-3766 MCOL-3766 EXPLAIN now doesn't execute query via SH thus doesn't fail --- dbcon/mysql/ha_mcs_pushdown.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs_pushdown.cpp b/dbcon/mysql/ha_mcs_pushdown.cpp index efb319cd6..da7c948bd 100644 --- a/dbcon/mysql/ha_mcs_pushdown.cpp +++ b/dbcon/mysql/ha_mcs_pushdown.cpp @@ -839,8 +839,13 @@ create_columnstore_select_handler(THD* thd, SELECT_LEX* select_lex) { handler= new ha_columnstore_select_handler(thd, select_lex); mcs_handler_info mhi= mcs_handler_info(reinterpret_cast(handler), SELECT); - // this::table is the place for the result set - int rc= ha_cs_impl_pushdown_init(&mhi, handler->table); + // handler::table is the place for the result set + int rc= 0; + // Skip execution for EXPLAIN queries + if (!thd->lex->describe) + { + rc= ha_cs_impl_pushdown_init(&mhi, handler->table); + } // Return SH even if init fails b/c CS changed SELECT_LEX structures // with simplify_joins_() From 3faa1600c34f5af0ba5227f04b7e521afdd14c87 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 10 Feb 2020 11:57:33 -0500 Subject: [PATCH 14/22] Merge pull request #1031 from pleblanc1976/we-splitter-read-from-s3 MCOL-3520: Fix importing files from S3 for mode 1 imports. --- writeengine/splitter/CMakeLists.txt | 4 +- writeengine/splitter/we_cmdargs.cpp | 13 +- writeengine/splitter/we_cmdargs.h | 52 +++++--- writeengine/splitter/we_filereadthread.cpp | 142 ++++++++++++++++----- writeengine/splitter/we_filereadthread.h | 18 +++ writeengine/splitter/we_sdhandler.cpp | 34 +---- writeengine/splitter/we_sdhandler.h | 2 +- 7 files changed, 186 insertions(+), 79 deletions(-) diff --git a/writeengine/splitter/CMakeLists.txt b/writeengine/splitter/CMakeLists.txt index c02645534..1ab3f4b20 100644 --- a/writeengine/splitter/CMakeLists.txt +++ b/writeengine/splitter/CMakeLists.txt @@ -1,5 +1,5 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ) +include_directories( ${ENGINE_COMMON_INCLUDES} ${S3API_DIR} ) ########### next target ############### @@ -17,7 +17,7 @@ set(cpimport_SRCS add_executable(cpimport ${cpimport_SRCS}) -target_link_libraries(cpimport ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} batchloader threadpool) +target_link_libraries(cpimport ${ENGINE_LDFLAGS} ${NETSNMP_LIBRARIES} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} batchloader threadpool marias3) install(TARGETS cpimport DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index f10dc690c..eafa6e559 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -217,10 +217,10 @@ std::string WECmdArgs::getCpImportCmdLine() if (fbTruncationAsError) aSS << " -S "; - if (!fS3Key.empty()) + if (!fS3Key.empty() && !(fMode == 0 || fMode == 1)) { if (fS3Secret.empty() || fS3Bucket.empty() || fS3Region.empty()) - throw (runtime_error("Not all requried S3 options provided")); + throw (runtime_error("Not all required S3 options provided")); aSS << " -y " << fS3Key; aSS << " -K " << fS3Secret; aSS << " -t " << fS3Bucket; @@ -432,6 +432,8 @@ bool WECmdArgs::checkForCornerCases() throw(runtime_error("Mode 2 require remote file opts -f and -l or "\ "a fully qualified path for the remote file." "\nTry 'cpimport -h' for more information.")); + if (!fS3Key.empty()) + throw(runtime_error("Mode 2 & an input file from S3 does not make sense.")); } if (fMode == 3) @@ -1233,7 +1235,12 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv) throw (runtime_error("No schema or local filename specified.")); } } - + + /* check for all-or-nothing cmdline args to enable S3 import */ + int s3Tmp = (fS3Key.empty() ? 0 : 1) + (fS3Bucket.empty() ? 0 : 1) + + (fS3Secret.empty() ? 0 : 1) + (fS3Region.empty() ? 0 : 1); + if (s3Tmp != 0 && s3Tmp != 4) + throw runtime_error("The access key, secret, bucket, and region are all required to import from S3"); } std::string WECmdArgs::getJobFileName() diff --git a/writeengine/splitter/we_cmdargs.h b/writeengine/splitter/we_cmdargs.h index d02600070..8c2b89c97 100644 --- a/writeengine/splitter/we_cmdargs.h +++ b/writeengine/splitter/we_cmdargs.h @@ -37,7 +37,10 @@ class WECmdArgs public: WECmdArgs(int argc, char** argv); virtual ~WECmdArgs() {} -public: + + typedef std::vector VecInts; + typedef std::vector VecArgs; + void appTestFunction(); void parseCmdLineArgs(int argc, char** argv); std::string getCpImportCmdLine(); @@ -54,7 +57,6 @@ public: char* pBuff, int FileIdx); void updateWithJobFile(int Idx); -public: std::string getJobFileName(); std::string getBrmRptFileName(); std::string getTmpFileDir(); @@ -195,19 +197,6 @@ public: return fTimeZone; } - -private: // variables for SplitterApp - typedef std::vector VecArgs; - VecArgs fVecArgs; - typedef std::vector VecInts; - VecInts fPmVec; - - VecArgs fVecJobFiles; //JobFiles splitter from master JobFile - int fMultiTableCount; //MultiTable count - VecArgs fColFldsFromJobFile;//List of columns from any job file, that - // represent fields in the import data - -public: bool getPmStatus(int Id); bool str2PmList(std::string& PmList, VecInts& V); int getPmVecSize() @@ -252,11 +241,44 @@ public: { fMultiTableCount = Count; } + + bool isS3Import() const + { + return !fS3Key.empty(); + } + std::string getS3Key() const + { + return fS3Key; + } + std::string getS3Bucket() const + { + return fS3Bucket; + } + std::string getS3Host() const + { + return fS3Host; + } + std::string getS3Secret() const + { + return fS3Secret; + } + std::string getS3Region() const + { + return fS3Region; + } std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342 void getColumnList( std::set& columnList ) const; private: // variables for SplitterApp + VecArgs fVecArgs; + VecInts fPmVec; + + VecArgs fVecJobFiles; //JobFiles splitter from master JobFile + int fMultiTableCount; //MultiTable count + VecArgs fColFldsFromJobFile;//List of columns from any job file, that + // represent fields in the import data + std::string fJobId; // JobID std::string fOrigJobId; // Original JobID, in case we have to split it bool fJobLogOnly; // Job number is only for log filename only diff --git a/writeengine/splitter/we_filereadthread.cpp b/writeengine/splitter/we_filereadthread.cpp index 2d7fb4731..f6aa8f3b1 100644 --- a/writeengine/splitter/we_filereadthread.cpp +++ b/writeengine/splitter/we_filereadthread.cpp @@ -30,6 +30,7 @@ #include "we_messages.h" #include "we_sdhandler.h" +#include "we_splitterapp.h" #include #include @@ -99,6 +100,8 @@ WEFileReadThread::WEFileReadThread(WESDHandler& aSdh): fSdh(aSdh), fBuff = new char [fBuffSize]; + const WECmdArgs &args = fSdh.fRef.fCmdArgs; + initS3Connection(args); } //WEFileReadThread::WEFileReadThread(const WEFileReadThread& rhs):fSdh(rhs.fSdh) @@ -122,6 +125,18 @@ WEFileReadThread::~WEFileReadThread() fpThread = 0; delete []fBuff; //cout << "WEFileReadThread destructor called" << endl; + + if (doS3Import) + { + ms3_deinit(s3Connection); + ms3_library_deinit(); + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + } + } } //------------------------------------------------------------------------------ @@ -141,6 +156,14 @@ void WEFileReadThread::reset() fpThread = 0; //cout << "WEFileReadThread destructor called" << endl; this->setContinue(true); + + if (buf) + { + arrSource.reset(); + s3Stream.reset(); + free(buf); + buf = NULL; + } } //------------------------------------------------------------------------------ @@ -200,37 +223,33 @@ bool WEFileReadThread::chkForListOfFiles(std::string& FileName) std::string aFileName = FileName; istringstream iss(aFileName); + ostringstream oss; size_t start = 0, end = 0; const char* sep = " ,|"; - - end = aFileName.find_first_of(sep); + ms3_status_st ms3status; do { - if (end != string::npos) - { - std::string aFile = aFileName.substr(start, end - start); - - if (fSdh.getDebugLvl() > 2) - cout << "File: " << aFileName.substr(start, end - start) << endl; - - start = end + 1; - fInfileList.push_back(aFile); - } - else - { - std::string aFile = aFileName.substr(start, end - start); - - if (fSdh.getDebugLvl() > 1) - cout << "Next Input File " << aFileName.substr(start, end - start) << endl; - - fInfileList.push_back(aFile); - break; - } - end = aFileName.find_first_of(sep, start); + std::string aFile = aFileName.substr(start, end - start); + if (aFile == "STDIN" || aFile == "stdin") + aFile = "/dev/stdin"; + + if (fSdh.getDebugLvl() > 1) + cout << "Next Input File " << aFile << endl; + + if ((!doS3Import && access(aFile.c_str(), O_RDONLY) != 0) || + (doS3Import && ms3_status(s3Connection, s3Bucket.c_str(), + aFile.c_str(), &ms3status) != 0)) + { + oss << "Could not access " << aFile; + throw runtime_error(oss.str()); + } + + fInfileList.push_back(aFile); + start = end + 1; } - while (start != end); + while (end != string::npos); //cout << "Going out chkForListOfFiles("<< FileName << ")" << endl; @@ -267,6 +286,13 @@ void WEFileReadThread::shutdown() //if(fInFile.is_open()) fInFile.close(); //@BUG 4326 if (fIfFile.is_open()) fIfFile.close(); + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + buf = NULL; + } } //------------------------------------------------------------------------------ @@ -451,9 +477,48 @@ void WEFileReadThread::openInFile() { try { - if (fSdh.getDebugLvl()) cout << "Input FileName: " << fInFileName << endl; + /* If an S3 transfer + use ms3 lib to d/l data into mem + use boost::iostreams to wrap the mem in a stream interface + point infile's stream buffer to it. + */ + + if (fSdh.getDebugLvl()) cout << "Input Filename: " << fInFileName << endl; - if (fInFileName == "/dev/stdin") + if (doS3Import) + { + size_t bufLen = 0; + if (buf) + { + s3Stream.reset(); + arrSource.reset(); + free(buf); + buf = NULL; + } + if (fSdh.getDebugLvl()) + cout << "Downloading " << fInFileName << endl; + int err = ms3_get(s3Connection, s3Bucket.c_str(), fInFileName.c_str(), + &buf, &bufLen); + if (fSdh.getDebugLvl()) + cout << "Download complete." << endl; + if (err) + { + ostringstream os; + if (ms3_server_error(s3Connection)) + os << "Download of '" << fInFileName << "' failed. Error from the server: " + << ms3_server_error(s3Connection); + else + os << "Download of '" << fInFileName << "' failed. Got '" << ms3_error(err) + << "'."; + throw runtime_error(os.str()); + } + + arrSource.reset(new boost::iostreams::array_source((char *) buf, bufLen)); + s3Stream.reset(new boost::iostreams::stream(*arrSource)); + fInFile.rdbuf(s3Stream->rdbuf()); + } + + else if (fInFileName == "/dev/stdin") { char aDefCon[16], aGreenCol[16]; snprintf(aDefCon, sizeof(aDefCon), "\033[0m"); @@ -463,12 +528,11 @@ void WEFileReadThread::openInFile() cout << aGreenCol << "trying to read from STDIN... " << aDefCon << endl; + fInFile.rdbuf(cin.rdbuf()); } - cout.flush(); - //@BUG 4326 - if (fInFileName != "/dev/stdin") + else if (fInFileName != "/dev/stdin") { if (!fIfFile.is_open()) { @@ -592,6 +656,26 @@ int WEFileReadThread::getNextRow(istream& ifs, char* pBuf, int MaxLen) return pEnd - pBuf; } +void WEFileReadThread::initS3Connection(const WECmdArgs &args) +{ + doS3Import = args.isS3Import(); + if (doS3Import) + { + s3Key = args.getS3Key(); + s3Secret = args.getS3Secret(); + s3Bucket = args.getS3Bucket(); + s3Region = args.getS3Region(); + s3Host = args.getS3Host(); + ms3_library_init(); + s3Connection = ms3_init(s3Key.c_str(), s3Secret.c_str(), s3Region.c_str(), (s3Host.empty() ? NULL : s3Host.c_str())); + if (!s3Connection) + throw runtime_error("failed to get an S3 connection"); + } + else + s3Connection = NULL; + buf = NULL; +} + //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_filereadthread.h b/writeengine/splitter/we_filereadthread.h index 2fb92332c..f81f5d492 100644 --- a/writeengine/splitter/we_filereadthread.h +++ b/writeengine/splitter/we_filereadthread.h @@ -30,6 +30,11 @@ #ifndef WE_FILEREADTHREAD_H_ #define WE_FILEREADTHREAD_H_ +#include "we_cmdargs.h" +#include "libmarias3/marias3.h" +#include +#include + namespace WriteEngine { @@ -153,6 +158,19 @@ private: char fDelim; // Column Delimit char char* fBuff; // main data buffer int fBuffSize; + + /* To support mode 1 imports from objects on S3 */ + void initS3Connection(const WECmdArgs &); + bool doS3Import; + std::string s3Key; + std::string s3Secret; + std::string s3Bucket; + std::string s3Region; + std::string s3Host; + ms3_st *s3Connection; + uint8_t *buf; + std::unique_ptr arrSource; + std::unique_ptr > s3Stream; }; } /* namespace WriteEngine */ diff --git a/writeengine/splitter/we_sdhandler.cpp b/writeengine/splitter/we_sdhandler.cpp index b23f61120..a819dbda2 100644 --- a/writeengine/splitter/we_sdhandler.cpp +++ b/writeengine/splitter/we_sdhandler.cpp @@ -501,14 +501,9 @@ void WESDHandler::setup() //fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), false); fLog.setLogFileName(aLogName.c_str(), aErrLogName.c_str(), getConsoleLog()); - // In mode 0 and Mode 1, we need to check for local file availability - if ((0 == fRef.fCmdArgs.getMode()) || (1 == fRef.fCmdArgs.getMode())) - { - if (!check4InputFile(fRef.getLocFile())) - { - throw (runtime_error("Could not open Input file " + fRef.getLocFile())); - } - } + // In mode 0 and Mode 1, we need to construct the input file list and check availability + if (0 == fRef.fCmdArgs.getMode() || 1 == fRef.fCmdArgs.getMode()) + setInputFileList(fRef.getLocFile()); fImportRslt.startTimer(); @@ -2726,28 +2721,9 @@ std::string WESDHandler::getTime2Str() const //------------------------------------------------------------------------------ -bool WESDHandler::check4InputFile(std::string InFileName) +void WESDHandler::setInputFileList(std::string InFileName) { - bool aRet = false; - - if ((0 == InFileName.compare("STDIN")) || (0 == InFileName.compare("stdin"))) - { - fFileReadThread.add2InputDataFileList(InFileName); - return true; - } - else - { - //BUG 4342 - Need to support "list of infiles" - fFileReadThread.chkForListOfFiles(InFileName); - std::string aFileName = fFileReadThread.getNextInputDataFile(); - std::ifstream aFile(aFileName.c_str()); - aRet = (aFile.good()) ? true : false; - - // add back to list, which we pop_front for checking the file. - if (aRet) fFileReadThread.add2InputDataFileList(aFileName); - } - - return aRet; + fFileReadThread.chkForListOfFiles(InFileName); } //------------------------------------------------------------------------------ diff --git a/writeengine/splitter/we_sdhandler.h b/writeengine/splitter/we_sdhandler.h index 551df4d9f..e8d624e1b 100644 --- a/writeengine/splitter/we_sdhandler.h +++ b/writeengine/splitter/we_sdhandler.h @@ -111,7 +111,7 @@ public: bool releaseTableLocks(); void check4CpiInvokeMode(); bool check4PmArguments(); - bool check4InputFile(std::string InFileName); + void setInputFileList(std::string InFileName); bool check4CriticalErrMsgs(std::string& Entry); void onStartCpiResponse(int PmId); From e0cb60dcd27a4e6639807849dd45a7adce18ae1d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 11 Feb 2020 19:28:07 +0300 Subject: [PATCH 15/22] Merge pull request #1024 from LinuxJedi/collate_warn MCOL-3721 Allow collate and warn on ORDER BY --- dbcon/ddlpackage/ddl.l | 1 + dbcon/ddlpackage/ddl.y | 24 +++++++++++++++++++++--- dbcon/mysql/ha_mcs_execplan.cpp | 5 +++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 4cef8182e..b4034c732 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -171,6 +171,7 @@ FLOAT {return IDB_FLOAT;} DOUBLE {return DOUBLE;} REAL {return REAL;} CHARSET {return CHARSET;} +COLLATE {return COLLATE;} IF {return IDB_IF;} EXISTS {return EXISTS;} CHANGE {return CHANGE;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 177a64caa..d0f408e86 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -111,7 +111,7 @@ MATCH MAX_ROWS MEDIUMBLOB MEDIUMTEXT MIN_ROWS MODIFY NO NOT NULL_TOK NUMBER NUMERIC ON PARTIAL PRECISION PRIMARY REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY -VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET IDB_IF EXISTS CHANGE TRUNCATE +VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET COLLATE IDB_IF EXISTS CHANGE TRUNCATE BOOL BOOLEAN MEDIUMINT TIMESTAMP %token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE TIME @@ -485,8 +485,14 @@ table_option: } | DEFAULT CHARSET opt_equal ident {$$ = new pair("default charset", $4);} + | + CHARSET opt_equal ident {$$ = new pair("default charset", $3);} | DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair("default charset", $5);} + | + DEFAULT COLLATE opt_equal ident {$$ = new pair("default collate", $4);} + | + COLLATE opt_equal ident {$$ = new pair("default collate", $3);} ; alter_table_statement: @@ -726,13 +732,25 @@ optional_braces: | '(' ')' {} ; +opt_column_charset: + /* empty */ {} + | + IDB_CHAR SET ident {} + ; + +opt_column_collate: + /* empty */ {} + | + COLLATE ident {} + ; + data_type: - character_string_type + character_string_type opt_column_charset opt_column_collate | binary_string_type | numeric_type | datetime_type | blob_type - | text_type + | text_type opt_column_charset opt_column_collate | IDB_BLOB { $$ = new ColumnType(DDL_BLOB); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 227b80a3b..f2234a2ed 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -7562,6 +7562,11 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, { gwi.fatalParseError = true; } + else if ((ord_item->type() == Item::FUNC_ITEM) && (((Item_func*)ord_item)->functype() == Item_func::COLLATE_FUNC)) + { + push_warning(gwi.thd, Sql_condition::WARN_LEVEL_NOTE, WARN_OPTION_IGNORED, "COLLATE is ignored in ColumnStore"); + continue; + } else { rc = buildReturnedColumn(ord_item, gwi, gwi.fatalParseError); From 635438599c4705cf3ab722955d20d8075aa9533f Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Wed, 12 Feb 2020 08:51:23 -0600 Subject: [PATCH 16/22] Merge pull request #1042 from benthompson15/MCOL-3736 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MCOL-3736: Change AND to OR. Loop could exit with ops still in pendin… --- storage-manager/src/Synchronizer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-manager/src/Synchronizer.cpp b/storage-manager/src/Synchronizer.cpp index 7e86f8e2d..ee47b9dfa 100644 --- a/storage-manager/src/Synchronizer.cpp +++ b/storage-manager/src/Synchronizer.cpp @@ -305,7 +305,7 @@ void Synchronizer::syncNow(const bf::path &prefix) // this should be redone to only remove items of given prefix eventually blockNewJobs = true; - while (pendingOps.size() != 0 && opsInProgress.size() != 0) + while (pendingOps.size() != 0 || opsInProgress.size() != 0) { for (auto &job : pendingOps) makeJob(job.first); @@ -329,7 +329,7 @@ void Synchronizer::syncNow() // Leaving S3 storage and local metadata directories sync'd for snapshot backups. blockNewJobs = true; - while (pendingOps.size() != 0 && opsInProgress.size() != 0) + while (pendingOps.size() != 0 || opsInProgress.size() != 0) { for (auto &job : pendingOps) makeJob(job.first); From eb52184a5d4a74de78e8f3bdcce225537004652b Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Wed, 12 Feb 2020 08:52:58 -0600 Subject: [PATCH 17/22] Merge pull request #1043 from benthompson15/MCOL-3748 MCOL-3748: Modify signal handling for different default actions. --- storage-manager/src/main.cpp | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/storage-manager/src/main.cpp b/storage-manager/src/main.cpp index e8d68390c..3d0108fe4 100644 --- a/storage-manager/src/main.cpp +++ b/storage-manager/src/main.cpp @@ -61,6 +61,14 @@ void shutdownSM(int sig) if (!signalCaught) { (SessionManager::get())->shutdownSM(sig); + } + signalCaught = true; +} + +void coreSM(int sig) +{ + if (!signalCaught) + { fatalHandler(sig); } signalCaught = true; @@ -78,15 +86,20 @@ int main(int argc, char** argv) struct sigaction sa; memset(&sa, 0, sizeof(sa)); - std::vector shutdownSignals{ SIGHUP, SIGINT, SIGQUIT, SIGILL, - SIGTRAP, SIGABRT, SIGBUS, SIGFPE, - SIGSEGV, SIGALRM, SIGTERM, SIGXCPU, - SIGXFSZ, SIGVTALRM, SIGPROF, SIGPOLL, - SIGPWR, SIGSYS}; + std::vector shutdownSignals{ SIGALRM, SIGHUP, SIGINT, SIGPOLL, + SIGPROF, SIGPWR, SIGTERM, SIGVTALRM}; + + std::vector coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL, + SIGQUIT, SIGSEGV, SIGSYS, SIGTRAP, + SIGXCPU, SIGXFSZ}; + sa.sa_handler = shutdownSM; for (int sig : shutdownSignals) sigaction(sig, &sa, NULL); - + + sa.sa_handler = coreSM; + for (int sig : coreSignals) + sigaction(sig, &sa, NULL); sa.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sa, NULL); From 61574cd00dcf2b3762f064ccc51027418dcb4143 Mon Sep 17 00:00:00 2001 From: Jose Rojas Date: Wed, 12 Feb 2020 09:59:55 -0600 Subject: [PATCH 18/22] Merge pull request #1047 from LinuxJedi/collate_warn MCOL-3721 Fix quoting issues for COLLATE --- dbcon/ddlpackage/ddl.y | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index d0f408e86..f95f3e3fb 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -199,7 +199,7 @@ BOOL BOOLEAN MEDIUMINT TIMESTAMP %type trunc_table_statement %type rename_table_statement %type ident - +%type opt_quoted_literal %% stmtblock: stmtmulti { x->fParseTree = $1; } ; @@ -490,9 +490,9 @@ table_option: | DEFAULT IDB_CHAR SET opt_equal ident {$$ = new pair("default charset", $5);} | - DEFAULT COLLATE opt_equal ident {$$ = new pair("default collate", $4);} + DEFAULT COLLATE opt_equal opt_quoted_literal {$$ = new pair("default collate", $4);} | - COLLATE opt_equal ident {$$ = new pair("default collate", $3);} + COLLATE opt_equal opt_quoted_literal {$$ = new pair("default collate", $3);} ; alter_table_statement: @@ -735,13 +735,13 @@ optional_braces: opt_column_charset: /* empty */ {} | - IDB_CHAR SET ident {} + IDB_CHAR SET opt_quoted_literal {} ; opt_column_collate: /* empty */ {} | - COLLATE ident {} + COLLATE opt_quoted_literal {} ; data_type: @@ -864,6 +864,12 @@ string_literal: '\'' SCONST '\'' {$$ = $2;} ; +opt_quoted_literal: + string_literal + | + ident + ; + character_string_type: CHARACTER { From da32afc0b29ccb7fafa7241dd6e43e2591186acb Mon Sep 17 00:00:00 2001 From: benthompson15 Date: Wed, 12 Feb 2020 10:01:26 -0600 Subject: [PATCH 19/22] Merge pull request #1023 from LinuxJedi/mariadb-systemctl MCOL-3718 Use systemd instead of mysql-Columnstore --- dbcon/mysql/CMakeLists.txt | 2 +- dbcon/mysql/mysql-Columnstore | 486 ------------------ .../columnstore-post-install.in | 2 - .../columnstore-pre-uninstall.in | 10 +- oam/install_scripts/columnstore.in | 8 +- oam/install_scripts/post-mysql-install | 42 +- oam/oamcpp/liboamcpp.cpp | 128 +++-- oamapps/columnstoreSupport/configReport.sh | 10 +- procmon/processmonitor.cpp | 13 +- .../clusterTester/columnstoreClusterTester.sh | 4 +- 10 files changed, 161 insertions(+), 544 deletions(-) delete mode 100755 dbcon/mysql/mysql-Columnstore diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 8f1610c02..85c13f2a1 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -52,7 +52,7 @@ install(FILES syscatalog_mysql.sql calshowprocesslist.sql columnstore_info.sql DESTINATION ${ENGINE_SUPPORTDIR} COMPONENT columnstore-engine) -install(PROGRAMS install_mcs_mysql.sh mysql-Columnstore +install(PROGRAMS install_mcs_mysql.sh DESTINATION ${ENGINE_SBINDIR} COMPONENT columnstore-engine) install(FILES columnstore.cnf diff --git a/dbcon/mysql/mysql-Columnstore b/dbcon/mysql/mysql-Columnstore deleted file mode 100755 index 5c5305726..000000000 --- a/dbcon/mysql/mysql-Columnstore +++ /dev/null @@ -1,486 +0,0 @@ -#!/bin/sh -# Copyright Abandoned 1996 TCX DataKonsult AB & Monty Program KB & Detron HB -# This file is public domain and comes with NO WARRANTY of any kind - -# MySQL daemon start/stop script. - -# Usually this is put in /etc/init.d (at least on machines SYSV R4 based -# systems) and linked to /etc/rc3.d/S99mysql and /etc/rc0.d/K01mysql. -# When this is done the mysql server will be started when the machine is -# started and shut down when the systems goes down. - -# Comments to support chkconfig on RedHat Linux -# chkconfig: 2345 64 36 -# description: A very fast and reliable SQL database engine. - -# Comments to support LSB init script conventions -### BEGIN INIT INFO -# Provides: mysql -# Required-Start: $local_fs $network $remote_fs -# Should-Start: ypbind nscd ldap ntpd xntpd -# Required-Stop: $local_fs $network $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start and stop MySQL -# Description: MySQL is a very fast and reliable SQL database engine. -### END INIT INFO - -# If you install MySQL on some other places than /usr/local/mariadb/columnstore/mysql, then you -# have to do one of the following things for this script to work: -# -# - Run this script from within the MySQL installation directory -# - Create a /etc/my.cnf file with the following information: -# [mysqld] -# basedir= -# - Add the above to any other configuration file (for example ~/.my.ini) -# and copy my_print_defaults to /usr/bin -# - Add the path to the mysql-installation-directory to the basedir variable -# below. -# -# If you want to affect other MySQL variables, you should make your changes -# in the /etc/my.cnf, ~/.my.cnf or other MySQL configuration files. - -# If you change base dir, you must also change datadir. These may get -# overwritten by settings in the MySQL configuration files. - - -# Source function library. -if [ -f /etc/init.d/functions ]; then - . /etc/init.d/functions -fi - -# Default value, in seconds, afterwhich the script should timeout waiting -# for server start. -# Value here is overriden by value in my.cnf. -# 0 means don't wait at all -# Negative numbers mean to wait indefinitely -service_startup_timeout=90 -user=`whoami 2>/dev/null` - -# Lock directory -lockdir=/var/lock/subsys - -lock_file_path="$lockdir/mysql-Columnstore" - -# The following variables are only set for letting mysql.server find things. - -# Set some defaults -mysqld_pid_file_path= -if test -z "$basedir" -then - basedir=/usr - bindir=/usr/bin - if test -z "$datadir" - then - datadir=/var/lib/mysql - fi - sbindir=/usr/bin - bindir=/usr/bin -else - bindir="$basedir/bin" - if test -z "$datadir" - then - datadir="$basedir/data" - fi - sbindir="$basedir/sbin" - if test -f "$basedir/sbin/mysqld" - then - bindir="$basedir/bin" - else - bindir="$basedir/bin" - fi -fi - -# datadir_set is used to determine if datadir was set (and so should be -# *not* set inside of the --basedir= handler.) -datadir_set= - -# -# Use LSB init script functions for printing messages, if possible -# -lsb_functions="/lib/lsb/init-functions" -if test -f $lsb_functions ; then - . $lsb_functions >/dev/null 2>&1 -else - # Include non-LSB RedHat init functions to make systemctl redirect work - init_functions="/etc/init.d/functions" - if test -f $init_functions; then - . $init_functions >/dev/null 2>&1 - fi - log_success_msg() - { - echo " SUCCESS! $@" - } - log_failure_msg() - { - echo " ERROR! $@" - } -fi - -PATH="/sbin:/usr/sbin:/bin:/usr/bin:$basedir/bin" -export PATH - -mode=$1 # start or stop - -[ $# -ge 1 ] && shift - -case `echo "testing\c"`,`echo -n testing` in - *c*,-n*) echo_n= echo_c= ;; - *c*,*) echo_n=-n echo_c= ;; - *) echo_n= echo_c='\c' ;; -esac - -parse_server_arguments() { - for arg do - case "$arg" in - --basedir=*) basedir=`echo "$arg" | sed -e 's/^[^=]*=//'` - bindir="$basedir/bin" - if test -z "$datadir_set"; then - datadir="$basedir/data" - fi - sbindir="$basedir/sbin" - if test -f "$basedir/sbin/mysqld" - then - bindir="$basedir/bin" - else - bindir="$basedir/bin" - fi - #bindir="$basedir/bin" - ;; - --datadir=*) datadir=`echo "$arg" | sed -e 's/^[^=]*=//'` - datadir_set=1 - ;; - --log-basename=*|--hostname=*|--loose-log-basename=*) - mysqld_pid_file_path=`echo "$arg.pid" | sed -e 's/^[^=]*=//'` - ;; - --pid-file=*) mysqld_pid_file_path=`echo "$arg" | sed -e 's/^[^=]*=//'` ;; - --service-startup-timeout=*) service_startup_timeout=`echo "$arg" | sed -e 's/^[^=]*=//'` ;; - esac - done -} - -# Get arguments from the my.cnf file, -# the only group, which is read from now on is [mysqld] -if test -x ./bin/my_print_defaults -then - print_defaults="./bin/my_print_defaults" -elif test -x $bindir/my_print_defaults -then - print_defaults="$bindir/my_print_defaults" -elif test -x $bindir/mysql_print_defaults -then - print_defaults="$bindir/mysql_print_defaults" -else - # Try to find basedir in /etc/my.cnf - conf=/etc/my.cnf - print_defaults= - if test -r $conf - then - subpat='^[^=]*basedir[^=]*=\(.*\)$' - dirs=`sed -e "/$subpat/!d" -e 's//\1/' $conf` - for d in $dirs - do - d=`echo $d | sed -e 's/[ ]//g'` - if test -x "$d/bin/my_print_defaults" - then - print_defaults="$d/bin/my_print_defaults" - break - fi - if test -x "$d/bin/mysql_print_defaults" - then - print_defaults="$d/bin/mysql_print_defaults" - break - fi - done - fi - - # Hope it's in the PATH ... but I doubt it - test -z "$print_defaults" && print_defaults="my_print_defaults" -fi - -# -# Read defaults file from 'basedir'. If there is no defaults file there -# check if it's in the old (depricated) place (datadir) and read it from there -# - -extra_args="" -if test -r "$basedir/my.cnf" -then - extra_args="-e $basedir/my.cnf" -else - if test -r "$datadir/my.cnf" - then - extra_args="-e $datadir/my.cnf" - fi -fi - -parse_server_arguments `$print_defaults $extra_args --mysqld mysql.server` -parse_server_arguments "$@" - -# wait for the pid file to disappear -wait_for_gone () { - pid="$1" # process ID of the program operating on the pid-file - pid_file_path="$2" # path to the PID file. - - i=0 - crash_protection="by checking again" - - while test $i -ne $service_startup_timeout ; do - - if kill -0 "$pid" 2>/dev/null; then - : # the server still runs - else - if test ! -s "$pid_file_path"; then - # no server process and no pid-file? great, we're done! - log_success_msg - return 0 - fi - - # pid-file exists, the server process doesn't. - # it must've crashed, and mysqld_safe will restart it - if test -n "$crash_protection"; then - crash_protection="" - sleep 5 - continue # Check again. - fi - - kill_by_pid - # Cannot help it - log_failure_msg "The server quit without updating PID file ($pid_file_path)." - return 1 # not waiting any more. - fi - - echo $echo_n ".$echo_c" - i=`expr $i + 1` - sleep 1 - - done - - log_failure_msg - kill_by_pid - - return 1 -} - -wait_for_ready () { - - i=0 - while test $i -ne $service_startup_timeout ; do - - if $bindir/mysqladmin ping >/dev/null 2>&1; then - log_success_msg - return 0 - elif kill -0 $! 2>/dev/null ; then - : # mysqld_safe is still running - else - # mysqld_safe is no longer running, abort the wait loop - break - fi - - echo $echo_n ".$echo_c" - i=`expr $i + 1` - sleep 1 - - done - - log_failure_msg - return 1 -} -# -# Set pid file if not given -# -if test -z "$mysqld_pid_file_path" -then - mysqld_pid_file_path=$datadir/`hostname`.pid -else - case "$mysqld_pid_file_path" in - /* ) ;; - * ) mysqld_pid_file_path="$datadir/$mysqld_pid_file_path" ;; - esac -fi - -# source other config files -[ -f /etc/default/mysql ] && . /etc/default/mysql -[ -f /etc/sysconfig/mysql ] && . /etc/sysconfig/mysql -[ -f /etc/conf.d/mysql ] && . /etc/conf.d/mysql - -kill_by_pid() { - # let's see if we can kill the 2 mysql procs by hand - # get the our mysql from ps - eval $(ps -ef | grep "bin/mysqld" | grep -v grep | head -1 | awk '{printf "pid=%d\n", $2}') - - if [ -n "$pid" ]; then - ppid=$(ps -o ppid= -p $pid) - kill -9 $ppid - sleep 1 - kill -9 $pid - echo $echo_n "Force shutting down (no/bad pid file)" - log_success_msg - exit 0 - fi - return -} - -case "$mode" in - 'start') - # Start daemon - - # Safeguard (relative paths, core dumps..) - cd $basedir - - echo $echo_n "Starting MySQL" - if test -x $bindir/mysqld_safe - then - # Give extra arguments to mysqld with the my.cnf file. This script - # may be overwritten at next upgrade. - $bindir/mysqld_safe --datadir="$datadir" --pid-file="$mysqld_pid_file_path" "$@" >/dev/null 2>&1 & - wait_for_ready; return_value=$? - - # Make lock for RedHat / SuSE - if test -w "$lockdir" - then - touch "$lock_file_path" - fi - - exit $return_value - else - log_failure_msg "Couldn't find MySQL server ($bindir/mysqld_safe)" - fi - ;; - - 'stop') - # Stop daemon. We use a signal here to avoid having to know the - # root password. - - if test -s "$mysqld_pid_file_path" - then - mysqld_pid=`cat "$mysqld_pid_file_path"` - - if (kill -0 $mysqld_pid 2>/dev/null) - then - echo $echo_n "Shutting down MySQL" - kill $mysqld_pid - # mysqld should remove the pid file when it exits, so wait for it. - wait_for_gone $mysqld_pid "$mysqld_pid_file_path"; return_value=$? - else - log_failure_msg "MySQL server process #$mysqld_pid is not running!" - kill_by_pid - rm "$mysqld_pid_file_path" - fi - - # Delete lock for RedHat / SuSE - if test -f "$lock_file_path" - then - rm -f "$lock_file_path" - fi - exit $return_value - else - kill_by_pid - log_failure_msg "MySQL server PID file could not be found!" - fi - ;; - - 'restart') - # Stop the service and regardless of whether it was - # running or not, start it again. - if $0 stop "$@"; then - if ! $0 start "$@"; then - log_failure_msg "Failed to restart server." - exit 1 - fi - else - log_failure_msg "Failed to stop running server, so refusing to try to start." - exit 1 - fi - ;; - - 'reload'|'force-reload') - if test -s "$mysqld_pid_file_path" ; then - read mysqld_pid < "$mysqld_pid_file_path" - kill -HUP $mysqld_pid && log_success_msg "Reloading service MySQL" - touch "$mysqld_pid_file_path" - else - log_failure_msg "MySQL PID file could not be found!" - exit 1 - fi - ;; - 'status') - # First, check to see if pid file exists - if test -s "$mysqld_pid_file_path" ; then - read mysqld_pid < "$mysqld_pid_file_path" - if kill -0 $mysqld_pid 2>/dev/null ; then - log_success_msg "MySQL running ($mysqld_pid)" - exit 0 - else - log_failure_msg "MySQL is not running, but PID file exists" - exit 1 - fi - else - # Try to find appropriate mysqld process - mysqld_pid=`pidof $bindir/mysqld` - - # test if multiple pids exist - pid_count=`echo $mysqld_pid | wc -w` - if test $pid_count -gt 1 ; then - log_failure_msg "Multiple MySQL running but PID file could not be found ($mysqld_pid)" - exit 5 - elif test -z $mysqld_pid ; then - if test -f "$lock_file_path" ; then - log_failure_msg "MySQL is not running, but lock file ($lock_file_path) exists" - exit 2 - fi - log_failure_msg "MySQL is not running" - exit 3 - else - log_failure_msg "MySQL is running but PID file could not be found" - exit 4 - fi - fi - ;; - 'configtest') - # Safeguard (relative paths, core dumps..) - cd $basedir - echo $echo_n "Testing MySQL configuration syntax" - daemon=$bindir/mysqld - if test -x $bindir/mysqld - then - daemon=$bindir/mysqld - elif test -x $sbindir/mysqld - then - daemon=$sbindir/mysqld - elif test -x `which mysqld` - then - daemon=`which mysqld` - else - log_failure_msg "Unable to locate the mysqld binary!" - exit 1 - fi - help_out=`$daemon --help 2>&1`; r=$? - if test "$r" != 0 ; then - log_failure_msg "$help_out" - log_failure_msg "There are syntax errors in the server configuration. Please fix them!" - else - log_success_msg "Syntax OK" - fi - exit $r - ;; - 'bootstrap') - if test "$_use_systemctl" == 1 ; then - log_failure_msg "Please use galera_new_cluster to start the mariadb service with --wsrep-new-cluster" - exit 1 - fi - # Bootstrap the cluster, start the first node - # that initiate the cluster - echo $echo_n "Bootstrapping the cluster.. " - $0 start $other_args --wsrep-new-cluster - exit $? - ;; - *) - # usage - basename=`basename "$0"` - echo "Usage: $basename {start|stop|restart|reload|force-reload|status|configtest|bootstrap} [ MySQL server options ]" - exit 1 - ;; -esac - -exit 0 diff --git a/oam/install_scripts/columnstore-post-install.in b/oam/install_scripts/columnstore-post-install.in index 2e253de53..c0384295f 100755 --- a/oam/install_scripts/columnstore-post-install.in +++ b/oam/install_scripts/columnstore-post-install.in @@ -108,7 +108,6 @@ fi if [ $user = "root" ]; then #setup the columnstore service script rm -f /etc/init.d/columnstore >/dev/null 2>&1 - rm -f /etc/init.d/mysql-Columnstore >/dev/null 2>&1 rm -f /etc/default/columnstore systemctl=`which systemctl 2>/dev/null` @@ -170,7 +169,6 @@ mcsSetConfig -d Installation LockFileDirectory $lockdir mkdir $lockdir >/dev/null 2>&1 -rm -f $lockdir/mysql-Columnstore rm -f $lockdir/columnstore #backup copy of Alarm Config File diff --git a/oam/install_scripts/columnstore-pre-uninstall.in b/oam/install_scripts/columnstore-pre-uninstall.in index 38f20686e..41896c937 100755 --- a/oam/install_scripts/columnstore-pre-uninstall.in +++ b/oam/install_scripts/columnstore-pre-uninstall.in @@ -25,7 +25,14 @@ done #stop services columnstore stop > /dev/null 2>&1 -mysql-Columnstore stop > /dev/null 2>&1 + +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 +else + pkill mysqld +fi cloud=`mcsGetConfig Installation Cloud` if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then @@ -55,7 +62,6 @@ rm -f /var/lib/columnstore/local/moveDbrootTransactionLog lockdir=`mcsGetConfig Installation LockFileDirectory` rm -f $lockdir/columnstore -rm -f $lockdir/mysql-Columnstore # delete core files #rm -f /var/log/mariadb/columnstore/corefiles/* > /dev/null 2>&1 diff --git a/oam/install_scripts/columnstore.in b/oam/install_scripts/columnstore.in index 8686b6683..9ef7cf7ad 100644 --- a/oam/install_scripts/columnstore.in +++ b/oam/install_scripts/columnstore.in @@ -93,7 +93,13 @@ stop() { RETVAL=$? rm -f $lockdir/columnstore fuser -k 8604/tcp > /dev/null 2>&1 - mysql-Columnstore stop > /dev/null 2>&1 + # Test we are using systemd + systemctl cat mariadb.service > /dev/null 2>&1 + if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 + else + pkill mysqld + fi pkill ProcMon pkill ProcMgr return $RETVAL diff --git a/oam/install_scripts/post-mysql-install b/oam/install_scripts/post-mysql-install index d8deaff73..81808c5da 100755 --- a/oam/install_scripts/post-mysql-install +++ b/oam/install_scripts/post-mysql-install @@ -11,7 +11,13 @@ checkForError() { if [ `cat ${tmpdir}/error.check | wc -c` -ne 0 ]; then echo "MySQL Password file missing or incorrect, check .my.cnf file" rm -f ${tmpdir}/error.check - mysql-Columnstore stop + # Test we are using systemd + systemctl cat mariadb.service > /dev/null 2>&1 + if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 + else + pkill mysqld + fi sleep 2 exit 2; fi @@ -54,9 +60,23 @@ for arg in "$@"; do done # Restart in the same way that mysqld will be started normally. -mysql-Columnstore stop >/dev/null 2>&1 +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 +else + pkill mysqld +fi sleep 2 -mysql-Columnstore start --skip-grant-tables +export MYSQL_OPTS="--skip-grant-tables" +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl start mariadb.service +else + /usr/bin/mysqld_safe --skip-grant-tables & +fi +unset MYSQL_OPTS sleep 5 @@ -65,12 +85,24 @@ install_mcs_mysql.sh --tmpdir=$tmpdir checkForError if [ $? -ne 0 ]; then echo "ERROR: Invalid password in .my.cnf, or Columnstore plugin install missing" - mysql-Columnstore stop + # Test we are using systemd + systemctl cat mariadb.service > /dev/null 2>&1 + if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 + else + pkill mysqld + fi sleep 2 exit 2; fi -mysql-Columnstore stop +# Test we are using systemd +systemctl cat mariadb.service > /dev/null 2>&1 +if [ $? -eq 0 ]; then + systemctl stop mariadb.service > /dev/null 2>&1 +else + pkill mysqld +fi exit 0 diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 5fb72f387..c0a020a10 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -7750,7 +7750,7 @@ int Oam::sendDeviceNotification(std::string deviceName, NOTIFICATION_TYPE type, * * Function: actionMysqlCalpont * - * Purpose: mysql-Columnstore service command + * Purpose: systemctl mariadb.service command * ****************************************************************************/ @@ -7790,24 +7790,33 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) else return; - // check if mysql-Columnstore is installed - string mysqlscript = "mysql-Columnstore"; + string mysqlscript = "systemctl"; string command; string pidtmp = tmpdir + "/mysql.pid"; + int no_systemd = system("systemctl cat mariadb.server > /dev/null 2>&1"); + switch (action) { case MYSQL_START: { - command = "start > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + if (no_systemd) + { + system("/usr/bin/mysqld_safe &"); + } + command = "start"; break; } case MYSQL_STOP: { - command = "stop > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + if (no_systemd) + { + system("pkill mysqld"); + } + command = "stop"; //set process status try @@ -7822,25 +7831,38 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) case MYSQL_RESTART: { - command = "restart > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + if (no_systemd) + { + system("pkill mysqld"); + system("/usr/bin/mysqld_safe &"); + } + command = "restart"; break; } case MYSQL_RELOAD: { - command = "reload > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + if (no_systemd) + { + system("pkill -HUP mysqld"); + } + command = "reload"; break; } case MYSQL_FORCE_RELOAD: { - command = "force-reload > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + if (no_systemd) + { + system("pkill -HUP mysqld"); + } + command = "force-reload"; break; } case MYSQL_STATUS: { - command = "status > " + tmpdir + "/mysql.status"; + command = "status"; break; } @@ -7852,19 +7874,36 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) } //RUN COMMAND - string cmd = mysqlscript + " " + command; - system(cmd.c_str()); + if (!no_systemd) + { + string cmd = mysqlscript + " " + command + " mariadb.service > " + tmpdir + "/actionMysqlCalpont.log 2>&1"; + system(cmd.c_str()); + } if (action == MYSQL_START || action == MYSQL_RESTART) { - //get pid - char buf[512]; - FILE *cmd_pipe = popen("pidof -s mysqld", "r"); + pid_t pid = 0; + // Loop check because we mysqld may not start immediately + for (int i=0; i < 10; i++) + { + //get pid + char buf[512]; + FILE *cmd_pipe = popen("pidof -s mysqld", "r"); - fgets(buf, 512, cmd_pipe); - pid_t pid = strtoul(buf, NULL, 10); + fgets(buf, 512, cmd_pipe); + pid = strtoul(buf, NULL, 10); - pclose( cmd_pipe ); + pclose( cmd_pipe ); + + if (pid) + { + break; + } + else + { + sleep(2); + } + } if (!pid) { @@ -7888,11 +7927,11 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) getProcessStatus("mysqld", moduleName, procstat); int state = procstat.ProcessOpState; pid_t pidStatus = procstat.ProcessID; - + pid_t pid = 0; string mysqlStatus = tmpdir + "/mysql.status"; - if (checkLogStatus(mysqlStatus, "MySQL running")) + if ( state != ACTIVE ) { - if ( state != ACTIVE ) + for (int i=0; i < 10; i++) { //get pid char buf[512]; @@ -7903,17 +7942,26 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) pclose( cmd_pipe ); - //set process status - try + if (pid) { - setProcessStatus("mysqld", moduleName, ACTIVE, pid); + //set process status + try + { + setProcessStatus("mysqld", moduleName, ACTIVE, pid); + } + catch (...) + {} + return; + } + else + { + sleep(2); } - catch (...) - {} - - return; } - else + } + else + { + for (int i=0; i < 10; i++) { //check if pid has changed char buf[512]; @@ -7924,18 +7972,28 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) pclose( cmd_pipe ); - if ( pidStatus != pid ) + if (pid) { - //set process status - try + if ( pidStatus != pid ) { - setProcessStatus("mysqld", moduleName, ACTIVE, pid); + //set process status + try + { + setProcessStatus("mysqld", moduleName, ACTIVE, pid); + } + catch (...) + {} + break; } - catch (...) - {} + } + else + { + sleep(2); } } - + } + if (pid) + { //check module status, if DEGRADED set to ACTIVE int opState; bool degraded; diff --git a/oamapps/columnstoreSupport/configReport.sh b/oamapps/columnstoreSupport/configReport.sh index a74bc995d..b81125127 100755 --- a/oamapps/columnstoreSupport/configReport.sh +++ b/oamapps/columnstoreSupport/configReport.sh @@ -25,9 +25,6 @@ if [ -n "$chkconfig" ]; then echo "################# chkconfig --list | grep columnstore #################" echo " " chkconfig --list | grep columnstore 2>/dev/null - echo "################# chkconfig --list | grep mysql-Columnstore #################" - echo " " - chkconfig --list | grep mysql-Columnstore 2>/dev/null fi systemctl=`which systemctl 2>/dev/null` @@ -37,9 +34,9 @@ if [ -n "$systemctl" ]; then echo "################# systemctl list-unit-files --type=service | grep columnstore #################" echo " " systemctl list-unit-files --type=service | grep columnstore 2>/dev/null - echo "################# systemctl list-unit-files --type=service | grep mysql-Columnstore #################" + echo "################# systemctl list-unit-files --type=service | grep mariadb #################" echo " " - systemctl list-unit-files --type=service | grep mysql-Columnstore 2>/dev/null + systemctl list-unit-files --type=service | grep mariadb 2>/dev/null fi updaterc=`which update-rc.d 2>/dev/null` @@ -49,9 +46,6 @@ if [ -n "$updaterc" ]; then echo "################# service --status-all | grep columnstore #################" echo " " service --status-all | grep columnstore 2>/dev/null - echo "################# service --status-all | grep mysql-Columnstore #################" - echo " " - service --status-all | grep mysql-Columnstore 2>/dev/null fi diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 6138a305c..e2da321ee 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1514,8 +1514,17 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO system(cmd.c_str()); cmd = "post-mysql-install >> " + tmpLogDir + "/rpminstall"; system(cmd.c_str()); - cmd = "mysql-Columnstore start > " + tmpLogDir + "/mysqldstart"; - system(cmd.c_str()); + int ret = system("systemctl cat mariadb.service > /dev/null 2>&1"); + if (!ret) + { + cmd = "systemctl start mariadb.service > " + tmpLogDir + "/mysqldstart"; + system(cmd.c_str()); + } + else + { + cmd = "/usr/bin/mysqld_safe & > " + tmpLogDir + "/mysqldstart"; + system(cmd.c_str()); + } string tmpFile = tmpLogDir + "/mysqldstart"; ifstream file (tmpFile.c_str()); diff --git a/utils/clusterTester/columnstoreClusterTester.sh b/utils/clusterTester/columnstoreClusterTester.sh index 282a4fade..ce968ec8f 100755 --- a/utils/clusterTester/columnstoreClusterTester.sh +++ b/utils/clusterTester/columnstoreClusterTester.sh @@ -753,7 +753,7 @@ checkMysqlPassword() #get MariaDB password pass=true - `mysql-Columnstore start > /dev/null 2>&1` + `systemctl start mariadb.service > /dev/null 2>&1` `mariadb-command-line.sh > /dev/null 2>&1` if [ "$?" -eq 2 ]; then echo "${bold}Failed${normal}, Local Node MariaDB login failed with missing password file, /root/.my.cnf" @@ -763,7 +763,7 @@ checkMysqlPassword() `/bin/cp -f mariadb-command-line.sh ${tmpDir}/.` for ipadd in "${NODE_IPADDRESS[@]}"; do - `remote_command.sh $ipadd $PASSWORD mysql-Columnstore start > /dev/null 2>&1` + `remote_command.sh $ipadd $PASSWORD systemctl start mariadb.service > /dev/null 2>&1` `remote_scp_put.sh $ipadd $PASSWORD ${tmpDir}/mariadb-command-line.sh 1 > ${tmpDir}/remote_scp_put_check 2>&1` if [ "$?" -ne 0 ]; then echo "Error running remote_scp_put.sh to $ipadd Node, check ${tmpDir}/remote_scp_put_check" From ffcd527c4e19456581d3acfe7a586e485f411c6d Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:06:22 -0600 Subject: [PATCH 20/22] Merge pull request #1052 from mariadb-corporation/MCOL-3791 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MCOL-3791 This patch implicitly enables disk space preallocation for … --- writeengine/shared/we_fileop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index d4f9f1665..f4c65ed52 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -1130,7 +1130,7 @@ int FileOp::initColumnExtent( // Couldn't avoid preallocation for full extents, // e.g. ADD COLUMN DDL b/c CS has to fill the file // with empty magics. - if ( !bOptExtension ) + if ( !bOptExtension || !m_compressionType ) { #ifdef PROFILE Stats::startParseEvent(WE_STATS_INIT_COL_EXTENT); From a12442f4b4978aa92f4f90abc46b357c591f29ba Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc <43503225+pleblanc1976@users.noreply.github.com> Date: Mon, 17 Feb 2020 08:12:38 -0600 Subject: [PATCH 21/22] Merge pull request #1053 from LinuxJedi/mariadb-systemctl MCOL-3793 Fix UM ACTIVE / DEGRADED flip-flop --- oam/oamcpp/liboamcpp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index c0a020a10..1a02a9cb9 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -7796,7 +7796,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) string pidtmp = tmpdir + "/mysql.pid"; - int no_systemd = system("systemctl cat mariadb.server > /dev/null 2>&1"); + int no_systemd = system("systemctl cat mariadb.service > /dev/null 2>&1"); switch (action) { @@ -7938,7 +7938,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) FILE *cmd_pipe = popen("pidof -s mysqld", "r"); fgets(buf, 512, cmd_pipe); - pid_t pid = strtoul(buf, NULL, 10); + pid = strtoul(buf, NULL, 10); pclose( cmd_pipe ); @@ -7968,7 +7968,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action) FILE *cmd_pipe = popen("pidof -s mysqld", "r"); fgets(buf, 512, cmd_pipe); - pid_t pid = strtoul(buf, NULL, 10); + pid = strtoul(buf, NULL, 10); pclose( cmd_pipe ); From 03a2283bac8ab7ce05a644781e94650cb6e18c12 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Fri, 28 Feb 2020 17:11:13 -0500 Subject: [PATCH 22/22] Fixed a merge error. --- dbcon/mysql/ha_mcs.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbcon/mysql/ha_mcs.cpp b/dbcon/mysql/ha_mcs.cpp index 67b683e21..c11617388 100644 --- a/dbcon/mysql/ha_mcs.cpp +++ b/dbcon/mysql/ha_mcs.cpp @@ -149,7 +149,6 @@ static int columnstore_init_func(void* p) (void) my_hash_init(&mcs_open_tables, system_charset_info, 32, 0, 0, (my_hash_get_key) mcs_get_key, 0, 0); - mcs_hton->state = SHOW_OPTION_YES; mcs_hton->create = mcs_create_handler; mcs_hton->flags = HTON_CAN_RECREATE; // mcs_hton->discover_table = mcs_discover;