From eff7d499ec9c34e2f1a68079fb3bfd7910af15ec Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 28 Nov 2017 13:19:28 -0600 Subject: [PATCH 1/2] MCOL-942 - backported mysql rep fixes from 1.1 --- cpackEngineRPM.cmake | 1 + oam/install_scripts/CMakeLists.txt | 1 + oamapps/postConfigure/helpers.cpp | 4 +- procmgr/processmanager.cpp | 6 +- procmon/processmonitor.cpp | 357 ++++++++++++----------------- procmon/processmonitor.h | 5 + 6 files changed, 160 insertions(+), 214 deletions(-) diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index f567aa40d..7ad235dc8 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -217,6 +217,7 @@ SET(CPACK_RPM_platform_USER_FILELIST "/usr/local/mariadb/columnstore/post/test-004.sh" "/usr/local/mariadb/columnstore/bin/os_detect.sh" "/usr/local/mariadb/columnstore/bin/columnstoreClusterTester.sh" +"/usr/local/mariadb/columnstore/bin/mariadb-command-line.sh" ${ignored}) SET(CPACK_RPM_libs_USER_FILELIST diff --git a/oam/install_scripts/CMakeLists.txt b/oam/install_scripts/CMakeLists.txt index 09e72fdb1..09f57c18f 100644 --- a/oam/install_scripts/CMakeLists.txt +++ b/oam/install_scripts/CMakeLists.txt @@ -34,6 +34,7 @@ install(PROGRAMS post-install myCnf-include-args.text myCnf-exclude-args.text columnstore.service + mariadb-command-line.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform) install(FILES module DESTINATION ${ENGINE_LOCALDIR} COMPONENT platform) diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp index 8638bebba..bbbaf6dd5 100644 --- a/oamapps/postConfigure/helpers.cpp +++ b/oamapps/postConfigure/helpers.cpp @@ -459,7 +459,9 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st { // set for slave repl request // don't do PMs unless PMwithUM flag is set string moduleType = (*pt).DeviceName.substr(0,MAX_MODULE_TYPE_SIZE); - if ( moduleType == "pm" && !pmwithum ) { + + if ( ( moduleType == "pm" && !pmwithum ) && + ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM ) ) { pt++; continue; } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 10a3c5916..ffbc0fc32 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -10048,8 +10048,10 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist MySQLRep = "n"; } - if ( MySQLRep == "n" && enable ) + if ( MySQLRep == "n" && enable ) { + log.writeLog(__LINE__, "setMySQLReplication: MySQLRep not set, exiting", LOG_TYPE_DEBUG); return oam::API_SUCCESS; + } log.writeLog(__LINE__, "Setup MySQL Replication", LOG_TYPE_DEBUG); @@ -10228,7 +10230,7 @@ int ProcessManager::setMySQLReplication(oam::DeviceNetworkList devicenetworklist for( ; listPT != devicenetworklist.end() ; listPT++) { string remoteModuleName = (*listPT).DeviceName; - log.writeLog(__LINE__, "Setup SlavMySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "Setup SlaveMySQL Replication on " + remoteModuleName, LOG_TYPE_DEBUG); //skip master if ( remoteModuleName == masterModule ) diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index cbef5f9ce..5080b9168 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1730,7 +1730,8 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string masterLogFile = oam::UnassignedName; string masterLogPos = oam::UnassignedName; - if ( (PMwithUM == "n") && (config.moduleType() == "pm") && ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) + if ( ( (PMwithUM == "n") && (config.moduleType() == "pm") ) && + ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) { ackMsg << (ByteStream::byte) ACK; ackMsg << (ByteStream::byte) MASTERREP; @@ -1803,7 +1804,8 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string port; msg >> port; - if ( (PMwithUM == "n") && (config.moduleType() == "pm") && ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) + if ( ( (PMwithUM == "n") && (config.moduleType() == "pm") ) && + ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) { ackMsg << (ByteStream::byte) ACK; ackMsg << (ByteStream::byte) SLAVEREP; @@ -1850,7 +1852,8 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string module; msg >> module; - if ( (PMwithUM == "n") && (config.moduleType() == "pm") && ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) + if ( ( (PMwithUM == "n") && (config.moduleType() == "pm") ) && + ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) { ackMsg << (ByteStream::byte) ACK; ackMsg << (ByteStream::byte) MASTERDIST; @@ -4659,218 +4662,103 @@ int ProcessMonitor::changeMyCnf(std::string type) string dbDir = startup::StartUp::installDir() + "/mysql/db"; -/* if ( type == "master" ) + //get server-id based on ExeMgrx setup + string serverID = "0"; + string localModuleName = config.moduleName(); + for ( int id = 1 ; ; id++ ) { - // set master replication entries - vector lines; - char line[200]; - string buf; - while (file.getline(line, 200)) - { - buf = line; - string::size_type pos = buf.find("server-id =",0); - if ( pos != string::npos ) { - buf = "server-id = 1"; + string Section = "ExeMgr" + oam.itoa(id); + + string moduleName; + + try { + Config* sysConfig = Config::makeConfig(); + moduleName = sysConfig->getConfig(Section, "Module"); + + if ( moduleName == localModuleName ) + { + serverID = oam.itoa(id); + break; } + } + catch (...) {} + } -// pos = buf.find("# binlog_format=ROW",0); -// if ( pos != string::npos ) { -// buf = "binlog_format=ROW"; -// } + if ( serverID == "0" ) + { + log.writeLog(__LINE__, "changeMyCnf: ExeMgr for local module doesn't exist", LOG_TYPE_ERROR); + return oam::API_FAILURE; + } - pos = buf.find("infinidb_local_query=1",0); - if ( pos != string::npos && pos == 0) { - buf = "# infinidb_local_query=1"; - } - - //output to temp file - lines.push_back(buf); + // set server-id and other options in my.cnf + vector lines; + char line[200]; + string buf; + while (file.getline(line, 200)) + { + buf = line; + string::size_type pos = buf.find("server-id",0); + if ( pos != string::npos ) { + buf = "server-id = " + serverID; + + string command = "SET GLOBAL server_id=" + serverID + ";"; + int ret = runMariaDBCommandLine(command); + if (ret != 0) + { + log.writeLog(__LINE__, "changeMyCnf: runMariaDBCommandLine Error", LOG_TYPE_ERROR); + return oam::API_FAILURE; + } } - file.close(); - unlink (mycnfFile.c_str()); - ofstream newFile (mycnfFile.c_str()); - - //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0664); - - copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); - newFile.close(); - - close(fd); - } -*/ -// if ( type == "slave" ) -// { - //get server-id based on ExeMgrx setup - string slaveID = "0"; - string slaveModuleName = config.moduleName(); - for ( int id = 1 ; ; id++ ) + // set local query flag if on pm + if ( (PMwithUM == "y") && config.moduleType() == "pm" ) { - string Section = "ExeMgr" + oam.itoa(id); + pos = buf.find("infinidb_local_query",0); + if ( pos != string::npos ) { + buf = "infinidb_local_query=1"; + + string command = "SET GLOBAL " + buf + ";"; - string moduleName; - - try { - Config* sysConfig = Config::makeConfig(); - moduleName = sysConfig->getConfig(Section, "Module"); - - if ( moduleName == slaveModuleName ) + int ret = runMariaDBCommandLine(command); + if (ret != 0) { - slaveID = oam.itoa(id); - - // if slave ID from above is 1, then it means this is a former Original master and use the ID of the current Master - if ( slaveID == "1" ) { - string PrimaryUMModuleName; - oam.getSystemConfig("PrimaryUMModuleName", PrimaryUMModuleName); - - for ( int mid = 1 ; ; mid++ ) - { - string Section = "ExeMgr" + oam.itoa(mid); - - string moduleName; - - try { - Config* sysConfig = Config::makeConfig(); - moduleName = sysConfig->getConfig(Section, "Module"); - - if ( moduleName == PrimaryUMModuleName ) - { - slaveID = oam.itoa(mid); - break; - } - } - catch (...) {} - } - } - break; - } + log.writeLog(__LINE__, "changeMyCnf: runMariaDBCommandLine Error", LOG_TYPE_ERROR); + return oam::API_FAILURE; + } } - catch (...) {} } + else + { // disable, if needed + pos = buf.find("infinidb_local_query",0); + if ( pos != string::npos ) { + buf = "infinidb_local_query=0"; - if ( slaveID == "0" ) - { - log.writeLog(__LINE__, "changeMyCnf: ExeMgr for local module doesn't exist", LOG_TYPE_ERROR); - return oam::API_FAILURE; + string command = "SET GLOBAL " + buf + ";"; + int ret = runMariaDBCommandLine(command); + if (ret != 0) + { + log.writeLog(__LINE__, "changeMyCnf: runMariaDBCommandLine Error", LOG_TYPE_ERROR); + return oam::API_FAILURE; + } + } } - - // set slave replication entries - vector lines; - char line[200]; - string buf; - while (file.getline(line, 200)) - { - buf = line; - string::size_type pos = buf.find("server-id",0); - if ( pos != string::npos ) { - buf = "server-id = " + slaveID; - } - - // set local query flag if on pm - if ( (PMwithUM == "y") && config.moduleType() == "pm" ) - { - pos = buf.find("# infinidb_local_query=1",0); - if ( pos != string::npos ) { - buf = "infinidb_local_query=1"; - } - } - else - { // disable, if needed - pos = buf.find("infinidb_local_query=1",0); - if ( pos != string::npos ) { - buf = "# infinidb_local_query=1"; - } - } - -// pos = buf.find("binlog_format=ROW",0); -// if ( pos != string::npos && pos == 0 ) { -// buf = "# binlog_format=ROW"; -// } - - //output to temp file - lines.push_back(buf); - } - - file.close(); - unlink (mycnfFile.c_str()); - ofstream newFile (mycnfFile.c_str()); - //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0664); - - copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); - newFile.close(); - - close(fd); -// } - -/* if ( type == "disable" ) - { - // set master replication entries - vector lines; - char line[200]; - string buf; - while (file.getline(line, 200)) - { - buf = line; - string::size_type pos = buf.find("server-id",0); - if ( pos != string::npos ) { - string::size_type pos1 = buf.find("1",pos); - if ( pos1 == string::npos ) { - buf = "# server-id = 1"; - } - } - - pos = buf.find("log-bin=mysql-bin",0); - if ( pos != string::npos ) { - buf = "# log-bin=mysql-bin"; - } - - pos = buf.find("binlog_format=ROW",0); - if ( pos != string::npos && pos == 0 ) { - buf = "# binlog_format=ROW"; - } - - pos = buf.find("infinidb_local_query=1",0); - if ( pos != string::npos && pos == 0) { - buf = "# infinidb_local_query=1"; - } - - pos = buf.find("# relay-log",0); - if ( pos != string::npos ) { - buf = buf; - } - else - { - pos = buf.find("relay-log",0); - if ( pos != string::npos ) { - buf = "# " + buf; - } - } - - pos = buf.find("slave-skip-errors=all",0); - if ( pos != string::npos ) { - buf = "# slave-skip-errors=all"; - } - - //output to temp file - lines.push_back(buf); - } - - file.close(); - unlink (mycnfFile.c_str()); - ofstream newFile (mycnfFile.c_str()); - - //create new file - int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0664); - - copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); - newFile.close(); - - close(fd); + //output to temp file + lines.push_back(buf); } -*/ + + file.close(); + unlink (mycnfFile.c_str()); + ofstream newFile (mycnfFile.c_str()); + + //create new file + int fd = open(mycnfFile.c_str(), O_RDWR|O_CREAT, 0664); + + copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); + newFile.close(); + + close(fd); + // set owner and permission string cmd = "chmod 664 " + mycnfFile + " >/dev/null 2>&1"; if ( !rootUser) @@ -4884,19 +4772,64 @@ int ProcessMonitor::changeMyCnf(std::string type) system(cmd.c_str()); - // restart mysql - try { - oam.actionMysqlCalpont(MYSQL_RESTART); - sleep(5); // give after mysql restart - } - catch(...) - {} - log.writeLog(__LINE__, "changeMyCnf function successfully completed", LOG_TYPE_DEBUG); return oam::API_SUCCESS; } +/****************************************************************************************** +* @brief runMariaDBCommandLine +* +* purpose: run MariaDB Command Line script +* +******************************************************************************************/ +int ProcessMonitor::runMariaDBCommandLine(std::string command) +{ + Oam oam; + + log.writeLog(__LINE__, "runMariaDBCommandLine function called: cmd = " + command, LOG_TYPE_DEBUG); + + // mysql port number + string MySQLPort; + try { + oam.getSystemConfig("MySQLPort", MySQLPort); + } + catch(...) { + MySQLPort = "3306"; + } + + if ( MySQLPort.empty() ) + MySQLPort = "3306"; + + string cmd = startup::StartUp::installDir() + "/bin/mariadb-command-line.sh --installdir=" + startup::StartUp::installDir() + " --command='" + command + "' --port=" + MySQLPort + " > /tmp/mariadb-command-line.sh.log 2>&1"; + + log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); + + system(cmd.c_str()); + + string logFile = "/tmp/mariadb-command-line.sh.log"; + + if (oam.checkLogStatus(logFile, "ERROR 1045") ) { + log.writeLog(__LINE__, "mariadb-command-line.sh: MySQL Password Error, check .my.cnf", LOG_TYPE_ERROR); + return oam::API_FAILURE; + } + else + { + if (oam.checkLogStatus(logFile, "OK")) + { + log.writeLog(__LINE__, "mariadb-command-line.sh: Successful return", LOG_TYPE_DEBUG); + return oam::API_SUCCESS; + } + else + { + log.writeLog(__LINE__, "mariadb-command-line.sh: Error return, check log /tmp/mariadb-command-line.sh.log", LOG_TYPE_ERROR); + return oam::API_FAILURE; + } + } + + return oam::API_FAILURE; +} + /****************************************************************************************** * @brief runMasterRep * @@ -4950,7 +4883,8 @@ int ProcessMonitor::runMasterRep(std::string& masterLogFile, std::string& master string moduleType = systemModuleTypeConfig.moduletypeconfig[i].ModuleType; - if ( (PMwithUM == "n") && (moduleType == "pm") && ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) + if ( ( (PMwithUM == "n") && (moduleType == "pm") ) && + ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) continue; HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); @@ -5246,7 +5180,8 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string moduleType = systemModuleTypeConfig.moduletypeconfig[i].ModuleType; - if ( (PMwithUM == "n") && (moduleType == "pm") && ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) + if ( ( (PMwithUM == "n") && (moduleType == "pm") ) && + ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM) ) continue; DeviceNetworkList::iterator pt = systemModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.begin(); @@ -5291,7 +5226,7 @@ int ProcessMonitor::runMasterDist(std::string& password, std::string& slaveModul string cmd = startup::StartUp::installDir() + "/bin/rsync.sh " + ipAddr + " " + password + " " + startup::StartUp::installDir() + " 1 > /tmp/master-dist_" + slaveModule + ".log"; system(cmd.c_str()); - log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); + log.writeLog(__LINE__, "cmd = " + cmd, LOG_TYPE_DEBUG); string logFile = "/tmp/master-dist_" + slaveModule + ".log"; if (!oam.checkLogStatus(logFile, "FAILED")) diff --git a/procmon/processmonitor.h b/procmon/processmonitor.h index 366217c84..e952936eb 100644 --- a/procmon/processmonitor.h +++ b/procmon/processmonitor.h @@ -494,6 +494,11 @@ public: */ int changeMyCnf(std::string type); + /** + *@brief run MariaDB Command Line script + */ + int runMariaDBCommandLine(std::string command); + /** *@brief run Master Replication script */ From bc7dbb316919917d053796ad3caeeeb0f95f311d Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 28 Nov 2017 13:24:18 -0600 Subject: [PATCH 2/2] MCOL-942 - backported mysql rep fixes from 1.1 - newscript --- oam/install_scripts/mariadb-command-line.sh | 59 +++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 oam/install_scripts/mariadb-command-line.sh diff --git a/oam/install_scripts/mariadb-command-line.sh b/oam/install_scripts/mariadb-command-line.sh new file mode 100644 index 000000000..bb2750581 --- /dev/null +++ b/oam/install_scripts/mariadb-command-line.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# $Id$ +# +# generic MariaDB Columnstore Command Line script. +# +# Notes: This script gets run by ProcMon during installs and upgrades: + +# check log for error +checkForError() { + grep ERROR /tmp/mariadb-command-line.log > /tmp/error.check + if [ `cat /tmp/error.check | wc -c` -ne 0 ]; then + echo "ERROR: check log file: /tmp/mariadb-command-line.log" + rm -f /tmp/error.check + exit 1 + fi + rm -f /tmp/error.check +} + +prefix=/usr/local +installdir=$prefix/mariadb/columnstore +pwprompt= +for arg in "$@"; do + if [ `expr -- "$arg" : '--command='` -eq 10 ]; then + command1="`echo $arg | awk -F= '{print $2}'`" + command2="`echo $arg | awk -F= '{print $3}'`" + command=$command1"="$command2 + elif [ `expr -- "$arg" : '--installdir='` -eq 13 ]; then + installdir="`echo $arg | awk -F= '{print $2}'`" + prefix=`dirname $installdir` + elif [ `expr -- "$arg" : '--port='` -eq 7 ]; then + port="`echo $arg | awk -F= '{print $2}'`" + fi +done + +test -f $installdir/post/functions && . $installdir/post/functions + + +>/tmp/mariadb-command-line.log + +# +# Run command +# +echo "Run command" >>/tmp/mariadb-command-line.log +cat >/tmp/mariadb-command-line.sql <> /tmp/mariadb-command-line.log +$installdir/mysql/bin/mysql \ + --defaults-extra-file=$installdir/mysql/my.cnf \ + --user=root \ + calpontsys < /tmp/mariadb-command-line.sql >> /tmp/mariadb-command-line.log 2>&1 + +checkForError + +#alls good, 'OK' for success +echo "OK" +exit 0