From 9454f6e7f619a8eb42e65cf2f3026e0d1f70c259 Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 26 Jan 2018 14:54:48 -0600 Subject: [PATCH] MCOL-1034 - incorporated Bens changes into latest branch --- procmgr/main.cpp | 136 ++++++++----------------------------- procmgr/processmanager.cpp | 56 ++++++--------- 2 files changed, 50 insertions(+), 142 deletions(-) diff --git a/procmgr/main.cpp b/procmgr/main.cpp index 81e1593af..a5ebb9183 100644 --- a/procmgr/main.cpp +++ b/procmgr/main.cpp @@ -1588,14 +1588,17 @@ void pingDeviceThread() if ( moduleName.substr(0,MAX_MODULE_TYPE_SIZE) == "pm" ) processManager.setStandbyModule(moduleName); } - DBRootConfigList::iterator pt = dbrootConfigList.begin(); - if (( DBRootStorageType == "DataRedundancy") && (*pt == 1)) + if ((moduleName.find("pm") == 0) && (dbrootConfigList.size() > 0)) { - log.writeLog(__LINE__, "stopModule, " + config.moduleName(), LOG_TYPE_DEBUG); - processManager.stopModule(config.moduleName(), oam::FORCEFUL, false); - processManager.switchParentOAMModule(moduleName); - processManager.stopProcess(config.moduleName(), "ProcessManager", oam::FORCEFUL, true); - break; + DBRootConfigList::iterator pt = dbrootConfigList.begin(); + if (( DBRootStorageType == "DataRedundancy") && (*pt == 1)) + { + log.writeLog(__LINE__, "stopModule, " + config.moduleName(), LOG_TYPE_DEBUG); + processManager.stopModule(config.moduleName(), oam::FORCEFUL, false); + processManager.switchParentOAMModule(moduleName); + processManager.stopProcess(config.moduleName(), "ProcessManager", oam::FORCEFUL, true); + break; + } } } else { @@ -1627,22 +1630,13 @@ void pingDeviceThread() oam.dbrmctl("resume"); log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG); + //set recycle process + processManager.recycleProcess(moduleName); + //distribute config file processManager.distributeConfigFile("system"); sleep(1); - // if a PM module was started successfully, restart ACTIVE DBRM(s), ExeMgr(s) / mysqld - if( moduleName.find("pm") == 0 ) { - processManager.restartProcessType("DBRMControllerNode", moduleName); - processManager.restartProcessType("DBRMWorkerNode"); - processManager.stopProcessType("DDLProc"); - processManager.stopProcessType("DMLProc"); - processManager.stopProcessType("ExeMgr"); - processManager.restartProcessType("PrimProc"); - sleep(1); - processManager.restartProcessType("ExeMgr"); - } - string moduleType = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); if ( MySQLRep == "y" ) { @@ -1660,22 +1654,6 @@ void pingDeviceThread() processManager.setMySQLReplication(devicenetworklist, oam::UnassignedName, false, true); } } - else - { - if( moduleName.find("pm") == 0 ) { - processManager.restartProcessType("mysql", moduleName); - sleep(1); - } - } - - // if a PM module was started successfully, DMLProc/DDLProc - if( moduleName.find("pm") == 0 ) { - processManager.restartProcessType("WriteEngineServer"); - sleep(1); - processManager.restartProcessType("DDLProc"); - sleep(1); - processManager.restartProcessType("DMLProc"); - } //enable query stats dbrm.setSystemQueryReady(true); @@ -1684,79 +1662,6 @@ void pingDeviceThread() processManager.setQuerySystemState(true); processManager.setSystemState(oam::ACTIVE); - - //reset standby module - string newStandbyModule = processManager.getStandbyModule(); - - //send message to start new Standby Process-Manager, if needed - if ( !newStandbyModule.empty() && newStandbyModule != "NONE") { - processManager.setStandbyModule(newStandbyModule); - } - else - { - Config* sysConfig = Config::makeConfig(); - - // clear Standby OAM Module - sysConfig->setConfig("SystemConfig", "StandbyOAMModuleName", oam::UnassignedName); - sysConfig->setConfig("ProcStatusControlStandby", "IPAddr", oam::UnassignedIpAddr); - - //update Calpont Config table - try { - sysConfig->write(); - } - catch(...) - { - log.writeLog(__LINE__, "ERROR: sysConfig->write", LOG_TYPE_ERROR); - } - } - - if ( moduletypeconfig.RunType == SIMPLEX ) { - //start SIMPLEX runtype processes on a SIMPLEX runtype module - string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); - DeviceNetworkList::iterator pt = moduletypeconfig.ModuleNetworkList.begin(); - for( ; pt != moduletypeconfig.ModuleNetworkList.end() ; pt++) - { - string launchModuleName = (*pt).DeviceName; - string launchModuletype = launchModuleName.substr(0,MAX_MODULE_TYPE_SIZE); - if ( moduletype != launchModuletype ) - continue; - - //skip if active pm module (local module) - if ( launchModuleName == config.moduleName() ) - continue; - - //check if module is active before starting any SIMPLEX STANDBY apps - try{ - int launchopState = oam::ACTIVE; - bool degraded; - oam.getModuleStatus(launchModuleName, launchopState, degraded); - - if (launchopState != oam::ACTIVE && launchopState != oam::STANDBY ) { - continue; - } - } - catch (exception& ex) - { -// string error = ex.what(); -// log.writeLog(__LINE__, "EXCEPTION ERROR on : " + error, LOG_TYPE_ERROR); - } - catch(...) - { -// log.writeLog(__LINE__, "EXCEPTION ERROR on getModuleStatus on module " + moduleName + ": Caught unknown exception!", LOG_TYPE_ERROR); - } - - int status; - log.writeLog(__LINE__, "Starting up STANDBY process on module " + launchModuleName, LOG_TYPE_DEBUG); - for ( int j = 0 ; j < 20 ; j ++ ) - { - status = processManager.startModule(launchModuleName, oam::FORCEFUL, oam::AUTO_OFFLINE); - if ( status == API_SUCCESS) - break; - } - log.writeLog(__LINE__, "pingDeviceThread: ACK received from '" + launchModuleName + "' Process-Monitor, return status = " + oam.itoa(status), LOG_TYPE_DEBUG); - } - } - //clear count moduleInfoList[moduleName] = 0; } @@ -2198,6 +2103,21 @@ void pingDeviceThread() //start SIMPLEX runtype processes on a SIMPLEX runtype module string moduletype = moduleName.substr(0,MAX_MODULE_TYPE_SIZE); + if ( MySQLRep == "y" ) { + if ( moduletype == "um" || + ( moduletype == "pm" && config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) || + ( moduletype == "pm" && PMwithUM == "y") ) { + + //setup MySQL Replication for started modules + log.writeLog(__LINE__, "Setup MySQL Replication for module recovering from outage on " + moduleName, LOG_TYPE_DEBUG); + DeviceNetworkList devicenetworklist; + DeviceNetworkConfig devicenetworkconfig; + devicenetworkconfig.DeviceName = moduleName; + devicenetworklist.push_back(devicenetworkconfig); + processManager.setMySQLReplication(devicenetworklist); + } + } + try{ oam.getSystemConfig(moduletype, moduletypeconfig); } diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 5a2739358..3eed6727f 100755 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -3438,40 +3438,27 @@ void ProcessManager::recycleProcess(string module) } catch(...) {} + //restart ExeMgrs/mysql if module is a pm - if ( moduleType == "pm" ) { - restartProcessType("DBRMControllerNode", module); - restartProcessType("DBRMWorkerNode"); + restartProcessType("DBRMControllerNode", module); + restartProcessType("DBRMWorkerNode"); + if( PrimaryUMModuleName == module) + { stopProcessType("DDLProc"); stopProcessType("DMLProc"); - stopProcessType("ExeMgr"); - restartProcessType("PrimProc"); - sleep(1); - restartProcessType("ExeMgr"); - sleep(1); - restartProcessType("mysql"); - } - else - { - restartProcessType("DBRMControllerNode", module); - restartProcessType("DBRMWorkerNode"); - restartProcessType("ExeMgr"); - } - if ( PrimaryUMModuleName == module ) - { - restartProcessType("DDLProc", module); - sleep(1); - restartProcessType("DMLProc", module); } + stopProcessType("ExeMgr"); + restartProcessType("PrimProc"); + sleep(1); + restartProcessType("ExeMgr"); + sleep(1); + restartProcessType("mysql"); - if( moduleType == "pm" && PrimaryUMModuleName != module) - { - restartProcessType("WriteEngineServer"); - sleep(1); - restartProcessType("DDLProc"); - sleep(1); - restartProcessType("DMLProc", module); - } + restartProcessType("WriteEngineServer"); + sleep(1); + restartProcessType("DDLProc",module); + sleep(1); + restartProcessType("DMLProc",module); return; } @@ -4263,7 +4250,6 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski SystemProcessStatus systemprocessstatus; ProcessStatus processstatus; int retStatus = API_SUCCESS; - bool setPMProcIPs = true; log.writeLog(__LINE__, "restartProcessType: Restart all " + processName, LOG_TYPE_DEBUG); @@ -4311,7 +4297,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski ( systemprocessstatus.processstatus[i].ProcessOpState == oam::COLD_STANDBY && !manualFlag ) ) continue; - if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) && setPMProcIPs ) + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { string procModuleType = systemprocessstatus.processstatus[i].Module.substr(0,MAX_MODULE_TYPE_SIZE); if ( procModuleType == "pm" && PMwithUM == "y" ) @@ -4339,11 +4325,13 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski // if DDL or DMLProc, change IP Address if ( retStatus == oam::API_SUCCESS ) { - if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) && setPMProcIPs ) + sleep(5); + ProcessStatus procstat; + oam.getProcessStatus(processName, systemprocessstatus.processstatus[i].Module, procstat); + if ( (processName.find("DDLProc") == 0 || processName.find("DMLProc") == 0) ) { processManager.setPMProcIPs(systemprocessstatus.processstatus[i].Module, processName); - setPMProcIPs = false; - continue; + break; } } }