MCOL-1370

oam/cloud/MCSVolumeCmds.sh

@@ -202,7 +202,7 @@ detachvolume() {
 	checkInfostatus
 	if [ $STATUS == "detaching" ]; then
 		retries=1
-		while [ $retries -ne 60 ]; do
+		while [ $retries -ne 10 ]; do
 			#retry until it's attached
 			$AWSCLI detach-volume --volume-id  $volumeName --region $Region > /tmp/volumeInfo_$volumeName 2>&1
 		
@@ -239,7 +239,7 @@ attachvolume() {
 	checkInfostatus
 	if [ $STATUS == "attaching" -o $STATUS == "already-attached" ]; then
 		retries=1
-		while [ $retries -ne 60 ]; do
+		while [ $retries -ne 10 ]; do
 			#check status until it's attached
 			describevolume
 			if [ $STATUS == "attached" ]; then

oam/oamcpp/liboamcpp.cpp

@@ -5479,7 +5479,7 @@ namespace oam
 
 		//detach first to make sure DBS can be detach before trying to move to another pm
 		DBRootConfigList::iterator pt3 = residedbrootConfigList.begin();
-		for( ; pt3 != residedbrootConfigList.end() ; )
+		for( ; pt3 != residedbrootConfigList.end() ; pt3++ )
 		{
 			int dbrootID = *pt3;
 
@@ -5494,6 +5494,14 @@ namespace oam
 			catch (exception& )
 			{
 				writeLog("ERROR: amazonDetach failure", LOG_TYPE_ERROR );
+				
+				//reattach
+				typedef std::vector<string> dbrootList;
+				dbrootList dbrootlist;
+				dbrootlist.push_back(itoa(dbrootID));
+
+				amazonAttach(residePM, dbrootlist);
+
 				exceptionControl("autoMovePmDbroot", API_DETACH_FAILURE);
 			}
 		}
@@ -5972,9 +5980,8 @@ namespace oam
 		}
 
 		if (!found) {
-			writeLog("ERROR: no dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_ERROR );
+			writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG );
-			cout << "ERROR: no dbroots found in " << fileName << endl;
+			cout << "No dbroots found in " << fileName << endl;
-			exceptionControl("autoUnMovePmDbroot", API_FAILURE);
 		}
 
 		oldFile.close();
@@ -7269,7 +7276,7 @@ namespace oam
 		else
 			return;
 
-		// check if mysql-Capont is installed
+		// check if mysql-Columnstore is installed
 		string mysqlscript = InstallDir + "/mysql/mysql-Columnstore";
 		if (access(mysqlscript.c_str(), X_OK) != 0)
 			return;
@@ -9727,6 +9734,83 @@ namespace oam
 		}
 	}
 
+    /***************************************************************************
+     *
+     * Function:  amazonAttach
+     *
+     * Purpose:   Amazon EC2 volume Attach needed
+     *
+     ****************************************************************************/
+
+    void Oam::amazonAttach(std::string toPM, dbrootList dbrootConfigList)
+    {
+		//if amazon cloud with external volumes, do the detach/attach moves
+		string cloud;
+		string DBRootStorageType;
+		try {
+			getSystemConfig("Cloud", cloud);
+			getSystemConfig("DBRootStorageType", DBRootStorageType);
+		}
+		catch(...) {}
+
+		if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && 
+			DBRootStorageType == "external" )
+		{
+			writeLog("amazonAttach function started ", LOG_TYPE_DEBUG );
+
+			//get Instance Name for to-pm
+			string toInstanceName = oam::UnassignedName;
+			try
+			{
+				ModuleConfig moduleconfig;
+				getSystemConfig(toPM, moduleconfig);
+				HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin();
+				toInstanceName = (*pt1).HostName;
+			}
+			catch(...)
+			{}
+
+			if ( toInstanceName == oam::UnassignedName || toInstanceName.empty() )
+			{
+				cout << "   ERROR: amazonAttach, invalid Instance Name for " << toPM << endl;
+				writeLog("ERROR: amazonAttach, invalid Instance Name " + toPM, LOG_TYPE_ERROR );
+				exceptionControl("amazonAttach", API_INVALID_PARAMETER);
+			}
+
+			dbrootList::iterator pt3 = dbrootConfigList.begin();
+			for( ; pt3 != dbrootConfigList.end() ; pt3++)
+			{
+				string dbrootid = *pt3;
+				string volumeNameID = "PMVolumeName" + dbrootid;
+				string volumeName = oam::UnassignedName;
+				string deviceNameID = "PMVolumeDeviceName" + dbrootid;
+				string deviceName = oam::UnassignedName;
+				try {
+					getSystemConfig( volumeNameID, volumeName);
+					getSystemConfig( deviceNameID, deviceName);
+				}
+				catch(...)
+				{}
+
+				if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName )
+				{
+					cout << "   ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName << endl;
+					writeLog("ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR );
+					exceptionControl("amazonAttach", API_INVALID_PARAMETER);
+				}
+
+				if (!attachEC2Volume(volumeName, deviceName, toInstanceName)) {
+					cout << "   ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName << endl;
+					writeLog("ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName, LOG_TYPE_ERROR );
+					exceptionControl("amazonAttach", API_FAILURE);
+				}
+
+				writeLog("amazonAttach, attachEC2Volume passed on " + volumeName + ":" + toPM, LOG_TYPE_DEBUG );
+			}
+		}
+	}
+
+
      /***************************************************************************
      *
      * Function:  amazonReattach

oam/oamcpp/liboamcpp.h

@@ -2434,6 +2434,7 @@ namespace oam
 		void amazonReattach(std::string toPM, dbrootList dbrootConfigList, bool attach = false);
 		void mountDBRoot(dbrootList dbrootConfigList, bool mount = true);
 		void amazonDetach(dbrootList dbrootConfigList);
+		void amazonAttach(std::string toPM, dbrootList dbrootConfigList);
 
 		/**
 		*@brief  gluster control

procmgr/main.cpp

@@ -1553,7 +1553,7 @@ void pingDeviceThread()
 									processManager.restartProcessType("WriteEngineServer", moduleName);
 
 									//set module to enable state
-									processManager.enableModule(moduleName, oam::AUTO_OFFLINE);
+									processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true);
 
 									downActiveOAMModule = false;
 									int retry;
@@ -1647,7 +1647,7 @@ void pingDeviceThread()
 								}
 								else
 								    //set module to enable state
-								    processManager.enableModule(moduleName, oam::AUTO_OFFLINE);
+								    processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true);
 
 								//restart module processes
 								int retry = 0;

procmgr/processmanager.cpp

@@ -3438,7 +3438,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule)
 	restartProcessType("ExeMgr");
 	sleep(1);
 
-	restartProcessType("mysql");
+	restartProcessType("mysqld");
 
 	restartProcessType("WriteEngineServer");
 	sleep(1);
@@ -3457,7 +3457,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule)
 * purpose:	Clear the Disable State on a specified module
 *
 ******************************************************************************************/
-int ProcessManager::enableModule(string target, int state)
+int ProcessManager::enableModule(string target, int state, bool failover)
 {
 	Oam oam;
 	ModuleConfig moduleconfig;
@@ -3496,6 +3496,7 @@ int ProcessManager::enableModule(string target, int state)
 		setStandbyModule(newStandbyModule);
 
 	//set recycle process
+	if (!failover)
 	      recycleProcess(target);
 	
 	log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG);
@@ -4256,7 +4257,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski
 		PMwithUM = "n";
 	}
 
-	// If mysql is the processName, then send to modules were ExeMgr is running
+	// If mysqld is the processName, then send to modules were ExeMgr is running
 	try
 	{
 		oam.getProcessStatus(systemprocessstatus);
@@ -4267,7 +4268,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski
 			if ( systemprocessstatus.processstatus[i].Module == skipModule )
 				continue;
 
-			if ( processName == "mysql" ) {
+			if ( processName == "mysqld" ) {
 				if ( systemprocessstatus.processstatus[i].ProcessName == "ExeMgr") {
 					ProcessStatus procstat;
 					oam.getProcessStatus("mysqld", systemprocessstatus.processstatus[i].Module, procstat);
@@ -8985,7 +8986,7 @@ int ProcessManager::OAMParentModuleChange()
 						if (systemstatus.SystemOpState == ACTIVE) {
 							log.writeLog(__LINE__, "System Active, restart needed processes", LOG_TYPE_DEBUG);
 
-							processManager.restartProcessType("mysql");
+							processManager.restartProcessType("mysqld");
 							processManager.restartProcessType("ExeMgr");
 							processManager.restartProcessType("WriteEngineServer");
 							processManager.reinitProcessType("DBRMWorkerNode");
@@ -10099,7 +10100,7 @@ void ProcessManager::stopProcessTypes(bool manualFlag)
 	log.writeLog(__LINE__, "stopProcessTypes Called");
 
 	//front-end first
-	processManager.stopProcessType("mysql", manualFlag);
+	processManager.stopProcessType("mysqld", manualFlag);
 	processManager.stopProcessType("DMLProc", manualFlag);
 	processManager.stopProcessType("DDLProc", manualFlag);
 	processManager.stopProcessType("ExeMgr", manualFlag);

procmgr/processmanager.h

@@ -307,7 +307,7 @@ public:
      /**
      *@brief Enable a specified module
      */
-    int enableModule(std::string target, int state);
+    int enableModule(std::string target, int state, bool failover = false);
 
      /**
      *@brief Enable a specified module

procmon/main.cpp

@@ -695,8 +695,8 @@ int main(int argc, char **argv)
 	if ( ret != 0 )
 		log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR);
 
-	//mysql status monitor thread
+	//mysqld status monitor thread
-	if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) ||
+	if ( ( config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) ||
 		(PMwithUM == "y") )
 	{
 
@@ -1127,7 +1127,7 @@ static void mysqlMonitorThread(MonitorConfig config)
 		catch(...)
 		{}
 
-		sleep(10);
+		sleep(5);
 	}
 
 }

procmon/processmonitor.cpp

@@ -457,7 +457,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 					log.writeLog(__LINE__,  "MSG RECEIVED: Stop process request on " + processName);
 					int requestStatus = API_SUCCESS;
 		
-					// check for mysql
+					// check for mysqld
 					if ( processName == "mysqld" ) {
 						try {
 							oam.actionMysqlCalpont(MYSQL_STOP);
@@ -520,7 +520,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 					msg >> manualFlag;
 					log.writeLog(__LINE__, "MSG RECEIVED: Start process request on: " + processName);
 		
-					// check for mysql
+					// check for mysqld
 					if ( processName == "mysqld" ) {
 						try {
 							oam.actionMysqlCalpont(MYSQL_START);
@@ -640,7 +640,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 					log.writeLog(__LINE__,  "MSG RECEIVED: Restart process request on " + processName);
 					int requestStatus = API_SUCCESS;
 
-					// check for mysql restart
+					// check for mysqld restart
 					if ( processName == "mysqld" ) {
 						try {
 							oam.actionMysqlCalpont(MYSQL_RESTART);
@@ -869,7 +869,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 								log.writeLog(__LINE__, "Error running DBRM clearShm", LOG_TYPE_ERROR);
 						}
 
-						//stop the mysql daemon
+						//stop the mysqld daemon
 						try {
 							oam.actionMysqlCalpont(MYSQL_STOP);
 							log.writeLog(__LINE__, "Stop MySQL Process", LOG_TYPE_DEBUG);
@@ -995,12 +995,12 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 				
 					system(cmd.c_str());
 
-					//start the mysql daemon 
+					//start the mysqld daemon 
 					try {
 						oam.actionMysqlCalpont(MYSQL_START);
 					}
 					catch(...)
-					{	// mysql didn't start, return with error
+					{	// mysqld didn't start, return with error
 						log.writeLog(__LINE__, "STARTALL: MySQL failed to start, start-module failure", LOG_TYPE_CRITICAL);
 						
 						ackMsg << (ByteStream::byte) ACK;
@@ -1265,7 +1265,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 					//send down notification
 					oam.sendDeviceNotification(config.moduleName(), MODULE_DOWN);
 
-					//stop the mysql daemon and then columnstore
+					//stop the mysqld daemon and then columnstore
 					try {
 						oam.actionMysqlCalpont(MYSQL_STOP);
 					}
@@ -1444,7 +1444,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 				}
 			}
 
-			// install mysql rpms if being reconfigured as a um
+			// install mysqld rpms if being reconfigured as a um
 			if ( reconfigureModuleName.find("um") != string::npos ) {
 				string cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install >> /tmp/rpminstall";
 				system(cmd.c_str());