MCOL-1370 - auto-failure, dont switch ebs when detahc fails

oam/oamcpp/liboamcpp.cpp

@@ -5477,6 +5477,21 @@ namespace oam
 			exceptionControl("autoMovePmDbroot", API_INVALID_PARAMETER);
 		}
 
+		//detach first to make sure DBS can be detach before trying to move to another pm
+		try
+		{
+			typedef std::vector<string> dbrootList;
+			dbrootList dbrootlist;
+			dbrootlist.push_back(itoa(dbrootID));
+
+			amazonDetach(dbrootlist);
+		}
+		catch (exception& )
+		{
+			writeLog("ERROR: amazonDetach failure", LOG_TYPE_ERROR );
+			exceptionControl("autoMovePmDbroot", API_DETACH_FAILURE);
+		}
+
 		//get dbroot id for other PMs
 		systemStorageInfo_t t;
 		DeviceDBRootList moduledbrootlist;
@@ -9644,6 +9659,69 @@ namespace oam
 }
 
     /***************************************************************************
+     *
+     * Function:  amazonDetach
+     *
+     * Purpose:   Amazon EC2 volume deattach needed
+     *
+     ****************************************************************************/
+
+    void Oam::amazonDetach(dbrootList dbrootConfigList)
+    {
+		//if amazon cloud with external volumes, do the detach/attach moves
+		string cloud;
+		string DBRootStorageType;
+		try {
+			getSystemConfig("Cloud", cloud);
+			getSystemConfig("DBRootStorageType", DBRootStorageType);
+		}
+		catch(...) {}
+
+		if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && 
+			DBRootStorageType == "external" )
+		{
+			writeLog("amazonDetach function started ", LOG_TYPE_DEBUG );
+
+			dbrootList::iterator pt3 = dbrootConfigList.begin();
+			for( ; pt3 != dbrootConfigList.end() ; pt3++)
+			{
+				string dbrootid = *pt3;
+				string volumeNameID = "PMVolumeName" + dbrootid;
+				string volumeName = oam::UnassignedName;
+				string deviceNameID = "PMVolumeDeviceName" + dbrootid;
+				string deviceName = oam::UnassignedName;
+				try {
+					getSystemConfig( volumeNameID, volumeName);
+					getSystemConfig( deviceNameID, deviceName);
+				}
+				catch(...)
+				{}
+
+				if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName )
+				{
+					cout << "   ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName << endl;
+					writeLog("ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR );
+					exceptionControl("amazonDetach", API_INVALID_PARAMETER);
+				}
+
+				//send msg to to-pm to umount volume
+				int returnStatus = sendMsgToProcMgr(UNMOUNT, dbrootid, FORCEFUL, ACK_YES);
+				if (returnStatus != API_SUCCESS) {
+					writeLog("ERROR: amazonDetach, umount failed on " + dbrootid, LOG_TYPE_ERROR );
+				}
+
+				if (!detachEC2Volume(volumeName)) {
+					cout << "   ERROR: amazonDetach, detachEC2Volume failed on " + volumeName << endl;
+					writeLog("ERROR: amazonDetach, detachEC2Volume failed on " + volumeName , LOG_TYPE_ERROR );
+					exceptionControl("amazonDetach", API_FAILURE);
+				}
+
+				writeLog("amazonDetach, detachEC2Volume passed on " + volumeName , LOG_TYPE_DEBUG );
+			}
+		}
+	}
+
+     /***************************************************************************
      *
      * Function:  amazonReattach
      *
@@ -9736,6 +9814,7 @@ namespace oam
 		}
 	}
 
+
     /***************************************************************************
      *
      * Function:  mountDBRoot

oam/oamcpp/liboamcpp.h

@@ -229,6 +229,7 @@ namespace oam
 		API_CONN_REFUSED,
         API_CANCELLED,
         API_STILL_WORKING,
+	API_DETACH_FAILURE,
         API_MAX
     };
 
@@ -2432,6 +2433,7 @@ namespace oam
 
 		void amazonReattach(std::string toPM, dbrootList dbrootConfigList, bool attach = false);
 		void mountDBRoot(dbrootList dbrootConfigList, bool mount = true);
+		void amazonDetach(dbrootList dbrootConfigList);
 
 		/**
 		*@brief  gluster control

procmgr/main.cpp

@@ -1922,7 +1922,7 @@ void pingDeviceThread()
 								if ( PrimaryUMModuleName == moduleName )
 								    downPrimaryUM = true;
 
-								// if not disabled and amazon, skip
+								// if disabled, skip
 								if (opState != oam::AUTO_DISABLED )
 								{	
 									//Log failure, issue alarm, set moduleOpState 
@@ -1968,6 +1968,7 @@ void pingDeviceThread()
 									if ( ( moduleName.find("pm") == 0 && !amazon && ( DBRootStorageType != "internal") ) ||
 										( moduleName.find("pm") == 0 && amazon && downActiveOAMModule ) ||
 										( moduleName.find("pm") == 0 && amazon && AmazonPMFailover == "y") ) {
+										string error;
 										try {
 											log.writeLog(__LINE__, "Call autoMovePmDbroot", LOG_TYPE_DEBUG);
 											oam.autoMovePmDbroot(moduleName);
@@ -1984,6 +1985,23 @@ void pingDeviceThread()
 										{
 											log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR);
 										}
+										
+										if ( error == OAM::API_DETACH_FAILURE )
+										{
+										      processManager.setModuleState(moduleName, oam::AUTO_DISABLED);
+
+										      // resume the dbrm
+										      oam.dbrmctl("resume");
+										      log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
+
+										      //enable query stats
+										      dbrm.setSystemQueryReady(true);
+
+										      //set query system state ready
+										      processManager.setQuerySystemState(true);
+										      
+										      break;
+										}
 									}
 								}