database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

MCOL-1370 - auto-failure, dont switch ebs when detahc fails

Browse Source
This commit is contained in:
david hill
2018-05-30 15:46:58 -05:00
parent 12f2e112e2
commit 4fe399e5c0
3 changed files with 100 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
oam/oamcpp/liboamcpp.cpp
View File

@@ -5477,6 +5477,21 @@ namespace oam
exceptionControl("autoMovePmDbroot", API_INVALID_PARAMETER);
}
//detach first to make sure DBS can be detach before trying to move to another pm
try
{
typedef std::vector<string> dbrootList;
dbrootList dbrootlist;
dbrootlist.push_back(itoa(dbrootID));
amazonDetach(dbrootlist);
}
catch (exception& )
{
writeLog("ERROR: amazonDetach failure", LOG_TYPE_ERROR );
exceptionControl("autoMovePmDbroot", API_DETACH_FAILURE);
}
//get dbroot id for other PMs
systemStorageInfo_t t;
DeviceDBRootList moduledbrootlist;
@@ -9644,6 +9659,69 @@ namespace oam
}
/***************************************************************************
*
* Function: amazonDetach
*
* Purpose: Amazon EC2 volume deattach needed
*
****************************************************************************/
void Oam::amazonDetach(dbrootList dbrootConfigList)
{
//if amazon cloud with external volumes, do the detach/attach moves
string cloud;
string DBRootStorageType;
try {
getSystemConfig("Cloud", cloud);
getSystemConfig("DBRootStorageType", DBRootStorageType);
}
catch(...) {}
if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") &&
DBRootStorageType == "external" )
{
writeLog("amazonDetach function started ", LOG_TYPE_DEBUG );
dbrootList::iterator pt3 = dbrootConfigList.begin();
for( ; pt3 != dbrootConfigList.end() ; pt3++)
{
string dbrootid = *pt3;
string volumeNameID = "PMVolumeName" + dbrootid;
string volumeName = oam::UnassignedName;
string deviceNameID = "PMVolumeDeviceName" + dbrootid;
string deviceName = oam::UnassignedName;
try {
getSystemConfig( volumeNameID, volumeName);
getSystemConfig( deviceNameID, deviceName);
}
catch(...)
{}
if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName )
{
cout << " ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName << endl;
writeLog("ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR );
exceptionControl("amazonDetach", API_INVALID_PARAMETER);
}
//send msg to to-pm to umount volume
int returnStatus = sendMsgToProcMgr(UNMOUNT, dbrootid, FORCEFUL, ACK_YES);
if (returnStatus != API_SUCCESS) {
writeLog("ERROR: amazonDetach, umount failed on " + dbrootid, LOG_TYPE_ERROR );
}
if (!detachEC2Volume(volumeName)) {
cout << " ERROR: amazonDetach, detachEC2Volume failed on " + volumeName << endl;
writeLog("ERROR: amazonDetach, detachEC2Volume failed on " + volumeName , LOG_TYPE_ERROR );
exceptionControl("amazonDetach", API_FAILURE);
}
writeLog("amazonDetach, detachEC2Volume passed on " + volumeName , LOG_TYPE_DEBUG );
}
}
}
/***************************************************************************
*
* Function: amazonReattach
*
@@ -9736,6 +9814,7 @@ namespace oam
}
}
/***************************************************************************
*
* Function: mountDBRoot

2
oam/oamcpp/liboamcpp.h
View File

@@ -229,6 +229,7 @@ namespace oam
API_CONN_REFUSED,
API_CANCELLED,
API_STILL_WORKING,
API_DETACH_FAILURE,
API_MAX
};
@@ -2432,6 +2433,7 @@ namespace oam
void amazonReattach(std::string toPM, dbrootList dbrootConfigList, bool attach = false);
void mountDBRoot(dbrootList dbrootConfigList, bool mount = true);
void amazonDetach(dbrootList dbrootConfigList);
/**
*@brief gluster control

20
procmgr/main.cpp
View File

@@ -1922,7 +1922,7 @@ void pingDeviceThread()
if ( PrimaryUMModuleName == moduleName )
downPrimaryUM = true;
// if not disabled and amazon, skip
// if disabled, skip
if (opState != oam::AUTO_DISABLED )
{
//Log failure, issue alarm, set moduleOpState
@@ -1968,6 +1968,7 @@ void pingDeviceThread()
if ( ( moduleName.find("pm") == 0 && !amazon && ( DBRootStorageType != "internal") ) ||
( moduleName.find("pm") == 0 && amazon && downActiveOAMModule ) ||
( moduleName.find("pm") == 0 && amazon && AmazonPMFailover == "y") ) {
string error;
try {
log.writeLog(__LINE__, "Call autoMovePmDbroot", LOG_TYPE_DEBUG);
oam.autoMovePmDbroot(moduleName);
@@ -1984,6 +1985,23 @@ void pingDeviceThread()
{
log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR);
}
if ( error == OAM::API_DETACH_FAILURE )
{
processManager.setModuleState(moduleName, oam::AUTO_DISABLED);
// resume the dbrm
oam.dbrmctl("resume");
log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
//enable query stats
dbrm.setSystemQueryReady(true);
//set query system state ready
processManager.setQuerySystemState(true);
break;
}
}
}