database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-07 09:01:10 +03:00
Code Activity

Tentative fix for procmon d/ling DBRM files into data1. That screws things

Browse Source 
up on shared filesystems, and storagemanager, which has sort-of a shared
filesystem view.
This commit is contained in:
Patrick LeBlanc
2019-07-19 09:54:54 -05:00
parent 3118acc3d4
commit d0a576483e
3 changed files with 39 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
procmgr/processmanager.cpp
View File

@ -9236,7 +9236,7 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa
} }
catch (...) catch (...)
{ {
log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknow exception", LOG_TYPE_ERROR); log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknown exception", LOG_TYPE_ERROR);
returnStatus = oam::API_FAILURE; returnStatus = oam::API_FAILURE;
} }
@ -9307,7 +9307,7 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa
} }
catch (...) catch (...)
{ {
log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknow exception", LOG_TYPE_ERROR); log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknown exception", LOG_TYPE_ERROR);
returnStatus = oam::API_FAILURE; returnStatus = oam::API_FAILURE;
} }
@ -9342,7 +9342,7 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa
} }
catch (...) catch (...)
{ {
log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknow exception", LOG_TYPE_ERROR); log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknown exception", LOG_TYPE_ERROR);
pthread_mutex_unlock(&THREAD_LOCK); pthread_mutex_unlock(&THREAD_LOCK);
return oam::API_FAILURE; return oam::API_FAILURE;
} }
@ -9416,7 +9416,7 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa
} }
catch (...) catch (...)
{ {
log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknow exception", LOG_TYPE_ERROR); log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknown exception", LOG_TYPE_ERROR);
pthread_mutex_unlock(&THREAD_LOCK); pthread_mutex_unlock(&THREAD_LOCK);
return oam::API_FAILURE; return oam::API_FAILURE;
} }
@ -9436,7 +9436,7 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa
} }
catch (...) catch (...)
{ {
log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknow exception", LOG_TYPE_ERROR); log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknown exception", LOG_TYPE_ERROR);
pthread_mutex_unlock(&THREAD_LOCK); pthread_mutex_unlock(&THREAD_LOCK);
return oam::API_FAILURE; return oam::API_FAILURE;
} }
@ -9453,7 +9453,7 @@ int ProcessManager::getDBRMData(messageqcpp::IOSocket fIos, std::string moduleNa
} }
catch (...) catch (...)
{ {
log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknow exception", LOG_TYPE_ERROR); log.writeLog(__LINE__, "EXCEPTION ERROR on cfIos.write: Unknown exception", LOG_TYPE_ERROR);
returnStatus = oam::API_FAILURE; returnStatus = oam::API_FAILURE;
} }

34
procmon/processmonitor.cpp
View File

@ -22,6 +22,10 @@
***************************************************************************/ ***************************************************************************/
#include <boost/scoped_ptr.hpp> #include <boost/scoped_ptr.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "columnstoreversion.h" #include "columnstoreversion.h"
#include "IDBDataFile.h" #include "IDBDataFile.h"
@ -40,6 +44,7 @@ using namespace logging;
using namespace config; using namespace config;
using namespace idbdatafile; using namespace idbdatafile;
namespace bf = boost::filesystem;
extern string systemOAM; extern string systemOAM;
extern string dm_server; extern string dm_server;
@ -2574,7 +2579,7 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName,
system(cmd.c_str()); system(cmd.c_str());
// if Non Parent OAM Module, get the dbmr data from Parent OAM Module // if Non Parent OAM Module, get the dbmr data from Parent OAM Module
if ( !gOAMParentModuleFlag && !HDFS && DBRootStorageType != "storagemanager") if ( !gOAMParentModuleFlag && !HDFS)
{ {
//create temp dbrm directory //create temp dbrm directory
@ -2599,12 +2604,16 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName,
// system(cmd.c_str()); // system(cmd.c_str());
// go request files from parent OAM module // go request files from parent OAM module
if ( getDBRMdata() != oam::API_SUCCESS ) if ( getDBRMdata(&DBRMDir) != oam::API_SUCCESS )
{ {
log.writeLog(__LINE__, "Error: getDBRMdata failed", LOG_TYPE_ERROR); log.writeLog(__LINE__, "Error: getDBRMdata failed", LOG_TYPE_ERROR);
sendAlarm("DBRM", DBRM_LOAD_DATA_ERROR, SET); sendAlarm("DBRM", DBRM_LOAD_DATA_ERROR, SET);
return oam::API_MINOR_FAILURE; return oam::API_MINOR_FAILURE;
} }
// DBRMDir might have changed, so need to change DBRMroot
bf::path tmp(DBRMroot);
tmp = tmp.filename();
DBRMroot = (bf::path(DBRMDir) / tmp).string();
sendAlarm("DBRM", DBRM_LOAD_DATA_ERROR, CLEAR); sendAlarm("DBRM", DBRM_LOAD_DATA_ERROR, CLEAR);
// change DBRMroot to temp DBRMDir path // change DBRMroot to temp DBRMDir path
@ -2680,11 +2689,17 @@ pid_t ProcessMonitor::startProcess(string processModuleType, string processName,
} }
// now delete the dbrm data from local disk // now delete the dbrm data from local disk
if ( !gOAMParentModuleFlag && !HDFS && DBRootStorageType != "storagemanager" && DataRedundancyConfig == "n") if ( !gOAMParentModuleFlag && !HDFS && DataRedundancyConfig == "n")
{ {
IDBFileSystem &fs = IDBPolicy::getFs(DBRMDir);
fs.remove(DBRMDir.c_str());
log.writeLog(__LINE__, "removed downloaded DBRM files at " + DBRMDir, LOG_TYPE_DEBUG);
#if 0
string cmd = "rm -f " + DBRMDir + "/*"; string cmd = "rm -f " + DBRMDir + "/*";
system(cmd.c_str()); system(cmd.c_str());
log.writeLog(__LINE__, "removed DBRM file with command: " + cmd, LOG_TYPE_DEBUG); log.writeLog(__LINE__, "removed DBRM file with command: " + cmd, LOG_TYPE_DEBUG);
#endif
} }
} }
else else
@ -4315,7 +4330,7 @@ int ProcessMonitor::processRestarted( std::string processName, bool manual)
* *
* *
******************************************************************************************/ ******************************************************************************************/
int ProcessMonitor::getDBRMdata() int ProcessMonitor::getDBRMdata(string *path)
{ {
MonitorLog log; MonitorLog log;
@ -4397,6 +4412,10 @@ int ProcessMonitor::getDBRMdata()
log.writeLog(__LINE__, oam.itoa(numFiles), LOG_TYPE_DEBUG); log.writeLog(__LINE__, oam.itoa(numFiles), LOG_TYPE_DEBUG);
bool journalFile = false; bool journalFile = false;
boost::uuids::uuid u = boost::uuids::random_generator()();
bf::path pTmp = bf::path(*path) / boost::uuids::to_string(u);
*path = pTmp.string();
log.writeLog(__LINE__, "Downloading DBRM files to " + *path, LOG_TYPE_DEBUG);
for ( int i = 0 ; i < numFiles ; i ++ ) for ( int i = 0 ; i < numFiles ; i ++ )
{ {
@ -4438,11 +4457,14 @@ int ProcessMonitor::getDBRMdata()
// string temp1 = temp + "data" + fileName.substr(pos1,80); // string temp1 = temp + "data" + fileName.substr(pos1,80);
// fileName = temp1; // fileName = temp1;
// } // }
bf::path pFilename(fileName);
pFilename = pTmp / pFilename.filename();
const char *cFilename = pFilename.string().c_str();
boost::scoped_ptr<IDBDataFile> out(IDBDataFile::open( boost::scoped_ptr<IDBDataFile> out(IDBDataFile::open(
IDBPolicy::getType(fileName.c_str(), IDBPolicy::getType(cFilename,
IDBPolicy::WRITEENG), IDBPolicy::WRITEENG),
fileName.c_str(), "w", 0)); cFilename, "w", 0));
// read file data // read file data
try try

2
procmon/processmonitor.h
View File

@ -478,7 +478,7 @@ public:
*/ */
int updateConfigFile(messageqcpp::ByteStream msg); int updateConfigFile(messageqcpp::ByteStream msg);
int getDBRMdata(); int getDBRMdata(std::string *path);
/** /**
*@brief Send Msg to Process Monitor *@brief Send Msg to Process Monitor