database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1049
dmlproc/batchinsertprocessor.cpp
View File

File diff suppressed because it is too large Load Diff

96
dmlproc/batchinsertprocessor.h
View File

@ -38,59 +38,59 @@
#include "dbrm.h"
#include "batchloader.h"
#include "../writeengine/client/we_clients.h"
namespace dmlprocessor
namespace dmlprocessor
{
class BatchInsertProc
{
public:
typedef std::queue<messageqcpp::ByteStream > pkg_type;
typedef boost::shared_ptr<pkg_type> SP_PKG;
typedef std::vector<BRM::BulkSetHWMArg> BulkSetHWMArgs;
BatchInsertProc(bool isAutocommitOn, uint32_t tableOid, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm);
BatchInsertProc(const BatchInsertProc& rhs);
~BatchInsertProc();
uint64_t grabTableLock(int32_t sessionId);
SP_PKG getInsertQueue ();
uint32_t getNumDBRoots();
void setLastPkg (bool lastPkg);
void addPkg(messageqcpp::ByteStream & insertBs);
messageqcpp::ByteStream getPkg();
void setError(int errorCode, std::string errMsg);
void getError(int & errorCode, std::string & errMsg);
int sendPkg(int pmId);
void buildPkg(messageqcpp::ByteStream& bs);
void buildLastPkg(messageqcpp::ByteStream& bs);
void sendFirstBatch();
void sendNextBatch();
void sendlastBatch();
void collectHwm();
void setHwm();
void receiveAllMsg();
void receiveOutstandingMsg();
typedef std::queue<messageqcpp::ByteStream > pkg_type;
typedef boost::shared_ptr<pkg_type> SP_PKG;
typedef std::vector<BRM::BulkSetHWMArg> BulkSetHWMArgs;
BatchInsertProc(bool isAutocommitOn, uint32_t tableOid, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm);
BatchInsertProc(const BatchInsertProc& rhs);
~BatchInsertProc();
uint64_t grabTableLock(int32_t sessionId);
SP_PKG getInsertQueue ();
uint32_t getNumDBRoots();
void setLastPkg (bool lastPkg);
void addPkg(messageqcpp::ByteStream& insertBs);
messageqcpp::ByteStream getPkg();
void setError(int errorCode, std::string errMsg);
void getError(int& errorCode, std::string& errMsg);
int sendPkg(int pmId);
void buildPkg(messageqcpp::ByteStream& bs);
void buildLastPkg(messageqcpp::ByteStream& bs);
void sendFirstBatch();
void sendNextBatch();
void sendlastBatch();
void collectHwm();
void setHwm();
void receiveAllMsg();
void receiveOutstandingMsg();
private:
SP_PKG fInsertPkgQueue;
boost::condition condvar;
execplan::CalpontSystemCatalog::SCN fTxnid;
int fErrorCode;
std::string fErrMsg;
bool fLastpkg;
bool fIsAutocommitOn;
uint32_t fTableOid;
uint64_t fUniqueId;
BRM::DBRM* fDbrm;
WriteEngine::WEClients* fWEClient;
oam::OamCache *fOamcache;
std::vector<uint32_t> fPMs; //active PMs
batchloader::BatchLoader* fBatchLoader;
std::map<unsigned, bool> fPmState;
uint32_t fCurrentPMid;
boost::shared_ptr<messageqcpp::ByteStream> bsIn;
messageqcpp::ByteStream::byte tmp8;
messageqcpp::ByteStream::quadbyte tmp32;
std::vector<BulkSetHWMArgs> fHwmArgsAllPms;
uint64_t fTableLockid;
SP_PKG fInsertPkgQueue;
boost::condition condvar;
execplan::CalpontSystemCatalog::SCN fTxnid;
int fErrorCode;
std::string fErrMsg;
bool fLastpkg;
bool fIsAutocommitOn;
uint32_t fTableOid;
uint64_t fUniqueId;
BRM::DBRM* fDbrm;
WriteEngine::WEClients* fWEClient;
oam::OamCache* fOamcache;
std::vector<uint32_t> fPMs; //active PMs
batchloader::BatchLoader* fBatchLoader;
std::map<unsigned, bool> fPmState;
uint32_t fCurrentPMid;
boost::shared_ptr<messageqcpp::ByteStream> bsIn;
messageqcpp::ByteStream::byte tmp8;
messageqcpp::ByteStream::quadbyte tmp32;
std::vector<BulkSetHWMArgs> fHwmArgsAllPms;
uint64_t fTableLockid;
};
} // namespace dmlprocessor

819
dmlproc/dmlproc.cpp
View File

@ -86,392 +86,432 @@ namespace fs = boost::filesystem;
namespace
{
DistributedEngineComm *Dec;
DistributedEngineComm* Dec;
void added_a_pm(int)
{
LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DMLProc caught SIGHUP. Resetting connections");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_DEBUG, msg, logid);
LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("DMLProc caught SIGHUP. Resetting connections");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_DEBUG, msg, logid);
Dec->Setup();
// MCOL-140 clear the waiting queue as all transactions are probably going to fail
PackageHandler::clearTableAccess();
Dec->Setup();
// MCOL-140 clear the waiting queue as all transactions are probably going to fail
PackageHandler::clearTableAccess();
logger.logMessage(LOG_TYPE_DEBUG, msg, logid);
logger.logMessage(LOG_TYPE_DEBUG, msg, logid);
//WriteEngine::WEClients::instance(WriteEngine::WEClients::DMLPROC)->Setup();
logger.logMessage(LOG_TYPE_DEBUG, msg, logid);
//WriteEngine::WEClients::instance(WriteEngine::WEClients::DMLPROC)->Setup();
logger.logMessage(LOG_TYPE_DEBUG, msg, logid);
}
int toInt(const string& val)
{
if (val.length() == 0)
return -1;
return static_cast<int>(Config::fromText(val));
if (val.length() == 0)
return -1;
return static_cast<int>(Config::fromText(val));
}
class CleanUpThread
{
public:
CleanUpThread() {}
~CleanUpThread() {}
void operator()()
{
ddlcleanuputil::ddl_cleanup();
return;
}
public:
CleanUpThread() {}
~CleanUpThread() {}
void operator()()
{
ddlcleanuputil::ddl_cleanup();
return;
}
};
// This function rolls back any active transactions in case of an abnormal shutdown.
void rollbackAll(DBRM* dbrm)
void rollbackAll(DBRM* dbrm)
{
Oam oam;
try {
alarmmanager::ALARMManager alarmMgr;
alarmMgr.sendAlarmReport("System", oam::ROLLBACK_FAILURE, alarmmanager::CLEAR);
}
catch (...)
{}
//Log a message in info.log
logging::Message::Args args;
try
{
alarmmanager::ALARMManager alarmMgr;
alarmMgr.sendAlarmReport("System", oam::ROLLBACK_FAILURE, alarmmanager::CLEAR);
}
catch (...)
{}
//Log a message in info.log
logging::Message::Args args;
logging::Message message(2);
args.add("DMLProc starts rollbackAll.");
message.format( args );
logging::LoggingID lid(20);
logging::LoggingID lid(20);
logging::MessageLog ml(lid);
ml.logInfoMessage( message );
boost::shared_array<BRM::SIDTIDEntry> activeTxns;
BRM::TxnID txnID;
SessionManager sessionManager;
int rc = 0;
rc = dbrm->isReadWrite();
if (rc != 0 )
throw std::runtime_error("Rollback will be deferred due to DBRM is in read only state.");
dbrm->setSystemReady(false);
//DDL clean up thread
thread_group tg;
tg.create_thread(CleanUpThread());
std::vector<TableLockInfo> tableLocks;
try
{
tableLocks = dbrm->getAllTableLocks();
}
catch (std::exception&)
{
throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE));
}
// If there are tables to rollback, set to ROLLBACK_INIT.
// This tells ProcMgr that we are rolling back and will be
// a while. A message to this effect should be printed.
if (tableLocks.size() > 0)
{
oam.processInitComplete("DMLProc", oam::ROLLBACK_INIT);
}
boost::shared_array<BRM::SIDTIDEntry> activeTxns;
BRM::TxnID txnID;
SessionManager sessionManager;
int rc = 0;
rc = dbrm->isReadWrite();
if (rc != 0 )
throw std::runtime_error("Rollback will be deferred due to DBRM is in read only state.");
dbrm->setSystemReady(false);
//DDL clean up thread
thread_group tg;
tg.create_thread(CleanUpThread());
std::vector<TableLockInfo> tableLocks;
try
{
tableLocks = dbrm->getAllTableLocks();
}
catch (std::exception&)
{
throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE));
}
// If there are tables to rollback, set to ROLLBACK_INIT.
// This tells ProcMgr that we are rolling back and will be
// a while. A message to this effect should be printed.
if (tableLocks.size() > 0)
{
oam.processInitComplete("DMLProc", oam::ROLLBACK_INIT);
}
uint64_t uniqueId = dbrm->getUnique64();
RollbackTransactionProcessor rollbackProcessor(dbrm);
std::string errorMsg;
unsigned int i = 0;
BRM::TxnID txnId;
ostringstream oss;
uint64_t uniqueId = dbrm->getUnique64();
RollbackTransactionProcessor rollbackProcessor(dbrm);
std::string errorMsg;
unsigned int i = 0;
BRM::TxnID txnId;
ostringstream oss;
oss << "DMLProc will rollback " << tableLocks.size() << " tables.";
//cout << oss.str() << endl;
logging::Message::Args args5;
logging::Message message5(2);
args5.add(oss.str());
message5.format( args5 );
ml.logInfoMessage( message5 );
OamCache * oamcache = OamCache::makeOamCache();
OamCache::dbRootPMMap_t dbRootPMMap = oamcache->getDBRootToPMMap();
int errorTxn = 0;
for ( i=0; i < tableLocks.size(); i++)
{
if (tableLocks[i].ownerTxnID > 0) //transaction rollback
{
ostringstream oss;
oss << "DMLProc is rolling back transaction " <<tableLocks[i].ownerTxnID;
//cout << oss.str() << endl;
logging::Message::Args args1;
logging::Message message2(2);
args1.add(oss.str());
message2.format( args1 );
ml.logInfoMessage( message2 );
dbrm->invalidateUncommittedExtentLBIDs(tableLocks[i].ownerTxnID);
uint32_t sessionid = 0;
txnId.id = tableLocks[i].ownerTxnID;
txnId.valid = true;
rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg);
if ( rc == 0 )
{
ostringstream oss;
oss << "DMLProc rolled back transaction " <<tableLocks[i].ownerTxnID << " and is releasing table lock id " << tableLocks[i].id;
logging::Message::Args args3;
logging::Message message3(2);
args3.add(oss.str());
message3.format( args3 );
ml.logInfoMessage( message3 );
//@Bug 5285 get the process name to see whether calling bulkrollback is necessary.
string::size_type namelen = tableLocks[i].ownerName.find_first_of(" ");
if (namelen != string::npos) {
rollbackProcessor.rollBackBatchAutoOnTransaction(uniqueId, txnId, sessionid, tableLocks[i].tableOID, errorMsg);
}
logging::logCommand(0, tableLocks[i].ownerTxnID, "ROLLBACK;");
//release table lock if not DDLProc
if (tableLocks[i].ownerName == "DDLProc")
continue;
bool lockReleased = true;
try {
lockReleased = dbrm->releaseTableLock(tableLocks[i].id);
}
catch (std::exception&)
{
throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE));
}
if (lockReleased)
{
sessionManager.rolledback(txnId);
ostringstream oss;
oss << "DMLProc rolled back transaction " <<tableLocks[i].ownerTxnID << " and table lock id " << tableLocks[i].id << " is released.";
//cout << oss.str() << endl;
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
else
{
ostringstream oss;
oss << "DMLProc rolled back transaction " <<tableLocks[i].ownerTxnID << " and table lock id " << tableLocks[i].id << " is not released.";
//cout << oss.str() << endl;
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
}
else
{
//@Bug 4524 still need to set readonly as transaction information is lost during system restart.
ostringstream oss;
oss << " problem with rollback transaction " << tableLocks[i].ownerTxnID << "and DBRM is setting to readonly and table lock is not released: " << errorMsg;
rc = dbrm->setReadOnly(true);
//Raise an alarm
try {
alarmmanager::ALARMManager alarmMgr;
alarmMgr.sendAlarmReport("System", oam::ROLLBACK_FAILURE, alarmmanager::SET);
//cout << oss.str() << endl;
logging::Message::Args args5;
logging::Message message5(2);
args5.add(oss.str());
message5.format( args5 );
ml.logInfoMessage( message5 );
OamCache* oamcache = OamCache::makeOamCache();
OamCache::dbRootPMMap_t dbRootPMMap = oamcache->getDBRootToPMMap();
int errorTxn = 0;
for ( i = 0; i < tableLocks.size(); i++)
{
if (tableLocks[i].ownerTxnID > 0) //transaction rollback
{
ostringstream oss;
oss << "DMLProc is rolling back transaction " << tableLocks[i].ownerTxnID;
//cout << oss.str() << endl;
logging::Message::Args args1;
logging::Message message2(2);
args1.add(oss.str());
message2.format( args1 );
ml.logInfoMessage( message2 );
dbrm->invalidateUncommittedExtentLBIDs(tableLocks[i].ownerTxnID);
uint32_t sessionid = 0;
txnId.id = tableLocks[i].ownerTxnID;
txnId.valid = true;
rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg);
if ( rc == 0 )
{
ostringstream oss;
oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and is releasing table lock id " << tableLocks[i].id;
logging::Message::Args args3;
logging::Message message3(2);
args3.add(oss.str());
message3.format( args3 );
ml.logInfoMessage( message3 );
//@Bug 5285 get the process name to see whether calling bulkrollback is necessary.
string::size_type namelen = tableLocks[i].ownerName.find_first_of(" ");
if (namelen != string::npos)
{
rollbackProcessor.rollBackBatchAutoOnTransaction(uniqueId, txnId, sessionid, tableLocks[i].tableOID, errorMsg);
}
logging::logCommand(0, tableLocks[i].ownerTxnID, "ROLLBACK;");
//release table lock if not DDLProc
if (tableLocks[i].ownerName == "DDLProc")
continue;
bool lockReleased = true;
try
{
lockReleased = dbrm->releaseTableLock(tableLocks[i].id);
}
catch (std::exception&)
{
throw std::runtime_error(IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE));
}
if (lockReleased)
{
sessionManager.rolledback(txnId);
ostringstream oss;
oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and table lock id " << tableLocks[i].id << " is released.";
//cout << oss.str() << endl;
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
else
{
ostringstream oss;
oss << "DMLProc rolled back transaction " << tableLocks[i].ownerTxnID << " and table lock id " << tableLocks[i].id << " is not released.";
//cout << oss.str() << endl;
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
}
else
{
//@Bug 4524 still need to set readonly as transaction information is lost during system restart.
ostringstream oss;
oss << " problem with rollback transaction " << tableLocks[i].ownerTxnID << "and DBRM is setting to readonly and table lock is not released: " << errorMsg;
rc = dbrm->setReadOnly(true);
//Raise an alarm
try
{
alarmmanager::ALARMManager alarmMgr;
alarmMgr.sendAlarmReport("System", oam::ROLLBACK_FAILURE, alarmmanager::SET);
}
catch (...)
{}
//Log to critical log
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logCriticalMessage( message6 );
errorTxn = tableLocks[i].ownerTxnID;
}
}
else if (tableLocks[i].ownerName == "WriteEngineServer") //redistribution
{
//Just release the table lock
bool lockReleased = true;
try {
lockReleased = dbrm->releaseTableLock(tableLocks[i].id);
}
catch (std::exception& ex)
{
ostringstream oss;
oss << "DMLProc didn't release table lock id " <<tableLocks[i].id << " for redistribution due to " << ex.what();
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
if (lockReleased)
{
ostringstream oss;
oss << "DMLProc released table lock id " <<tableLocks[i].id << " for redistribution.";
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
}
else //bulkrollback
{
//change owner, still keep transaction id to -1, and session id to -1.
bool lockReleased = true;
try {
rollbackProcessor.processBulkRollback(tableLocks[i], dbrm, uniqueId, dbRootPMMap, lockReleased);
ostringstream oss;
oss << "DMLProc started bulkrollback on table OID " <<tableLocks[i].tableOID << " and table lock id " << tableLocks[i].id
<< " finished and tablelock is released.";
//cout << oss.str() << endl;
logging::Message::Args args4;
logging::Message message4(2);
args4.add(oss.str());
message4.format( args4 );
ml.logInfoMessage( message4 );
}
catch (std::exception& ex)
{
ostringstream oss;
oss << "DMLProc started bulkrollback on table OID " <<tableLocks[i].tableOID << " and table lock id " << tableLocks[i].id
<< " failed:" << ex.what();
if (lockReleased)
oss << " but the tablelock is released due to it is in CLEANUP state";
else
oss << " and tablelock is not released.";
//cout << oss.str() << endl;
logging::Message::Args args3;
logging::Message message3(2);
args3.add(oss.str());
message3.format( args3 );
ml.logWarningMessage( message3 );
}
}
}
// Clear out the session manager session list of sessions / transactions.
//Take care of any transaction left from create table as there is no table lock information
set<VER_t> txnList;
rc = dbrm->getCurrentTxnIDs(txnList);
if(txnList.size() > 0) {
oam.processInitComplete("DMLProc", oam::ROLLBACK_INIT);
ostringstream oss;
oss << "DMLProc will rollback " << txnList.size() << " transactions.";
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
set<VER_t>::const_iterator curTxnID;
for(curTxnID = txnList.begin(); curTxnID != txnList.end(); ++curTxnID) {
//Log to critical log
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logCriticalMessage( message6 );
errorTxn = tableLocks[i].ownerTxnID;
}
}
else if (tableLocks[i].ownerName == "WriteEngineServer") //redistribution
{
//Just release the table lock
bool lockReleased = true;
try
{
lockReleased = dbrm->releaseTableLock(tableLocks[i].id);
}
catch (std::exception& ex)
{
ostringstream oss;
oss << "DMLProc didn't release table lock id " << tableLocks[i].id << " for redistribution due to " << ex.what();
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
if (lockReleased)
{
ostringstream oss;
oss << "DMLProc released table lock id " << tableLocks[i].id << " for redistribution.";
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
}
}
else //bulkrollback
{
//change owner, still keep transaction id to -1, and session id to -1.
bool lockReleased = true;
try
{
rollbackProcessor.processBulkRollback(tableLocks[i], dbrm, uniqueId, dbRootPMMap, lockReleased);
ostringstream oss;
oss << "DMLProc started bulkrollback on table OID " << tableLocks[i].tableOID << " and table lock id " << tableLocks[i].id
<< " finished and tablelock is released.";
//cout << oss.str() << endl;
logging::Message::Args args4;
logging::Message message4(2);
args4.add(oss.str());
message4.format( args4 );
ml.logInfoMessage( message4 );
}
catch (std::exception& ex)
{
ostringstream oss;
oss << "DMLProc started bulkrollback on table OID " << tableLocks[i].tableOID << " and table lock id " << tableLocks[i].id
<< " failed:" << ex.what();
if (lockReleased)
oss << " but the tablelock is released due to it is in CLEANUP state";
else
oss << " and tablelock is not released.";
//cout << oss.str() << endl;
logging::Message::Args args3;
logging::Message message3(2);
args3.add(oss.str());
message3.format( args3 );
ml.logWarningMessage( message3 );
}
}
}
// Clear out the session manager session list of sessions / transactions.
//Take care of any transaction left from create table as there is no table lock information
set<VER_t> txnList;
rc = dbrm->getCurrentTxnIDs(txnList);
if (txnList.size() > 0)
{
oam.processInitComplete("DMLProc", oam::ROLLBACK_INIT);
ostringstream oss;
oss << "DMLProc will rollback " << txnList.size() << " transactions.";
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
set<VER_t>::const_iterator curTxnID;
for (curTxnID = txnList.begin(); curTxnID != txnList.end(); ++curTxnID)
{
dbrm->invalidateUncommittedExtentLBIDs(*curTxnID);
txnId.id = *curTxnID;
txnId.valid = true;
uint32_t sessionid = 0;
ostringstream oss;
oss << "DMLProc will roll back transaction " <<*curTxnID;
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg);
if ( rc == 0 )
{
logging::logCommand(0, *curTxnID, "ROLLBACK;");
ostringstream oss;
oss << " DMLProc rolled back transaction " << *curTxnID;
//Log to warning log
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logInfoMessage( message6 );
}
else
{
//@Bug 4524 still need to set readonly as transaction information is lost during system restart.
ostringstream oss;
oss << " problem with rollback transaction " << txnId.id << "and DBRM is setting to readonly and table lock is not released: " << errorMsg;
rc = dbrm->setReadOnly(true);
//Raise an alarm
try {
alarmmanager::ALARMManager alarmMgr;
alarmMgr.sendAlarmReport("System", oam::ROLLBACK_FAILURE, alarmmanager::SET);
txnId.id = *curTxnID;
txnId.valid = true;
uint32_t sessionid = 0;
ostringstream oss;
oss << "DMLProc will roll back transaction " << *curTxnID;
logging::Message::Args args2;
logging::Message message2(2);
args2.add(oss.str());
message2.format( args2 );
ml.logInfoMessage( message2 );
rc = rollbackProcessor.rollBackTransaction(uniqueId, txnId, sessionid, errorMsg);
if ( rc == 0 )
{
logging::logCommand(0, *curTxnID, "ROLLBACK;");
ostringstream oss;
oss << " DMLProc rolled back transaction " << *curTxnID;
//Log to warning log
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logInfoMessage( message6 );
}
else
{
//@Bug 4524 still need to set readonly as transaction information is lost during system restart.
ostringstream oss;
oss << " problem with rollback transaction " << txnId.id << "and DBRM is setting to readonly and table lock is not released: " << errorMsg;
rc = dbrm->setReadOnly(true);
//Raise an alarm
try
{
alarmmanager::ALARMManager alarmMgr;
alarmMgr.sendAlarmReport("System", oam::ROLLBACK_FAILURE, alarmmanager::SET);
}
catch (...)
{}
//Log to critical log
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logCriticalMessage( message6 );
errorTxn = *curTxnID;
}
}
}
int len;
activeTxns = sessionManager.SIDTIDMap(len);
//@Bug 1627 Don't start DMLProc if either controller or worker node is not up
if ( activeTxns == NULL )
{
std::string err = "DBRM problem is encountered.";
throw std::runtime_error(err);
}
for(int i=0; i < len; i++) {
txnID = activeTxns[i].txnid;
if (txnID.id != errorTxn)
{
sessionManager.rolledback(txnID);
oss << " DMLProc released transaction " << txnID.id;
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logInfoMessage( message6 );
}
}
// Clear out the DBRM.
dbrm->clear();
//Log to critical log
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logCriticalMessage( message6 );
errorTxn = *curTxnID;
}
}
}
//Flush the cache
cacheutils::flushPrimProcCache();
//Log a message in info.log
logging::Message::Args args1;
int len;
activeTxns = sessionManager.SIDTIDMap(len);
//@Bug 1627 Don't start DMLProc if either controller or worker node is not up
if ( activeTxns == NULL )
{
std::string err = "DBRM problem is encountered.";
throw std::runtime_error(err);
}
for (int i = 0; i < len; i++)
{
txnID = activeTxns[i].txnid;
if (txnID.id != errorTxn)
{
sessionManager.rolledback(txnID);
oss << " DMLProc released transaction " << txnID.id;
logging::Message::Args args6;
logging::Message message6(2);
args6.add(oss.str());
message6.format( args6 );
ml.logInfoMessage( message6 );
}
}
// Clear out the DBRM.
dbrm->clear();
//Flush the cache
cacheutils::flushPrimProcCache();
//Log a message in info.log
logging::Message::Args args1;
logging::Message message1(2);
args1.add("DMLProc finished rollbackAll.");
message1.format( args1 );
ml.logInfoMessage( message1 );
dbrm->setSystemReady(true);
dbrm->setSystemReady(true);
}
void setupCwd()
{
string workdir = Config::makeConfig()->getConfig("SystemConfig", "WorkingDir");
if (workdir.length() == 0)
workdir = ".";
(void)chdir(workdir.c_str());
if (access(".", W_OK) != 0)
(void)chdir("/tmp");
string workdir = Config::makeConfig()->getConfig("SystemConfig", "WorkingDir");
if (workdir.length() == 0)
workdir = ".";
(void)chdir(workdir.c_str());
if (access(".", W_OK) != 0)
(void)chdir("/tmp");
}
} // Namewspace
int main(int argc, char* argv[])
{
// get and set locale language
string systemLang = "C";
string systemLang = "C";
BRM::DBRM dbrm;
BRM::DBRM dbrm;
Oam oam;
//BUG 5362
systemLang = funcexp::utf8::idb_setlocale();
//BUG 5362
systemLang = funcexp::utf8::idb_setlocale();
Config* cf = Config::makeConfig();
@ -483,57 +523,60 @@ int main(int argc, char* argv[])
// in the main program
idbdatafile::IDBPolicy::configIDBPolicy();
#endif
try
{
// At first we set to BUSY_INIT
oam.processInitComplete("DMLProc", oam::BUSY_INIT);
// At first we set to BUSY_INIT
oam.processInitComplete("DMLProc", oam::BUSY_INIT);
}
catch (...)
{
}
//@Bug 1627
try
{
rollbackAll(&dbrm); // Rollback any
}
catch ( std::exception &e )
{
//@Bug 2299 Set DMLProc process to fail and log a message
try
{
oam.processInitFailure();
}
catch (...)
//@Bug 1627
try
{
rollbackAll(&dbrm); // Rollback any
}
catch ( std::exception& e )
{
//@Bug 2299 Set DMLProc process to fail and log a message
try
{
}
logging::Message::Args args;
oam.processInitFailure();
}
catch (...)
{
}
logging::Message::Args args;
logging::Message message(2);
args.add("DMLProc failed to start due to :");
args.add(e.what());
args.add(e.what());
message.format( args );
logging::LoggingID lid(20);
logging::LoggingID lid(20);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
cerr << "DMLProc failed to start due to : " << e.what() << endl;
return 1;
}
int temp;
int serverThreads = 10;
int serverQueueSize = 50;
const string DMLProc("DMLProc");
temp = toInt(cf->getConfig(DMLProc, "ServerThreads"));
if (temp > 0)
serverThreads = temp;
cerr << "DMLProc failed to start due to : " << e.what() << endl;
return 1;
}
temp = toInt(cf->getConfig(DMLProc, "ServerQueueSize"));
if (temp > 0)
serverQueueSize = temp;
int temp;
int serverThreads = 10;
int serverQueueSize = 50;
const string DMLProc("DMLProc");
temp = toInt(cf->getConfig(DMLProc, "ServerThreads"));
if (temp > 0)
serverThreads = temp;
temp = toInt(cf->getConfig(DMLProc, "ServerQueueSize"));
if (temp > 0)
serverQueueSize = temp;
bool rootUser = true;
@ -541,33 +584,38 @@ int main(int argc, char* argv[])
//check if root-user
int user;
user = getuid();
if (user != 0)
rootUser = false;
#endif
//read and cleanup port before trying to use
try {
try
{
string port = cf->getConfig(DMLProc, "Port");
string cmd = "fuser -k " + port + "/tcp >/dev/null 2>&1";
if ( !rootUser)
cmd = "sudo fuser -k " + port + "/tcp >/dev/null 2>&1";
(void)::system(cmd.c_str());
}
catch(...)
catch (...)
{
}
DMLServer dmlserver(serverThreads, serverQueueSize,&dbrm);
ResourceManager *rm = ResourceManager::instance();
DMLServer dmlserver(serverThreads, serverQueueSize, &dbrm);
ResourceManager* rm = ResourceManager::instance();
// jobstepThreadPool is used by other processes. We can't call
// resourcemanaager (rm) functions during the static creation of threadpool
// because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton).
// From the pools perspective, it has no idea if it is ExeMgr doing the
// jobstepThreadPool is used by other processes. We can't call
// resourcemanaager (rm) functions during the static creation of threadpool
// because rm has a "isExeMgr" flag that is set upon creation (rm is a singleton).
// From the pools perspective, it has no idea if it is ExeMgr doing the
// creation, so it has no idea which way to set the flag. So we set the max here.
JobStep::jobstepThreadPool.setMaxThreads(rm->getJLThreadPoolSize());
JobStep::jobstepThreadPool.setName("DMLProcJobList");
// if (rm->getJlThreadPoolDebug() == "Y" || rm->getJlThreadPoolDebug() == "y")
// {
// JobStep::jobstepThreadPool.setDebug(true);
@ -582,16 +630,17 @@ int main(int argc, char* argv[])
catch (...)
{
}
Dec = DistributedEngineComm::instance(rm);
Dec = DistributedEngineComm::instance(rm);
#ifndef _MSC_VER
/* set up some signal handlers */
/* set up some signal handlers */
struct sigaction ign;
memset(&ign, 0, sizeof(ign));
ign.sa_handler = added_a_pm;
sigaction(SIGHUP, &ign, 0);
ign.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &ign, 0);
sigaction(SIGHUP, &ign, 0);
ign.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &ign, 0);
#endif
dmlserver.start();

2771
dmlproc/dmlprocessor.cpp
View File

File diff suppressed because it is too large Load Diff

207
dmlproc/dmlprocessor.h
View File

@ -45,39 +45,58 @@
#include "batchinsertprocessor.h"
#include "querytele.h"
template<typename T, typename Container=std::deque<T> >
class iterable_queue : public std::queue<T,Container>
template<typename T, typename Container = std::deque<T> >
class iterable_queue : public std::queue<T, Container>
{
public:
typedef typename Container::iterator iterator;
typedef typename Container::const_iterator const_iterator;
iterator begin() { return this->c.begin(); }
iterator end() { return this->c.end(); }
const_iterator begin() const { return this->c.begin(); }
const_iterator end() const { return this->c.end(); }
iterator find(T t)
{
iterator it;
for (it = begin(); it != end(); ++it)
{
if (*it == t)
{
break;
}
}
return it;
}
iterator erase(typename Container::iterator it) { return this->c.erase(it); }
iterator erase(T t)
{
iterator it = find(t);
if (it != end())
{
erase(it);
}
return it;
}
iterator begin()
{
return this->c.begin();
}
iterator end()
{
return this->c.end();
}
const_iterator begin() const
{
return this->c.begin();
}
const_iterator end() const
{
return this->c.end();
}
iterator find(T t)
{
iterator it;
for (it = begin(); it != end(); ++it)
{
if (*it == t)
{
break;
}
}
return it;
}
iterator erase(typename Container::iterator it)
{
return this->c.erase(it);
}
iterator erase(T t)
{
iterator it = find(t);
if (it != end())
{
erase(it);
}
return it;
}
};
namespace dmlprocessor
@ -88,37 +107,49 @@ namespace dmlprocessor
class DMLServer
{
public:
DMLServer(int packageMaxThreads, int packageWorkQueueSize, BRM::DBRM * aDbrm);
DMLServer(int packageMaxThreads, int packageWorkQueueSize, BRM::DBRM* aDbrm);
~DMLServer() { }
~DMLServer() { }
void start(); //Does not return
void start(); //Does not return
/** @brief get the dml package thread pool size
*/
inline int getPackageThreadPoolSize() const { return fPackageMaxThreads; }
inline int getPackageThreadPoolSize() const
{
return fPackageMaxThreads;
}
/** @brief set the dml package thread pool size
*
* @param threadPoolSize the maximum number of threads to process dml packages
*/
inline void setPackageThreadPoolSize( int threadPoolSize ) { fPackageMaxThreads = threadPoolSize; }
inline void setPackageThreadPoolSize( int threadPoolSize )
{
fPackageMaxThreads = threadPoolSize;
}
/** @brief get the maximum number of dml packages allowed in the work queue
*/
inline int getPackageWorkQueueSize() const { return fPackageWorkQueueSize; }
inline int getPackageWorkQueueSize() const
{
return fPackageWorkQueueSize;
}
/** @brief set the dml package work queue size
*
* @param workQueueSize the maximum number of dml packages in the work queue
*/
inline void setPackageWorkQueueSize( int workQueueSize ) { fPackageWorkQueueSize = workQueueSize; }
inline void setPackageWorkQueueSize( int workQueueSize )
{
fPackageWorkQueueSize = workQueueSize;
}
private:
//not copyable
DMLServer(const DMLServer& rhs);
DMLServer& operator=(const DMLServer& rhs);
//not copyable
DMLServer(const DMLServer& rhs);
DMLServer& operator=(const DMLServer& rhs);
/** @brief the thread pool for processing dml packages
*/
@ -128,7 +159,7 @@ private:
int fPackageWorkQueueSize; /** @brief max number of packages waiting in the work queue */
boost::scoped_ptr<messageqcpp::MessageQueueServer> fMqServer;
BRM::DBRM* fDbrm;
BRM::DBRM* fDbrm;
};
/** @brief Thread to process a single dml package.
@ -138,30 +169,36 @@ class PackageHandler
{
public:
PackageHandler(const messageqcpp::IOSocket& ios, boost::shared_ptr<messageqcpp::ByteStream> bs,
uint8_t packageType, joblist::DistributedEngineComm *ec, bool concurrentSuport, uint64_t maxDeleteRows,
uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM * aDbrm,
const querytele::QueryTeleClient& qtc, boost::shared_ptr<execplan::CalpontSystemCatalog> csc);
~PackageHandler();
uint8_t packageType, joblist::DistributedEngineComm* ec, bool concurrentSuport, uint64_t maxDeleteRows,
uint32_t sessionID, execplan::CalpontSystemCatalog::SCN txnId, BRM::DBRM* aDbrm,
const querytele::QueryTeleClient& qtc, boost::shared_ptr<execplan::CalpontSystemCatalog> csc);
~PackageHandler();
void run();
void rollbackPending();
execplan::CalpontSystemCatalog::SCN getTxnid() {return fTxnid;}
uint32_t getSessionID() {return fSessionID;}
void rollbackPending();
execplan::CalpontSystemCatalog::SCN getTxnid()
{
return fTxnid;
}
uint32_t getSessionID()
{
return fSessionID;
}
private:
messageqcpp::IOSocket fIos;
boost::shared_ptr<messageqcpp::ByteStream> fByteStream;
boost::scoped_ptr<dmlpackageprocessor::DMLPackageProcessor> fProcessor;
messageqcpp::ByteStream::quadbyte fPackageType;
joblist::DistributedEngineComm *fEC;
bool fConcurrentSupport;
joblist::DistributedEngineComm* fEC;
bool fConcurrentSupport;
uint64_t fMaxDeleteRows;
uint32_t fSessionID;
uint32_t fTableOid;
uint32_t fTableOid;
execplan::CalpontSystemCatalog::SCN fTxnid;
execplan::SessionManager sessionManager;
BRM::DBRM *fDbrm;
BRM::DBRM* fDbrm;
querytele::QueryTeleClient fQtc;
boost::shared_ptr<execplan::CalpontSystemCatalog> fcsc;
@ -173,17 +210,17 @@ private:
// for the table oid. If found, wait until it is no onger here.
// If this transactionID (SCN) is < the transactionID in the table, don't delay
// and hope for the best, as we're already out of order.
// When the VSS is engineered to handle transactions out of order, all MCOL-140
// When the VSS is engineered to handle transactions out of order, all MCOL-140
// code is to be removed.
int synchTableAccess();
int releaseTableAccess();
int forceReleaseTableAccess();
typedef iterable_queue<execplan::CalpontSystemCatalog::SCN> tableAccessQueue_t;
int synchTableAccess();
int releaseTableAccess();
int forceReleaseTableAccess();
typedef iterable_queue<execplan::CalpontSystemCatalog::SCN> tableAccessQueue_t;
static std::map<uint32_t, tableAccessQueue_t> tableOidMap;
static boost::condition_variable tableOidCond;
static boost::mutex tableOidMutex;
public:
static int clearTableAccess();
static int clearTableAccess();
};
/** @brief processes dml packages as they arrive
@ -202,48 +239,48 @@ public:
*/
void operator()();
static void log(const std::string& msg, logging::LOG_TYPE level);
static void log(const std::string& msg, logging::LOG_TYPE level);
private:
messageqcpp::IOSocket fIos;
execplan::SessionManager sessionManager;
boost::shared_ptr<execplan::CalpontSystemCatalog> csc;
BRM::DBRM* fDbrm;
querytele::QueryTeleClient fQtc;
bool fConcurrentSupport;
messageqcpp::IOSocket fIos;
// A map to hold pointers to all active PackageProcessors
typedef std::map<uint32_t, boost::shared_ptr<PackageHandler> > PackageHandlerMap_t;
static PackageHandlerMap_t packageHandlerMap;
static boost::mutex packageHandlerMapLock;
execplan::SessionManager sessionManager;
boost::shared_ptr<execplan::CalpontSystemCatalog> csc;
BRM::DBRM* fDbrm;
querytele::QueryTeleClient fQtc;
bool fConcurrentSupport;
// A map to hold pointers to all active PackageProcessors
typedef std::map<uint32_t, boost::shared_ptr<PackageHandler> > PackageHandlerMap_t;
static PackageHandlerMap_t packageHandlerMap;
static boost::mutex packageHandlerMapLock;
// A map to hold pointers to all BatchInsertProc object
static std::map<uint32_t, BatchInsertProc*> batchinsertProcessorMap;
static boost::mutex batchinsertProcessorMapLock;
// A map to hold pointers to all BatchInsertProc object
static std::map<uint32_t, BatchInsertProc*> batchinsertProcessorMap;
static boost::mutex batchinsertProcessorMapLock;
friend struct CancellationThread;
friend class PackageHandler;
friend class PackageHandler;
};
class RollbackTransactionProcessor : public dmlpackageprocessor::DMLPackageProcessor
{
public:
RollbackTransactionProcessor(BRM::DBRM* aDbrm) : DMLPackageProcessor(aDbrm, 1) {}
RollbackTransactionProcessor(BRM::DBRM* aDbrm) : DMLPackageProcessor(aDbrm, 1) {}
/** @brief process an Rollback transactions
*
* @param cpackage the UpdateDMLPackage to process
*/
inline DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage)
{
DMLResult result;
result.result = NO_ERROR;
return result;
}
void processBulkRollback (BRM::TableLockInfo lockInfo, BRM::DBRM * dbrm, uint64_t uniqueId,
oam::OamCache::dbRootPMMap_t& dbRootPMMap, bool & lockReleased);
inline DMLResult processPackage(dmlpackage::CalpontDMLPackage& cpackage)
{
DMLResult result;
result.result = NO_ERROR;
return result;
}
void processBulkRollback (BRM::TableLockInfo lockInfo, BRM::DBRM* dbrm, uint64_t uniqueId,
oam::OamCache::dbRootPMMap_t& dbRootPMMap, bool& lockReleased);
protected:
};

10
dmlproc/dmlresultbuffer.cpp
View File

@ -28,13 +28,13 @@ namespace dmlprocessor
{
DMLResultBuffer::DMLResultBuffer()
:fBufferSize(1), fFull(0)
: fBufferSize(1), fFull(0)
{
}
DMLResultBuffer::DMLResultBuffer(int bufferSize)
:fBufferSize(bufferSize), fFull(0)
: fBufferSize(bufferSize), fFull(0)
{
}
@ -47,11 +47,13 @@ DMLResultBuffer::~DMLResultBuffer()
void DMLResultBuffer::put(dmlpackageprocessor::DMLPackageProcessor::DMLResult result, int sessionID)
{
scoped_lock lock(fMutex);
if (fFull == fBufferSize)
{
while (fFull == fBufferSize)
fCond.wait(lock);
}
ResultPair rp;
rp.result = result;
rp.sessionID = sessionID;
@ -64,11 +66,13 @@ void DMLResultBuffer::put(dmlpackageprocessor::DMLPackageProcessor::DMLResult re
DMLResultBuffer::ResultPair DMLResultBuffer::get()
{
scoped_lock lk(fMutex);
if (fFull == 0)
{
while (fFull == 0)
fCond.wait(lk);
}
ResultPair rp = fResultBuffer[0];
fResultBuffer.pop_front();
--fFull;

13
dmlproc/dmlresultbuffer.h
View File

@ -48,9 +48,9 @@ public:
};
/** @brief ctor
*/
*/
DMLResultBuffer();
/** @brief ctor
*
*/
@ -60,13 +60,16 @@ public:
*
*/
~DMLResultBuffer();
/** @brief set the size of the buffer
*
* @param bufferSize the size of the buffer
*/
inline void setBufferSize(int bufferSize) {fBufferSize=bufferSize;}
inline void setBufferSize(int bufferSize)
{
fBufferSize = bufferSize;
}
/** @brief put results in the buffer
*
*/

28
dmlproc/resource.h
View File

@ -1,14 +1,14 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by DMLProc.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by DMLProc.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif