database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

clang format apply

Browse Source
This commit is contained in:
Leonid Fedorov
2022-01-21 16:43:49 +00:00
parent 6b6411229f
commit 04752ec546
1376 changed files with 393460 additions and 412662 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
writeengine/server/resource.h
View File

@ -6,9 +6,9 @@
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

204
writeengine/server/we_brmrprtparser.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
/*
* we_brmrprtparser.cpp
@ -35,15 +35,12 @@ using namespace std;
namespace WriteEngine
{
BrmReportParser::BrmReportParser(): fRptFile()
BrmReportParser::BrmReportParser() : fRptFile()
{
}
BrmReportParser::~BrmReportParser()
{
}
//------------------------------------------------------------------------------
@ -53,53 +50,52 @@ BrmReportParser::~BrmReportParser()
// Function is limited to records that are no longer than 255 bytes.
// If a record is longer than 255 bytes, the function misbehaves.
//------------------------------------------------------------------------------
bool BrmReportParser::serialize(std::string RptFileName,
messageqcpp::ByteStream& Bs)
bool BrmReportParser::serialize(std::string RptFileName, messageqcpp::ByteStream& Bs)
{
try
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch (std::exception& ex)
{
cout << "Failed to open BRMRptFile " << RptFileName << endl;
cout << ex.what() << endl;
throw runtime_error(ex.what());
}
try
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch (std::exception& ex)
{
cout << "Failed to open BRMRptFile " << RptFileName << endl;
cout << ex.what() << endl;
throw runtime_error(ex.what());
}
if (fRptFile.good())
{
char aBuff[10240];
unsigned int aLen = 0;
if (fRptFile.good())
{
char aBuff[10240];
unsigned int aLen = 0;
while (fRptFile.good() && !fRptFile.eof())
while (fRptFile.good() && !fRptFile.eof())
{
fRptFile.getline(aBuff, sizeof(aBuff) - 1);
aLen = fRptFile.gcount();
if ((aLen != (sizeof(aBuff) - 1)) && (aLen > 0))
{
// aBuff[aLen-1] = '\n';
// aBuff[aLen]=0;
// cout << "Data Read " << aBuff <<endl;
if (aBuff[0] != '#') // do not serialize comments
{
fRptFile.getline(aBuff, sizeof(aBuff) - 1);
aLen = fRptFile.gcount();
std::string strData = aBuff;
Bs << strData;
}
}
} // while
if ((aLen != (sizeof(aBuff) - 1)) && (aLen > 0))
{
//aBuff[aLen-1] = '\n';
//aBuff[aLen]=0;
//cout << "Data Read " << aBuff <<endl;
if (aBuff[0] != '#') // do not serialize comments
{
std::string strData = aBuff;
Bs << strData;
}
}
}// while
fRptFile.close();
cout << "Closed File : " << RptFileName << " " << errno << endl;
}
else
{
cout << "Failed to open : " << RptFileName << " " << errno << endl;
return false;
}
fRptFile.close();
cout << "Closed File : " << RptFileName << " " << errno << endl;
}
else
{
cout << "Failed to open : " << RptFileName << " " << errno << endl;
return false;
}
return true;
return true;
}
//------------------------------------------------------------------------------
@ -107,76 +103,70 @@ bool BrmReportParser::serialize(std::string RptFileName,
// Serialization is done in 8192 byte blocks instead of by line or by record,
// so that this function will work for both character and binary files.
//------------------------------------------------------------------------------
bool BrmReportParser::serializeBlocks(std::string RptFileName,
messageqcpp::ByteStream& Bs)
bool BrmReportParser::serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs)
{
try
try
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch (std::exception& ex)
{
cout << "Failed to open Report File " << RptFileName << endl;
cout << ex.what() << endl;
return false;
}
if (fRptFile.good())
{
char aBuff[8192];
unsigned int aLen = 0;
std::string strBuff;
while (fRptFile.good())
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch (std::exception& ex)
{
cout << "Failed to open Report File " << RptFileName << endl;
cout << ex.what() << endl;
return false;
fRptFile.read(aBuff, sizeof(aBuff));
aLen = fRptFile.gcount();
if (aLen > 0)
{
strBuff.assign(aBuff, aLen);
Bs << strBuff;
}
}
if (fRptFile.good())
{
char aBuff[8192];
unsigned int aLen = 0;
std::string strBuff;
fRptFile.close();
cout << "Closed Report File : " << RptFileName << endl;
}
else
{
std::ostringstream oss;
oss << "Failed to open Report File " << RptFileName << endl;
cout << oss.str() << endl;
return false;
}
while ( fRptFile.good() )
{
fRptFile.read(aBuff, sizeof(aBuff));
aLen = fRptFile.gcount();
if (aLen > 0)
{
strBuff.assign( aBuff, aLen );
Bs << strBuff;
}
}
fRptFile.close();
cout << "Closed Report File : " << RptFileName << endl;
}
else
{
std::ostringstream oss;
oss << "Failed to open Report File " << RptFileName << endl;
cout << oss.str() << endl;
return false;
}
return true;
return true;
}
void BrmReportParser::unserialize(messageqcpp::ByteStream& Bs)
{
//TODO to be changed. left it here to understand how to implement
/*
ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN);
ReturnedColumn::unserialize(b); // parent class unserialize
b >> (uint32_t&) fOid;
b >> fData;
b >> reinterpret_cast<ByteStream::doublebyte&>(fReturnAll);
b >> (uint32_t&) fSequence;
*/
// TODO to be changed. left it here to understand how to implement
/*
ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN);
ReturnedColumn::unserialize(b); // parent class unserialize
b >> (uint32_t&) fOid;
b >> fData;
b >> reinterpret_cast<ByteStream::doublebyte&>(fReturnAll);
b >> (uint32_t&) fSequence;
*/
std::string aStrLine;
while (Bs.length() > 0)
{
Bs >> aStrLine;
cout << aStrLine;
}
std::string aStrLine;
while (Bs.length() > 0)
{
Bs >> aStrLine;
cout << aStrLine;
}
}
} /* namespace WriteEngine */

26
writeengine/server/we_brmrprtparser.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
/*
* we_brmrprtparser.h
*
@ -52,21 +52,19 @@
namespace WriteEngine
{
class BrmReportParser
{
public:
BrmReportParser();
virtual ~BrmReportParser();
public:
BrmReportParser();
virtual ~BrmReportParser();
public:
bool serialize(std::string RptFileName, messageqcpp::ByteStream& Bs);
bool serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs);
void unserialize(messageqcpp::ByteStream& Bs);
private:
ifstream fRptFile;
public:
bool serialize(std::string RptFileName, messageqcpp::ByteStream& Bs);
bool serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs);
void unserialize(messageqcpp::ByteStream& Bs);
private:
ifstream fRptFile;
};
} /* namespace WriteEngine */

164
writeengine/server/we_cleartablelockcmd.cpp
View File

@ -31,60 +31,52 @@
namespace WriteEngine
{
//------------------------------------------------------------------------------
// Process a bulk rollback request based on input from the specified input
// bytestream object.
//------------------------------------------------------------------------------
int WE_ClearTableLockCmd::processRollback(
messageqcpp::ByteStream& bs,
std::string& errMsg)
int WE_ClearTableLockCmd::processRollback(messageqcpp::ByteStream& bs, std::string& errMsg)
{
uint8_t rc = 0;
errMsg.clear();
uint8_t rc = 0;
errMsg.clear();
try
{
uint32_t tableOID;
uint64_t tableLockID;
std::string tableName;
std::string appName;
try
{
uint32_t tableOID;
uint64_t tableLockID;
std::string tableName;
std::string appName;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processRollback for " << fUserDesc;
bs >> tableLockID;
std::cout << ": tableLock-" << tableLockID;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processRollback for " << fUserDesc;
bs >> tableLockID;
std::cout << ": tableLock-" << tableLockID;
bs >> tableOID;
std::cout << "; tableOID-" << tableOID;
bs >> tableOID;
std::cout << "; tableOID-" << tableOID;
bs >> tableName;
std::cout << "; table-" << tableName;
bs >> tableName;
std::cout << "; table-" << tableName;
bs >> appName;
std::cout << "; app-" << appName << std::endl;
bs >> appName;
std::cout << "; app-" << appName << std::endl;
int we_rc = fWEWrapper.bulkRollback(
tableOID,
tableLockID,
tableName,
appName,
false, // no extra debug logging to the console
errMsg );
int we_rc = fWEWrapper.bulkRollback(tableOID, tableLockID, tableName, appName,
false, // no extra debug logging to the console
errMsg);
if (we_rc != NO_ERROR)
rc = 2;
}
catch (std::exception& ex)
{
std::cout << "ClearTableLockCmd::Rollback exception-" << ex.what() <<
std::endl;
errMsg = ex.what();
rc = 1;
}
if (we_rc != NO_ERROR)
rc = 2;
}
catch (std::exception& ex)
{
std::cout << "ClearTableLockCmd::Rollback exception-" << ex.what() << std::endl;
errMsg = ex.what();
rc = 1;
}
return rc;
return rc;
}
//------------------------------------------------------------------------------
@ -95,61 +87,55 @@ int WE_ClearTableLockCmd::processRollback(
// We only need this for the "successful" case, as the bulk rollback takes
// care of db file cleanup in the "unsuccessful" case.
//------------------------------------------------------------------------------
int WE_ClearTableLockCmd::processCleanup(
messageqcpp::ByteStream& bs,
std::string& errMsg)
int WE_ClearTableLockCmd::processCleanup(messageqcpp::ByteStream& bs, std::string& errMsg)
{
uint8_t rc = 0;
errMsg.clear();
uint8_t rc = 0;
errMsg.clear();
try
try
{
uint32_t tableOID;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processCleanup for " << fUserDesc;
bs >> tableOID;
std::cout << ": tableOID-" << tableOID << std::endl;
// On an HDFS system, this is where we delete any DB files
// (ex: *.orig) that we no longer need.
if ((idbdatafile::IDBPolicy::useHdfs()) && (bs.length() >= sizeof(messageqcpp::ByteStream::byte)))
{
uint32_t tableOID;
messageqcpp::ByteStream::byte deleteHdfsTempDbFiles;
bs >> deleteHdfsTempDbFiles;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processCleanup for " << fUserDesc;
bs >> tableOID;
std::cout << ": tableOID-" << tableOID << std::endl;
if (deleteHdfsTempDbFiles)
{
std::string endDbErrMsg;
ConfirmHdfsDbFile confirmHdfs;
// On an HDFS system, this is where we delete any DB files
// (ex: *.orig) that we no longer need.
if ((idbdatafile::IDBPolicy::useHdfs()) &&
(bs.length() >= sizeof(messageqcpp::ByteStream::byte)))
{
messageqcpp::ByteStream::byte deleteHdfsTempDbFiles;
bs >> deleteHdfsTempDbFiles;
// We always pass "true", as this only applies to the "success-
// ful" case. See comments that precede this function.
int endRc = confirmHdfs.endDbFileListFromMetaFile(tableOID, true, endDbErrMsg);
if (deleteHdfsTempDbFiles)
{
std::string endDbErrMsg;
ConfirmHdfsDbFile confirmHdfs;
// We always pass "true", as this only applies to the "success-
// ful" case. See comments that precede this function.
int endRc = confirmHdfs.endDbFileListFromMetaFile(
tableOID, true, endDbErrMsg);
// In this case, a deletion error is not fatal.
// so we don't propagate a bad return code. We
// may want to add syslog msg (TBD)
if (endRc != NO_ERROR)
std::cout << "Orig db file deletion error: " <<
endDbErrMsg << std::endl;
}
}
BulkRollbackMgr::deleteMetaFile( tableOID );
}
catch (std::exception& ex)
{
std::cout << "ClearTableLockCmd::Cleanup exception-" << ex.what() <<
std::endl;
errMsg = ex.what();
rc = 1;
// In this case, a deletion error is not fatal.
// so we don't propagate a bad return code. We
// may want to add syslog msg (TBD)
if (endRc != NO_ERROR)
std::cout << "Orig db file deletion error: " << endDbErrMsg << std::endl;
}
}
return rc;
BulkRollbackMgr::deleteMetaFile(tableOID);
}
catch (std::exception& ex)
{
std::cout << "ClearTableLockCmd::Cleanup exception-" << ex.what() << std::endl;
errMsg = ex.what();
rc = 1;
}
return rc;
}
}
} // namespace WriteEngine

48
writeengine/server/we_cleartablelockcmd.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_cleartablelockcmd.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
* $Id: we_cleartablelockcmd.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
#pragma once
#include <string>
@ -29,34 +29,32 @@
namespace WriteEngine
{
/** @brief Process messages from a cleartablelock or filesplitter client
*/
class WE_ClearTableLockCmd
{
public:
/** @brief WE_ClearTableLockCmd constructor
*/
WE_ClearTableLockCmd(const char* userDesc) : fUserDesc(userDesc) { }
public:
/** @brief WE_ClearTableLockCmd constructor
*/
WE_ClearTableLockCmd(const char* userDesc) : fUserDesc(userDesc)
{
}
/** @brief Process bulk rollback request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processRollback(messageqcpp::ByteStream& ibs,
std::string& errMsg);
/** @brief Process bulk rollback request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processRollback(messageqcpp::ByteStream& ibs, std::string& errMsg);
/** @brief Process bulk rollback cleanup request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processCleanup (messageqcpp::ByteStream& ibs,
std::string& errMsg);
/** @brief Process bulk rollback cleanup request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processCleanup(messageqcpp::ByteStream& ibs, std::string& errMsg);
private:
WriteEngineWrapper fWEWrapper; // WriteEngineWrapper object
std::string fUserDesc;
private:
WriteEngineWrapper fWEWrapper; // WriteEngineWrapper object
std::string fUserDesc;
};
}
} // namespace WriteEngine

163
writeengine/server/we_cpifeederthread.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
/*
* we_cpifeederthread.cpp
@ -27,7 +27,6 @@
* Author: bpaul
*/
#include <iostream>
#include <string>
#include <queue>
@ -48,148 +47,138 @@ using namespace messageqcpp;
namespace WriteEngine
{
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
void WECpiFeederRunner::operator()()
{
fOwner.feedData2Cpi();
cout << "Finished running Feeder Thread!!" << endl;
fOwner.feedData2Cpi();
cout << "Finished running Feeder Thread!!" << endl;
}
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
WECpiFeederThread::WECpiFeederThread(WEDataLoader& Ref):
fOwner(Ref),
fpThread(0),
fContinue(true),
fStopped(true)
WECpiFeederThread::WECpiFeederThread(WEDataLoader& Ref)
: fOwner(Ref), fpThread(0), fContinue(true), fStopped(true)
{
cout << "Inside WECpiFeederThread constructor" << endl;
cout << "Inside WECpiFeederThread constructor" << endl;
}
//------------------------------------------------------------------------------
WECpiFeederThread::~WECpiFeederThread()
{
if (fpThread)
{
delete fpThread;
}
if (fpThread)
{
delete fpThread;
}
fpThread = 0;
fpThread = 0;
}
//------------------------------------------------------------------------------
void WECpiFeederThread::startFeederThread()
{
fStopped = false;
cout << "Starting Feeder Thread!!" << endl;
fpThread = new boost::thread(WECpiFeederRunner(*this));
fStopped = false;
cout << "Starting Feeder Thread!!" << endl;
fpThread = new boost::thread(WECpiFeederRunner(*this));
}
//------------------------------------------------------------------------------
void WECpiFeederThread::add2MsgQueue(ByteStream& Ibs)
{
//TODO creating copy is NOT good; later read from socket using a SBS
messageqcpp::SBS aSbs(new messageqcpp::ByteStream(Ibs));
Ibs.reset(); //forcefully clearing it
boost::mutex::scoped_lock aLock(fMsgQMutex);
//cout << "pushing to the MsgQueue" << endl;
fMsgQueue.push(aSbs);
fFeederCond.notify_one(); // as per preference of Damon
aLock.unlock();
// TODO creating copy is NOT good; later read from socket using a SBS
messageqcpp::SBS aSbs(new messageqcpp::ByteStream(Ibs));
Ibs.reset(); // forcefully clearing it
boost::mutex::scoped_lock aLock(fMsgQMutex);
// cout << "pushing to the MsgQueue" << endl;
fMsgQueue.push(aSbs);
fFeederCond.notify_one(); // as per preference of Damon
aLock.unlock();
}
//------------------------------------------------------------------------------
void WECpiFeederThread::feedData2Cpi()
{
while (isContinue())
while (isContinue())
{
boost::mutex::scoped_lock aLock(fMsgQMutex);
if (fMsgQueue.empty())
{
bool aTimedOut = fFeederCond.timed_wait(aLock, boost::posix_time::milliseconds(3000));
boost::mutex::scoped_lock aLock(fMsgQMutex);
if (fMsgQueue.empty())
{
bool aTimedOut = fFeederCond.timed_wait(aLock, boost::posix_time::milliseconds(3000));
if (!isContinue())
{
aLock.unlock();
break;
}
// to handle spurious wake ups and timeout wake ups
if ((fMsgQueue.empty()) || (!aTimedOut))
{
aLock.unlock();
continue;
}
}
messageqcpp::SBS aSbs = fMsgQueue.front();
fMsgQueue.pop();
if (!isContinue())
{
aLock.unlock();
break;
}
try
{
fOwner.pushData2Cpimport((*aSbs));
//cout << "Finished PUSHING data " << endl;
}
catch (runtime_error&)
{
//cout << "Caught exception : " << e.what() << endl;
//break;
}
aSbs.reset(); //forcefully clearing it
// We start sending data request from here ONLY
if (getQueueSize() == WEDataLoader::MAX_QSIZE) fOwner.sendDataRequest();
// to handle spurious wake ups and timeout wake ups
if ((fMsgQueue.empty()) || (!aTimedOut))
{
aLock.unlock();
continue;
}
}
cout << "CpiFeedThread Stopped!! " << endl;
fStopped = true;
messageqcpp::SBS aSbs = fMsgQueue.front();
fMsgQueue.pop();
aLock.unlock();
try
{
fOwner.pushData2Cpimport((*aSbs));
// cout << "Finished PUSHING data " << endl;
}
catch (runtime_error&)
{
// cout << "Caught exception : " << e.what() << endl;
// break;
}
aSbs.reset(); // forcefully clearing it
// We start sending data request from here ONLY
if (getQueueSize() == WEDataLoader::MAX_QSIZE)
fOwner.sendDataRequest();
}
cout << "CpiFeedThread Stopped!! " << endl;
fStopped = true;
}
//------------------------------------------------------------------------------
bool WECpiFeederThread::isMsgQueueEmpty()
{
bool aRet = false;
boost::mutex::scoped_lock aLock(fMsgQMutex);
aRet = fMsgQueue.empty();
aLock.unlock();
return aRet;
bool aRet = false;
boost::mutex::scoped_lock aLock(fMsgQMutex);
aRet = fMsgQueue.empty();
aLock.unlock();
return aRet;
}
//------------------------------------------------------------------------------
void WECpiFeederThread::stopThread()
{
boost::mutex::scoped_lock aCondLock(fContMutex);
fContinue = false;
aCondLock.unlock();
fFeederCond.notify_all();
cout << "Notified all" << endl;
boost::mutex::scoped_lock aCondLock(fContMutex);
fContinue = false;
aCondLock.unlock();
fFeederCond.notify_all();
cout << "Notified all" << endl;
}
//------------------------------------------------------------------------------
bool WECpiFeederThread::isContinue()
{
boost::mutex::scoped_lock aCondLock(fContMutex);
return fContinue;
boost::mutex::scoped_lock aCondLock(fContMutex);
return fContinue;
}
//------------------------------------------------------------------------------
} /* namespace WriteEngine */

88
writeengine/server/we_cpifeederthread.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
/*
* we_cpifeederthread.h
@ -33,62 +33,62 @@
namespace WriteEngine
{
class WEDataLoader;
class WECpiFeederThread;
class WECpiFeederRunner
{
public:
WECpiFeederRunner(WECpiFeederThread& Ref): fOwner(Ref) { /* ctor */ }
virtual ~WECpiFeederRunner() {/* dtor */}
void operator()();
public:
WECpiFeederRunner(WECpiFeederThread& Ref) : fOwner(Ref)
{ /* ctor */
}
virtual ~WECpiFeederRunner()
{ /* dtor */
}
void operator()();
public:
WECpiFeederThread& fOwner;
public:
WECpiFeederThread& fOwner;
};
class WECpiFeederThread
{
public:
WECpiFeederThread(WEDataLoader& Ref);
virtual ~WECpiFeederThread();
public:
WECpiFeederThread(WEDataLoader& Ref);
virtual ~WECpiFeederThread();
public:
void startFeederThread();
void add2MsgQueue(messageqcpp::ByteStream& Ibs);
void feedData2Cpi();
void stopThread();
bool isMsgQueueEmpty();
//bool isPushing() { return fPushing; }
bool isStopped()
{
return fStopped;
}
int getQueueSize()
{
return fMsgQueue.size();
}
bool isContinue();
private:
public:
void startFeederThread();
void add2MsgQueue(messageqcpp::ByteStream& Ibs);
void feedData2Cpi();
void stopThread();
bool isMsgQueueEmpty();
// bool isPushing() { return fPushing; }
bool isStopped()
{
return fStopped;
}
int getQueueSize()
{
return fMsgQueue.size();
}
bool isContinue();
WEDataLoader& fOwner;
private:
WEDataLoader& fOwner;
boost::condition fFeederCond;
boost::mutex fMsgQMutex;
typedef std::queue<messageqcpp::SBS> WEMsgQueue;
WEMsgQueue fMsgQueue;
boost::condition fFeederCond;
boost::mutex fMsgQMutex;
typedef std::queue<messageqcpp::SBS> WEMsgQueue;
WEMsgQueue fMsgQueue;
boost::thread* fpThread;
bool fContinue;
boost::mutex fContMutex;
//bool fPushing;
bool fStopped;
boost::thread* fpThread;
bool fContinue;
boost::mutex fContMutex;
// bool fPushing;
bool fStopped;
friend class WEDataLoader;
friend class WEDataLoader;
};
} /* namespace WriteEngine */

2260
writeengine/server/we_dataloader.cpp
View File

File diff suppressed because it is too large Load Diff

295
writeengine/server/we_dataloader.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
/*
* we_dataloader.h
@ -28,8 +28,6 @@
*
*/
#pragma once
#include "rwlock_local.h"
@ -44,7 +42,6 @@
namespace WriteEngine
{
class SplitterReadThread;
// This class will go with the read thread & client socket
@ -52,168 +49,172 @@ class SplitterReadThread;
class WEDataLoader : public Observer
{
public:
explicit WEDataLoader(SplitterReadThread& pSrt );
virtual ~WEDataLoader();
public:
explicit WEDataLoader(SplitterReadThread& pSrt);
virtual ~WEDataLoader();
virtual bool update(Subject* pSub);
virtual bool update(Subject* pSub);
public:
bool setupCpimport(); // fork the cpimport
void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267
void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue
void closeWritePipe();
void str2Argv(std::string CmdLine, std::vector<char*>& V);
public:
bool setupCpimport(); // fork the cpimport
void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267
void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue
void closeWritePipe();
void str2Argv(std::string CmdLine, std::vector<char*>& V);
public:
void onReceiveKeepAlive(ByteStream& Ibs);
void onReceiveData(ByteStream& Ibs);
void onReceiveEod(ByteStream& Ibs); // end of data
void onReceiveMode(ByteStream& Ibs);
// void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs);
void onReceiveCmd(ByteStream& bs); // {(ByteStream& Ibs);
void onReceiveAck(ByteStream& Ibs);
void onReceiveNak(ByteStream& Ibs);
void onReceiveError(ByteStream& Ibs);
public:
void onReceiveKeepAlive(ByteStream& Ibs);
void onReceiveData(ByteStream& Ibs);
void onReceiveEod(ByteStream& Ibs); //end of data
void onReceiveMode(ByteStream& Ibs);
//void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs);
void onReceiveCmd(ByteStream& bs);// {(ByteStream& Ibs);
void onReceiveAck(ByteStream& Ibs);
void onReceiveNak(ByteStream& Ibs);
void onReceiveError(ByteStream& Ibs);
void onReceiveJobId(ByteStream& Ibs);
void onReceiveJobData(ByteStream& Ibs);
void onReceiveImportFileName(ByteStream& Ibs);
void onReceiveCmdLineArgs(ByteStream& Ibs);
void onReceiveStartCpimport();
void onReceiveBrmRptFileName(ByteStream& Ibs);
void onReceiveCleanup(ByteStream& Ibs);
void onReceiveRollback(ByteStream& Ibs);
void onReceiveJobId(ByteStream& Ibs);
void onReceiveJobData(ByteStream& Ibs);
void onReceiveImportFileName(ByteStream& Ibs);
void onReceiveCmdLineArgs(ByteStream& Ibs);
void onReceiveStartCpimport();
void onReceiveBrmRptFileName(ByteStream& Ibs);
void onReceiveCleanup(ByteStream& Ibs);
void onReceiveRollback(ByteStream& Ibs);
void onReceiveErrFileRqst(ByteStream& Ibs);
void onReceiveBadFileRqst(ByteStream& Ibs);
void onReceiveErrFileRqst(ByteStream& Ibs);
void onReceiveBadFileRqst(ByteStream& Ibs);
void onCpimportSuccess();
void onCpimportFailure();
void onCpimportSuccess();
void onCpimportFailure();
void sendDataRequest();
void sendCpimportFailureNotice();
void sendDataRequest();
void sendCpimportFailureNotice();
void serialize(messageqcpp::ByteStream& b) const;
void unserialize(messageqcpp::ByteStream& b);
void serialize(messageqcpp::ByteStream& b) const;
void unserialize(messageqcpp::ByteStream& b);
// setup the signal handlers for the main app
void setupSignalHandlers();
static void onSigChild(int aInt);
public:
void setMode(int Mode)
{
fMode = Mode;
}
void updateTxBytes(unsigned int Tx)
{
fTxBytes += Tx;
}
void updateRxBytes(unsigned int Rx)
{
fRxBytes += Rx;
}
void setChPid(pid_t pid)
{
fCh_pid = pid;
}
void setPid(pid_t pid)
{
fThis_pid = pid;
}
void setPPid(pid_t pid)
{
fP_pid = pid;
}
void setCmdLineStr(std::string& Str)
{
fCmdLineStr = Str;
}
void setObjId(int ObjId)
{
fObjId = ObjId;
}
// setup the signal handlers for the main app
void setupSignalHandlers();
static void onSigChild(int aInt);
unsigned int getTxBytes()
{
return fTxBytes;
}
unsigned int getRxBytes()
{
return fRxBytes;
}
public:
void setMode(int Mode)
{
fMode = Mode;
}
void updateTxBytes(unsigned int Tx)
{
fTxBytes += Tx;
}
void updateRxBytes(unsigned int Rx)
{
fRxBytes += Rx;
}
void setChPid(pid_t pid)
{
fCh_pid = pid;
}
void setPid(pid_t pid)
{
fThis_pid = pid;
}
void setPPid(pid_t pid)
{
fP_pid = pid;
}
void setCmdLineStr(std::string& Str)
{
fCmdLineStr = Str;
}
void setObjId(int ObjId)
{
fObjId = ObjId;
}
int getObjId()
{
return fObjId;
}
int getMode()
{
return fMode;
}
pid_t getChPid()
{
return fCh_pid;
}
pid_t getPid()
{
return fThis_pid;
}
pid_t getPPid()
{
return fP_pid;
}
std::string getCmdLineStr()
{
return fCmdLineStr;
}
unsigned int getTxBytes()
{
return fTxBytes;
}
unsigned int getRxBytes()
{
return fRxBytes;
}
private:
SplitterReadThread& fRef;
int getObjId()
{
return fObjId;
}
int getMode()
{
return fMode;
}
pid_t getChPid()
{
return fCh_pid;
}
pid_t getPid()
{
return fThis_pid;
}
pid_t getPPid()
{
return fP_pid;
}
std::string getCmdLineStr()
{
return fCmdLineStr;
}
int fMode;
std::ofstream fDataDumpFile;
std::ofstream fJobFile;
unsigned int fTxBytes;
unsigned int fRxBytes;
char fPmId;
int fObjId; // Object Identifier for logging
private:
SplitterReadThread& fRef;
// CpImport related Member variables
int fFIFO[2]; // I/O Pipes
pid_t fCh_pid;
pid_t fThis_pid;
pid_t fP_pid;
bool fCpIStarted;
std::string fCmdLineStr;
std::string fBrmRptFileName;
int fMode;
std::ofstream fDataDumpFile;
std::ofstream fJobFile;
unsigned int fTxBytes;
unsigned int fRxBytes;
char fPmId;
int fObjId; // Object Identifier for logging
// CPI Feeder Thread
WECpiFeederThread* fpCfThread;
// CpImport related Member variables
int fFIFO[2]; //I/O Pipes
pid_t fCh_pid;
pid_t fThis_pid;
pid_t fP_pid;
bool fCpIStarted;
std::string fCmdLineStr;
std::string fBrmRptFileName;
boost::mutex fClntMsgMutex; // mutex in sending messages to client.
//CPI Feeder Thread
WECpiFeederThread* fpCfThread;
// static bool fTearDownCpimport; // @bug 4267
bool fTearDownCpimport; // @bug 4267
pid_t fWaitPidRc; // @bug 4267
int fWaitPidStatus; // @bug 4267
boost::mutex fClntMsgMutex; //mutex in sending messages to client.
bool fForceKill;
bool fPipeErr; // Err Flag to restrict err msgs logging.
//static bool fTearDownCpimport; // @bug 4267
bool fTearDownCpimport; // @bug 4267
pid_t fWaitPidRc; // @bug 4267
int fWaitPidStatus; // @bug 4267
private:
// more enums follow
enum CmdId
{
BULKFILENAME
};
bool fForceKill;
bool fPipeErr; // Err Flag to restrict err msgs logging.
private:
// more enums follow
enum CmdId { BULKFILENAME };
public:
enum { MIN_QSIZE = 25, MAX_QSIZE = 250};
public:
SimpleSysLog* fpSysLog;
public:
enum
{
MIN_QSIZE = 25,
MAX_QSIZE = 250
};
public:
SimpleSysLog* fpSysLog;
};
}
} // namespace WriteEngine

9138
writeengine/server/we_ddlcommandproc.cpp
View File

File diff suppressed because it is too large Load Diff

154
writeengine/server/we_ddlcommandproc.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $
*
*******************************************************************************/
* $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $
*
*******************************************************************************/
#pragma once
#include <unistd.h>
@ -38,80 +38,88 @@
namespace WriteEngine
{
class WE_DDLCommandProc
{
public:
enum LogFileType { DROPTABLE_LOG, DROPPART_LOG, TRUNCATE_LOG};
EXPORT WE_DDLCommandProc();
EXPORT WE_DDLCommandProc(const WE_DDLCommandProc& rhs);
EXPORT ~WE_DDLCommandProc();
/** @brief Update SYSCOLUMN nextval column for the columnoid with nextVal.
*
* Update SYSCOLUMN nextval column for the columnoid with nexValue.
* @param columnOid (in) The column OID
* @param nextVal (in) The partition number
* @return 0 on success, non-0 on error.
*/
EXPORT uint8_t updateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeSystable(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeCreateSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t createtablefiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSyscolumnRow(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSystable(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSystables(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t dropFiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnAuto(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnNextvalCol(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystableAuto(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystableTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystablesTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnColumnposCol(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t fillNewColumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnRenameColumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnSetDefault(messageqcpp::ByteStream& bs, std::string& err);
// EXPORT uint8_t updateSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeTruncateLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeDropPartitionLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeDropTableLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteDDLLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t fetchDDLLog(messageqcpp::ByteStream& bs, std::string& err);
void purgeFDCache();
/** @brief drop a set of partitions
*
* Do many VSS lookups under one lock.
* @param bs (in) bytestream carry command info
* @param err (out) error message when error occurs
* @return 0 on success, otherwise error.
*/
EXPORT uint8_t dropPartitions(messageqcpp::ByteStream& bs, std::string& err);
inline void convertRidToColumn (uint64_t& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, const int32_t oid )
{
fDbrm.getSysCatDBRoot(oid, dbRoot);
partition = rid / ( filesPerColumnPartition * extentsPerSegmentFile * extentRows );
public:
enum LogFileType
{
DROPTABLE_LOG,
DROPPART_LOG,
TRUNCATE_LOG
};
EXPORT WE_DDLCommandProc();
EXPORT WE_DDLCommandProc(const WE_DDLCommandProc& rhs);
EXPORT ~WE_DDLCommandProc();
/** @brief Update SYSCOLUMN nextval column for the columnoid with nextVal.
*
* Update SYSCOLUMN nextval column for the columnoid with nexValue.
* @param columnOid (in) The column OID
* @param nextVal (in) The partition number
* @return 0 on success, non-0 on error.
*/
EXPORT uint8_t updateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeSystable(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeCreateSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t createtablefiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSyscolumnRow(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSystable(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSystables(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t dropFiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnAuto(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnNextvalCol(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystableAuto(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystableTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystablesTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnColumnposCol(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t fillNewColumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnRenameColumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnSetDefault(messageqcpp::ByteStream& bs, std::string& err);
// EXPORT uint8_t updateSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeTruncateLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeDropPartitionLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeDropTableLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteDDLLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t fetchDDLLog(messageqcpp::ByteStream& bs, std::string& err);
void purgeFDCache();
/** @brief drop a set of partitions
*
* Do many VSS lookups under one lock.
* @param bs (in) bytestream carry command info
* @param err (out) error message when error occurs
* @return 0 on success, otherwise error.
*/
EXPORT uint8_t dropPartitions(messageqcpp::ByteStream& bs, std::string& err);
inline void convertRidToColumn(uint64_t& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment,
const int32_t oid)
{
fDbrm.getSysCatDBRoot(oid, dbRoot);
partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows);
segment = ( ( ( rid % ( filesPerColumnPartition * extentsPerSegmentFile * extentRows )) / extentRows ) ) % filesPerColumnPartition;
segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) %
filesPerColumnPartition;
//Calculate the relative rid for this segment file
uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
assert ( relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows );
uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows;
}
// Calculate the relative rid for this segment file
uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition *
(uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
assert(relRidInPartition <=
(uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows;
}
private:
WriteEngineWrapper fWEWrapper;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
private:
WriteEngineWrapper fWEWrapper;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
};
}
} // namespace WriteEngine
#undef EXPORT

346
writeengine/server/we_ddlcommon.h
View File

@ -16,10 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
#pragma once
@ -53,280 +52,221 @@
#include <boost/algorithm/string/case_conv.hpp>
template <class T>
bool from_string(T& t,
const std::string& s,
std::ios_base & (*f)(std::ios_base&))
bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&))
{
std::istringstream iss(s);
return !(iss >> f >> t).fail();
std::istringstream iss(s);
return !(iss >> f >> t).fail();
}
namespace WriteEngine
{
struct DDLColumn
{
execplan::CalpontSystemCatalog::OID oid;
execplan::CalpontSystemCatalog::ColType colType;
execplan::CalpontSystemCatalog::TableColName tableColName;
execplan::CalpontSystemCatalog::OID oid;
execplan::CalpontSystemCatalog::ColType colType;
execplan::CalpontSystemCatalog::TableColName tableColName;
};
typedef std::vector<DDLColumn> ColumnList;
struct DictOID
{
int dictOID;
int listOID;
int treeOID;
int colWidth;
int compressionType;
int dictOID;
int listOID;
int treeOID;
int colWidth;
int compressionType;
};
struct extentInfo
{
uint16_t dbRoot;
uint32_t partition;
uint16_t segment;
bool operator==(const extentInfo& rhs) const
{
return (dbRoot == rhs.dbRoot && partition == rhs.partition && segment == rhs.segment);
}
bool operator!=(const extentInfo& rhs) const
{
return !(*this == rhs);
}
uint16_t dbRoot;
uint32_t partition;
uint16_t segment;
bool operator==(const extentInfo& rhs) const
{
return (dbRoot == rhs.dbRoot && partition == rhs.partition && segment == rhs.segment);
}
bool operator!=(const extentInfo& rhs) const
{
return !(*this == rhs);
}
};
inline void getColumnsForTable(uint32_t sessionID, std::string schema, std::string table,
ColumnList& colList)
inline void getColumnsForTable(uint32_t sessionID, std::string schema, std::string table, ColumnList& colList)
{
execplan::CalpontSystemCatalog::TableName tableName;
tableName.schema = schema;
tableName.table = table;
std::string err;
tableName.schema = schema;
tableName.table = table;
std::string err;
try
try
{
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID);
systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC);
const execplan::CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName);
execplan::CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin();
while (rid_iterator != ridList.end())
{
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr = execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID);
systemCatalogPtr->identity(execplan::CalpontSystemCatalog::EC);
execplan::CalpontSystemCatalog::ROPair roPair = *rid_iterator;
const execplan::CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName);
DDLColumn column;
column.oid = roPair.objnum;
column.colType = systemCatalogPtr->colType(column.oid);
column.tableColName = systemCatalogPtr->colName(column.oid);
execplan::CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin();
while (rid_iterator != ridList.end())
{
execplan::CalpontSystemCatalog::ROPair roPair = *rid_iterator;
DDLColumn column;
column.oid = roPair.objnum;
column.colType = systemCatalogPtr->colType(column.oid);
column.tableColName = systemCatalogPtr->colName(column.oid);
colList.push_back(column);
++rid_iterator;
}
colList.push_back(column);
++rid_iterator;
}
catch (exception& ex)
{
err = "DDLPackageProcessor::getColumnsForTable: while reading columns for table " + schema + '.' + table + ": " + ex.what();
throw std::runtime_error(err);
}
catch (...)
{
err = "DDLPackageProcessor::getColumnsForTable: caught unkown exception!" ;
throw std::runtime_error(err);
}
}
catch (exception& ex)
{
err = "DDLPackageProcessor::getColumnsForTable: while reading columns for table " + schema + '.' + table +
": " + ex.what();
throw std::runtime_error(err);
}
catch (...)
{
err = "DDLPackageProcessor::getColumnsForTable: caught unkown exception!";
throw std::runtime_error(err);
}
}
inline int convertDataType(int dataType)
{
int calpontDataType;
int calpontDataType;
switch (dataType)
{
case ddlpackage::DDL_CHAR:
calpontDataType = execplan::CalpontSystemCatalog::CHAR;
break;
switch (dataType)
{
case ddlpackage::DDL_CHAR: calpontDataType = execplan::CalpontSystemCatalog::CHAR; break;
case ddlpackage::DDL_VARCHAR:
calpontDataType = execplan::CalpontSystemCatalog::VARCHAR;
break;
case ddlpackage::DDL_VARCHAR: calpontDataType = execplan::CalpontSystemCatalog::VARCHAR; break;
case ddlpackage::DDL_VARBINARY:
calpontDataType = execplan::CalpontSystemCatalog::VARBINARY;
break;
case ddlpackage::DDL_VARBINARY: calpontDataType = execplan::CalpontSystemCatalog::VARBINARY; break;
case ddlpackage::DDL_BIT:
calpontDataType = execplan::CalpontSystemCatalog::BIT;
break;
case ddlpackage::DDL_BIT: calpontDataType = execplan::CalpontSystemCatalog::BIT; break;
case ddlpackage::DDL_REAL:
case ddlpackage::DDL_DECIMAL:
case ddlpackage::DDL_NUMERIC:
case ddlpackage::DDL_NUMBER:
calpontDataType = execplan::CalpontSystemCatalog::DECIMAL;
break;
case ddlpackage::DDL_REAL:
case ddlpackage::DDL_DECIMAL:
case ddlpackage::DDL_NUMERIC:
case ddlpackage::DDL_NUMBER: calpontDataType = execplan::CalpontSystemCatalog::DECIMAL; break;
case ddlpackage::DDL_FLOAT:
calpontDataType = execplan::CalpontSystemCatalog::FLOAT;
break;
case ddlpackage::DDL_FLOAT: calpontDataType = execplan::CalpontSystemCatalog::FLOAT; break;
case ddlpackage::DDL_DOUBLE:
calpontDataType = execplan::CalpontSystemCatalog::DOUBLE;
break;
case ddlpackage::DDL_DOUBLE: calpontDataType = execplan::CalpontSystemCatalog::DOUBLE; break;
case ddlpackage::DDL_INT:
case ddlpackage::DDL_INTEGER:
calpontDataType = execplan::CalpontSystemCatalog::INT;
break;
case ddlpackage::DDL_INT:
case ddlpackage::DDL_INTEGER: calpontDataType = execplan::CalpontSystemCatalog::INT; break;
case ddlpackage::DDL_BIGINT:
calpontDataType = execplan::CalpontSystemCatalog::BIGINT;
break;
case ddlpackage::DDL_BIGINT: calpontDataType = execplan::CalpontSystemCatalog::BIGINT; break;
case ddlpackage::DDL_MEDINT:
calpontDataType = execplan::CalpontSystemCatalog::MEDINT;
break;
case ddlpackage::DDL_MEDINT: calpontDataType = execplan::CalpontSystemCatalog::MEDINT; break;
case ddlpackage::DDL_SMALLINT:
calpontDataType = execplan::CalpontSystemCatalog::SMALLINT;
break;
case ddlpackage::DDL_SMALLINT: calpontDataType = execplan::CalpontSystemCatalog::SMALLINT; break;
case ddlpackage::DDL_TINYINT:
calpontDataType = execplan::CalpontSystemCatalog::TINYINT;
break;
case ddlpackage::DDL_TINYINT: calpontDataType = execplan::CalpontSystemCatalog::TINYINT; break;
case ddlpackage::DDL_DATE:
calpontDataType = execplan::CalpontSystemCatalog::DATE;
break;
case ddlpackage::DDL_DATE: calpontDataType = execplan::CalpontSystemCatalog::DATE; break;
case ddlpackage::DDL_DATETIME:
calpontDataType = execplan::CalpontSystemCatalog::DATETIME;
break;
case ddlpackage::DDL_DATETIME: calpontDataType = execplan::CalpontSystemCatalog::DATETIME; break;
case ddlpackage::DDL_TIME:
calpontDataType = execplan::CalpontSystemCatalog::TIME;
break;
case ddlpackage::DDL_TIME: calpontDataType = execplan::CalpontSystemCatalog::TIME; break;
case ddlpackage::DDL_TIMESTAMP:
calpontDataType = execplan::CalpontSystemCatalog::TIMESTAMP;
break;
case ddlpackage::DDL_TIMESTAMP: calpontDataType = execplan::CalpontSystemCatalog::TIMESTAMP; break;
case ddlpackage::DDL_CLOB:
calpontDataType = execplan::CalpontSystemCatalog::CLOB;
break;
case ddlpackage::DDL_CLOB: calpontDataType = execplan::CalpontSystemCatalog::CLOB; break;
case ddlpackage::DDL_BLOB:
calpontDataType = execplan::CalpontSystemCatalog::BLOB;
break;
case ddlpackage::DDL_BLOB: calpontDataType = execplan::CalpontSystemCatalog::BLOB; break;
case ddlpackage::DDL_TEXT:
calpontDataType = execplan::CalpontSystemCatalog::TEXT;
break;
case ddlpackage::DDL_TEXT: calpontDataType = execplan::CalpontSystemCatalog::TEXT; break;
case ddlpackage::DDL_UNSIGNED_TINYINT:
calpontDataType = execplan::CalpontSystemCatalog::UTINYINT;
break;
case ddlpackage::DDL_UNSIGNED_TINYINT: calpontDataType = execplan::CalpontSystemCatalog::UTINYINT; break;
case ddlpackage::DDL_UNSIGNED_SMALLINT:
calpontDataType = execplan::CalpontSystemCatalog::USMALLINT;
break;
case ddlpackage::DDL_UNSIGNED_SMALLINT:
calpontDataType = execplan::CalpontSystemCatalog::USMALLINT;
break;
case ddlpackage::DDL_UNSIGNED_MEDINT:
calpontDataType = execplan::CalpontSystemCatalog::UMEDINT;
break;
case ddlpackage::DDL_UNSIGNED_MEDINT: calpontDataType = execplan::CalpontSystemCatalog::UMEDINT; break;
case ddlpackage::DDL_UNSIGNED_INT:
calpontDataType = execplan::CalpontSystemCatalog::UINT;
break;
case ddlpackage::DDL_UNSIGNED_INT: calpontDataType = execplan::CalpontSystemCatalog::UINT; break;
case ddlpackage::DDL_UNSIGNED_BIGINT:
calpontDataType = execplan::CalpontSystemCatalog::UBIGINT;
break;
case ddlpackage::DDL_UNSIGNED_BIGINT: calpontDataType = execplan::CalpontSystemCatalog::UBIGINT; break;
case ddlpackage::DDL_UNSIGNED_DECIMAL:
case ddlpackage::DDL_UNSIGNED_NUMERIC:
calpontDataType = execplan::CalpontSystemCatalog::UDECIMAL;
break;
case ddlpackage::DDL_UNSIGNED_DECIMAL:
case ddlpackage::DDL_UNSIGNED_NUMERIC: calpontDataType = execplan::CalpontSystemCatalog::UDECIMAL; break;
case ddlpackage::DDL_UNSIGNED_FLOAT:
calpontDataType = execplan::CalpontSystemCatalog::UFLOAT;
break;
case ddlpackage::DDL_UNSIGNED_FLOAT: calpontDataType = execplan::CalpontSystemCatalog::UFLOAT; break;
case ddlpackage::DDL_UNSIGNED_DOUBLE:
calpontDataType = execplan::CalpontSystemCatalog::UDOUBLE;
break;
default:
throw runtime_error("Unsupported datatype!");
case ddlpackage::DDL_UNSIGNED_DOUBLE: calpontDataType = execplan::CalpontSystemCatalog::UDOUBLE; break;
}
default: throw runtime_error("Unsupported datatype!");
}
return calpontDataType;
return calpontDataType;
}
inline void findColumnData(uint32_t sessionID, execplan::CalpontSystemCatalog::TableName& systableName,
const std::string& colName, DDLColumn& sysCol)
{
ColumnList columns;
ColumnList::const_iterator column_iterator;
std::string err;
ColumnList columns;
ColumnList::const_iterator column_iterator;
std::string err;
try
try
{
getColumnsForTable(sessionID, systableName.schema, systableName.table, columns);
column_iterator = columns.begin();
while (column_iterator != columns.end())
{
getColumnsForTable(sessionID, systableName.schema, systableName.table, columns);
column_iterator = columns.begin();
sysCol = *column_iterator;
boost::to_lower(sysCol.tableColName.column);
while (column_iterator != columns.end())
{
sysCol = *column_iterator;
boost::to_lower(sysCol.tableColName.column);
if (colName == sysCol.tableColName.column)
{
break;
}
if (colName == sysCol.tableColName.column)
{
break;
}
++column_iterator;
}
}
catch (exception& ex)
{
err = ex.what();
throw std::runtime_error(err);
}
catch (...)
{
err = "findColumnData:Unknown exception caught";
throw std::runtime_error(err);
++column_iterator;
}
}
catch (exception& ex)
{
err = ex.what();
throw std::runtime_error(err);
}
catch (...)
{
err = "findColumnData:Unknown exception caught";
throw std::runtime_error(err);
}
}
inline void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition,
unsigned& segment, unsigned filesPerColumnPartition,
unsigned extentsPerSegmentFile, unsigned extentRows,
unsigned startDBRoot, unsigned dbrootCnt)
inline void convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition, unsigned& segment,
unsigned filesPerColumnPartition, unsigned extentsPerSegmentFile,
unsigned extentRows, unsigned startDBRoot, unsigned dbrootCnt)
{
partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows);
partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows);
segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) % filesPerColumnPartition;
segment = (((rid % (filesPerColumnPartition * extentsPerSegmentFile * extentRows)) / extentRows)) %
filesPerColumnPartition;
dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1;
dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1;
//Calculate the relative rid for this segment file
uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
idbassert(relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows;
// Calculate the relative rid for this segment file
uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition *
(uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
idbassert(relRidInPartition <=
(uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows;
}
}
} // namespace WriteEngine
#undef EXPORT

8863
writeengine/server/we_dmlcommandproc.cpp
View File

File diff suppressed because it is too large Load Diff

165
writeengine/server/we_dmlcommandproc.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $
*
*******************************************************************************/
* $Id: we_ddlcommandproc.h 3043 2011-08-29 22:03:03Z chao $
*
*******************************************************************************/
#pragma once
#include <unistd.h>
@ -48,91 +48,92 @@
namespace WriteEngine
{
class WE_DMLCommandProc
{
public:
typedef std::vector<std::string> ColValues;
public:
typedef std::vector<std::string> ColValues;
EXPORT WE_DMLCommandProc();
EXPORT WE_DMLCommandProc(const WE_DMLCommandProc& rhs);
EXPORT ~WE_DMLCommandProc();
inline void isFirstBatchPm (bool firstBatch)
EXPORT WE_DMLCommandProc();
EXPORT WE_DMLCommandProc(const WE_DMLCommandProc& rhs);
EXPORT ~WE_DMLCommandProc();
inline void isFirstBatchPm(bool firstBatch)
{
fIsFirstBatchPm = firstBatch;
}
inline bool isFirstBatchPm()
{
return fIsFirstBatchPm;
}
// Convert rid from logical block rid to file relative rid
inline void convertToRelativeRid(uint64_t& rid, const uint8_t extentNum, const uint16_t blockNum)
{
rid = rid + extentNum * extentRows + blockNum * 8192;
}
EXPORT uint8_t processSingleInsert(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err,
ByteStream::quadbyte& PMId);
EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err,
ByteStream::quadbyte& PMId);
EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processUpdate(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId,
uint64_t& blocksChanged);
EXPORT uint8_t processUpdate1(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processFlushFiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processDelete(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId,
uint64_t& blocksChanged);
EXPORT uint8_t processRemoveMeta(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBulkRollback(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnNextval(ByteStream& bs, std::string& err);
EXPORT uint8_t processPurgeFDCache(ByteStream& bs, std::string& err);
EXPORT uint8_t processEndTransaction(ByteStream& bs, std::string& err);
EXPORT uint8_t processFixRows(ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
EXPORT uint8_t getWrittenLbids(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
int validateColumnHWMs(execplan::CalpontSystemCatalog::RIDList& ridList,
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr,
const std::vector<DBRootExtentInfo>& segFileInfo, const char* stage);
private:
WriteEngineWrapper fWEWrapper;
boost::scoped_ptr<RBMetaWriter> fRBMetaWriter;
std::vector<boost::shared_ptr<DBRootExtentTracker> > dbRootExtTrackerVec;
inline bool isDictCol(execplan::CalpontSystemCatalog::ColType& colType)
{
if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) ||
((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) ||
((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) ||
((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 38)) ||
(colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) ||
(colType.colDataType == execplan::CalpontSystemCatalog::BLOB) ||
(colType.colDataType == execplan::CalpontSystemCatalog::TEXT))
{
fIsFirstBatchPm = firstBatch;
return true;
}
else
return false;
}
uint8_t processBatchInsertHwmFlushChunks(uint32_t tableOID, int txnID,
const std::vector<BRM::FileInfo>& files,
const std::vector<BRM::OID_t>& oidsToFlush, std::string& err);
inline bool isFirstBatchPm ()
{
return fIsFirstBatchPm;
}
//Convert rid from logical block rid to file relative rid
inline void convertToRelativeRid (uint64_t& rid, const uint8_t extentNum, const uint16_t blockNum)
{
rid = rid + extentNum * extentRows + blockNum * 8192;
}
EXPORT uint8_t processSingleInsert(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processUpdate(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, uint64_t& blocksChanged);
EXPORT uint8_t processUpdate1(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processFlushFiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processDelete(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, uint64_t& blocksChanged);
EXPORT uint8_t processRemoveMeta(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBulkRollback(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnNextval(ByteStream& bs, std::string& err);
EXPORT uint8_t processPurgeFDCache(ByteStream& bs, std::string& err);
EXPORT uint8_t processEndTransaction(ByteStream& bs, std::string& err);
EXPORT uint8_t processFixRows(ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
EXPORT uint8_t getWrittenLbids(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
int validateColumnHWMs(
execplan::CalpontSystemCatalog::RIDList& ridList,
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr,
const std::vector<DBRootExtentInfo>& segFileInfo,
const char* stage );
private:
WriteEngineWrapper fWEWrapper;
boost::scoped_ptr<RBMetaWriter> fRBMetaWriter;
std::vector<boost::shared_ptr<DBRootExtentTracker> > dbRootExtTrackerVec;
inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType &colType )
{
if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 38))
|| (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)
|| (colType.colDataType == execplan::CalpontSystemCatalog::BLOB)
|| (colType.colDataType == execplan::CalpontSystemCatalog::TEXT))
{
return true;
}
else
return false;
}
uint8_t processBatchInsertHwmFlushChunks(uint32_t tableOID, int txnID,
const std::vector<BRM::FileInfo>& files,
const std::vector<BRM::OID_t>& oidsToFlush,
std::string& err);
bool fIsFirstBatchPm;
std::map<uint32_t, rowgroup::RowGroup*> rowGroups;
std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
Log fLog;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
bool fIsFirstBatchPm;
std::map<uint32_t, rowgroup::RowGroup*> rowGroups;
std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
Log fLog;
static const uint32_t DEFAULT_EXTENTS_PER_SEGMENT_FILE = 2;
};
}
} // namespace WriteEngine
#undef EXPORT

597
writeengine/server/we_getfilesizes.cpp
View File

@ -46,19 +46,19 @@ namespace WriteEngine
{
struct FileInfo
{
uint32_t partition; /** @brief Partition for a file*/
uint16_t segment; /** @brief Segment for a file */
uint16_t dbRoot; /** @brief DbRoot for a file */
std::string segFileName; /** @brief seg file path */
double fileSize; /** @brief seg file size in giga bytes */
void serialize(messageqcpp::ByteStream& bs)
{
bs << partition;
bs << segment;
bs << dbRoot;
bs << segFileName;
bs << (*(uint64_t*)(&fileSize));
}
uint32_t partition; /** @brief Partition for a file*/
uint16_t segment; /** @brief Segment for a file */
uint16_t dbRoot; /** @brief DbRoot for a file */
std::string segFileName; /** @brief seg file path */
double fileSize; /** @brief seg file size in giga bytes */
void serialize(messageqcpp::ByteStream& bs)
{
bs << partition;
bs << segment;
bs << dbRoot;
bs << segFileName;
bs << (*(uint64_t*)(&fileSize));
}
};
typedef std::vector<FileInfo> Files;
typedef std::map<uint32_t, Files> columnMap;
@ -67,358 +67,349 @@ allColumnMap wholeMap;
boost::mutex columnMapLock;
ActiveThreadCounter* activeThreadCounter;
size_t readFillBuffer(
idbdatafile::IDBDataFile* pFile,
char* buffer,
size_t bytesReq)
size_t readFillBuffer(idbdatafile::IDBDataFile* pFile, char* buffer, size_t bytesReq)
{
char* pBuf = buffer;
ssize_t nBytes;
size_t bytesToRead = bytesReq;
size_t totalBytesRead = 0;
char* pBuf = buffer;
ssize_t nBytes;
size_t bytesToRead = bytesReq;
size_t totalBytesRead = 0;
while (1)
{
nBytes = pFile->read(pBuf, bytesToRead);
while (1)
{
nBytes = pFile->read(pBuf, bytesToRead);
if (nBytes > 0)
totalBytesRead += nBytes;
else
break;
if (nBytes > 0)
totalBytesRead += nBytes;
else
break;
if ((size_t)nBytes == bytesToRead)
break;
if ((size_t)nBytes == bytesToRead)
break;
pBuf += nBytes;
bytesToRead = bytesToRead - (size_t)nBytes;
}
pBuf += nBytes;
bytesToRead = bytesToRead - (size_t)nBytes;
}
return totalBytesRead;
return totalBytesRead;
}
static off64_t getCompressedDataSize(string& fileName)
{
off64_t dataSize = 0;
IDBDataFile* pFile = 0;
size_t nBytes;
// Some IDBPolicy functions can throw exceptions, caller will catch it
IDBPolicy::configIDBPolicy();
bool bHdfsFile = IDBPolicy::useHdfs();
off64_t dataSize = 0;
IDBDataFile* pFile = 0;
size_t nBytes;
// Some IDBPolicy functions can throw exceptions, caller will catch it
IDBPolicy::configIDBPolicy();
bool bHdfsFile = IDBPolicy::useHdfs();
if (bHdfsFile)
pFile = IDBDataFile::open(IDBDataFile::HDFS,
fileName.c_str(), "r", 0);
else
pFile = IDBDataFile::open(IDBDataFile::BUFFERED,
fileName.c_str(), "r", 0);
if (bHdfsFile)
pFile = IDBDataFile::open(IDBDataFile::HDFS, fileName.c_str(), "r", 0);
else
pFile = IDBDataFile::open(IDBDataFile::BUFFERED, fileName.c_str(), "r", 0);
if (!pFile)
{
std::ostringstream oss;
oss << "Cannot open file " << fileName << " for read.";
throw std::runtime_error(oss.str());
}
if (!pFile)
{
std::ostringstream oss;
oss << "Cannot open file " << fileName << " for read.";
throw std::runtime_error(oss.str());
}
//--------------------------------------------------------------------------
// Read headers and extract compression pointers
//--------------------------------------------------------------------------
char hdr1[CompressInterface::HDR_BUF_LEN];
nBytes = readFillBuffer( pFile, hdr1, CompressInterface::HDR_BUF_LEN);
//--------------------------------------------------------------------------
// Read headers and extract compression pointers
//--------------------------------------------------------------------------
char hdr1[CompressInterface::HDR_BUF_LEN];
nBytes = readFillBuffer(pFile, hdr1, CompressInterface::HDR_BUF_LEN);
if ( nBytes != CompressInterface::HDR_BUF_LEN )
{
std::ostringstream oss;
oss << "Error reading first header from file " << fileName;
throw std::runtime_error(oss.str());
}
if (nBytes != CompressInterface::HDR_BUF_LEN)
{
std::ostringstream oss;
oss << "Error reading first header from file " << fileName;
throw std::runtime_error(oss.str());
}
int64_t ptrSecSize = compress::CompressInterface::getHdrSize(hdr1) -
CompressInterface::HDR_BUF_LEN;
char* hdr2 = new char[ptrSecSize];
nBytes = readFillBuffer( pFile, hdr2, ptrSecSize);
int64_t ptrSecSize = compress::CompressInterface::getHdrSize(hdr1) - CompressInterface::HDR_BUF_LEN;
char* hdr2 = new char[ptrSecSize];
nBytes = readFillBuffer(pFile, hdr2, ptrSecSize);
if ( (int64_t)nBytes != ptrSecSize )
{
std::ostringstream oss;
oss << "Error reading second header from file " << fileName;
throw std::runtime_error(oss.str());
}
if ((int64_t)nBytes != ptrSecSize)
{
std::ostringstream oss;
oss << "Error reading second header from file " << fileName;
throw std::runtime_error(oss.str());
}
CompChunkPtrList chunkPtrs;
int rc =
compress::CompressInterface::getPtrList(hdr2, ptrSecSize, chunkPtrs);
delete[] hdr2;
CompChunkPtrList chunkPtrs;
int rc = compress::CompressInterface::getPtrList(hdr2, ptrSecSize, chunkPtrs);
delete[] hdr2;
if (rc != 0)
{
std::ostringstream oss;
oss << "Error decompressing second header from file " << fileName;
throw std::runtime_error(oss.str());
}
if (rc != 0)
{
std::ostringstream oss;
oss << "Error decompressing second header from file " << fileName;
throw std::runtime_error(oss.str());
}
unsigned k = chunkPtrs.size();
// last header's offset + length will be the data bytes
dataSize = chunkPtrs[k - 1].first + chunkPtrs[k - 1].second;
delete pFile;
return dataSize;
unsigned k = chunkPtrs.size();
// last header's offset + length will be the data bytes
dataSize = chunkPtrs[k - 1].first + chunkPtrs[k - 1].second;
delete pFile;
return dataSize;
}
struct ColumnThread
{
ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key)
: fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key)
{}
void operator()()
ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key)
: fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key)
{
}
void operator()()
{
Config config;
config.initConfigCache();
std::vector<uint16_t> rootList;
config.getRootIdList(rootList);
FileOp fileOp;
Files aFiles;
// This function relies on IDBPolicy being initialized by
// IDBPolicy::init(). This is done when WriteEngineServer main() calls
// IDBPolicy::configIDBPolicy();
IDBDataFile::Types fileType;
bool bUsingHdfs = IDBPolicy::useHdfs();
if (bUsingHdfs)
fileType = IDBDataFile::HDFS;
else if (IDBPolicy::useCloud())
fileType = IDBDataFile::CLOUD;
else
fileType = IDBDataFile::UNBUFFERED;
IDBFileSystem& fs = IDBFileSystem::getFs(fileType);
for (uint32_t i = 0; i < rootList.size(); i++)
{
Config config;
config.initConfigCache();
std::vector<uint16_t> rootList;
config.getRootIdList( rootList );
FileOp fileOp;
Files aFiles;
std::vector<struct BRM::EMEntry> entries;
(void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]);
std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
// This function relies on IDBPolicy being initialized by
// IDBPolicy::init(). This is done when WriteEngineServer main() calls
// IDBPolicy::configIDBPolicy();
IDBDataFile::Types fileType;
bool bUsingHdfs = IDBPolicy::useHdfs();
while (iter != entries.end()) // organize extents into files
{
// Find the size of this file
// string fileName;
char fileName[200];
(void)fileOp.getFileName(fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum);
string aFile(fileName); // convert between char* and string
off64_t fileSize = 0;
if (bUsingHdfs)
fileType = IDBDataFile::HDFS;
else if (IDBPolicy::useCloud())
fileType = IDBDataFile::CLOUD;
if (fReportRealUse && (fCompressionType > 0))
{
try
{
fileSize = getCompressedDataSize(aFile);
}
catch (std::exception& ex)
{
cerr << ex.what();
}
}
else
fileType = IDBDataFile::UNBUFFERED;
fileSize = fs.size(fileName);
IDBFileSystem& fs = IDBFileSystem::getFs( fileType );
for (uint32_t i = 0; i < rootList.size(); i++)
if (fileSize > 0) // File exists, add to list
{
std::vector<struct BRM::EMEntry> entries;
(void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]);
std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
while ( iter != entries.end() ) //organize extents into files
{
//Find the size of this file
//string fileName;
char fileName[200];
(void)fileOp.getFileName( fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum);
string aFile(fileName); //convert between char* and string
off64_t fileSize = 0;
if (fReportRealUse && (fCompressionType > 0))
{
try
{
fileSize = getCompressedDataSize(aFile);
}
catch (std::exception& ex)
{
cerr << ex.what();
}
}
else
fileSize = fs.size( fileName );
if (fileSize > 0) // File exists, add to list
{
FileInfo aFileInfo;
aFileInfo.partition = entries[0].partitionNum;
aFileInfo.segment = entries[0].segmentNum;
aFileInfo.dbRoot = rootList[i];
aFileInfo.segFileName = aFile;
aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024);
aFiles.push_back(aFileInfo);
//cout.precision(15);
//cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize << "GB" << endl;
}
//erase the entries from this dbroot.
std::vector<struct BRM::EMEntry> entriesTrimed;
for (uint32_t m = 0; m < entries.size(); m++)
{
if ((entries[0].partitionNum != entries[m].partitionNum) || (entries[0].segmentNum != entries[m].segmentNum))
entriesTrimed.push_back(entries[m]);
}
entriesTrimed.swap(entries);
iter = entries.begin();
}
FileInfo aFileInfo;
aFileInfo.partition = entries[0].partitionNum;
aFileInfo.segment = entries[0].segmentNum;
aFileInfo.dbRoot = rootList[i];
aFileInfo.segFileName = aFile;
aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024);
aFiles.push_back(aFileInfo);
// cout.precision(15);
// cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize <<
// "GB" << endl;
}
boost::mutex::scoped_lock lk(columnMapLock);
//cout << "Current size of columnsMap is " << columnsMap.size() << endl;
allColumnMap::iterator colMapiter = wholeMap.find(fKey);
// erase the entries from this dbroot.
std::vector<struct BRM::EMEntry> entriesTrimed;
if (colMapiter != wholeMap.end())
for (uint32_t m = 0; m < entries.size(); m++)
{
(colMapiter->second)->insert(make_pair(fOid, aFiles));
activeThreadCounter->decr();
//cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl;
if ((entries[0].partitionNum != entries[m].partitionNum) ||
(entries[0].segmentNum != entries[m].segmentNum))
entriesTrimed.push_back(entries[m]);
}
entriesTrimed.swap(entries);
iter = entries.begin();
}
}
uint32_t fOid;
int32_t fCompressionType;
bool fReportRealUse;
int fKey;
boost::mutex::scoped_lock lk(columnMapLock);
// cout << "Current size of columnsMap is " << columnsMap.size() << endl;
allColumnMap::iterator colMapiter = wholeMap.find(fKey);
if (colMapiter != wholeMap.end())
{
(colMapiter->second)->insert(make_pair(fOid, aFiles));
activeThreadCounter->decr();
// cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl;
}
}
uint32_t fOid;
int32_t fCompressionType;
bool fReportRealUse;
int fKey;
};
//------------------------------------------------------------------------------
// Get file size from file name in bytestream object
//------------------------------------------------------------------------------
int WE_GetFileSizes::processFileName(
messageqcpp::ByteStream& bs,
std::string& errMsg, int key)
int WE_GetFileSizes::processFileName(messageqcpp::ByteStream& bs, std::string& errMsg, int key)
{
uint8_t rc = 0;
off_t fileSize = 0;
off_t compressedFileSize = 0;
errMsg.clear();
uint8_t rc = 0;
off_t fileSize = 0;
off_t compressedFileSize = 0;
errMsg.clear();
try
{
std::string fileName;
try
{
std::string fileName;
bs >> fileName;
fileSize = IDBPolicy::size(fileName.c_str());
compressedFileSize = IDBPolicy::compressedSize(fileName.c_str());
}
catch (std::exception& ex)
{
errMsg = ex.what();
rc = 1;
}
bs >> fileName;
fileSize = IDBPolicy::size(fileName.c_str());
compressedFileSize = IDBPolicy::compressedSize(fileName.c_str());
}
catch (std::exception& ex)
{
errMsg = ex.what();
rc = 1;
}
bs.reset();
bs << fileSize;
bs << compressedFileSize;
return rc;
bs.reset();
bs << fileSize;
bs << compressedFileSize;
return rc;
}
//------------------------------------------------------------------------------
// Process a table size based on input from the
// bytestream object.
//------------------------------------------------------------------------------
int WE_GetFileSizes::processTable(
messageqcpp::ByteStream& bs,
std::string& errMsg, int key)
int WE_GetFileSizes::processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key)
{
uint8_t rc = 0;
errMsg.clear();
uint8_t rc = 0;
errMsg.clear();
try
try
{
std::string aTableName;
std::string schemaName;
bool reportRealUse = false;
ByteStream::byte tmp8;
bs >> schemaName;
// cout << "schema: "<< schemaName << endl;
bs >> aTableName;
// cout << "tableName: " << aTableName << endl;
bs >> tmp8;
reportRealUse = (tmp8 != 0);
// get column oids
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr =
CalpontSystemCatalog::makeCalpontSystemCatalog(0);
CalpontSystemCatalog::TableName tableName;
tableName.schema = schemaName;
tableName.table = aTableName;
CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName);
CalpontSystemCatalog::ColType colType;
CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName);
int serverThreads = 20;
int serverQueueSize = serverThreads * 100;
threadpool::ThreadPool tp(serverThreads, serverQueueSize);
int totalSize = columnList.size() + dictOidList.size();
activeThreadCounter = new ActiveThreadCounter(totalSize);
columnMap* columnsMap = new columnMap();
{
std::string aTableName;
std::string schemaName;
bool reportRealUse = false;
ByteStream::byte tmp8;
bs >> schemaName;
//cout << "schema: "<< schemaName << endl;
boost::mutex::scoped_lock lk(columnMapLock);
wholeMap[key] = columnsMap;
}
bs >> aTableName;
//cout << "tableName: " << aTableName << endl;
bs >> tmp8;
reportRealUse = (tmp8 != 0);
for (uint32_t i = 0; i < columnList.size(); i++)
{
colType = systemCatalogPtr->colType(columnList[i].objnum);
tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key));
//get column oids
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(0);
CalpontSystemCatalog::TableName tableName;
tableName.schema = schemaName;
tableName.table = aTableName;
CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName);
CalpontSystemCatalog::ColType colType;
CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName);
int serverThreads = 20;
int serverQueueSize = serverThreads * 100;
threadpool::ThreadPool tp(serverThreads, serverQueueSize);
int totalSize = columnList.size() + dictOidList.size();
activeThreadCounter = new ActiveThreadCounter(totalSize);
if (colType.ddn.dictOID > 0)
tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key));
}
columnMap* columnsMap = new columnMap();
{
boost::mutex::scoped_lock lk(columnMapLock);
wholeMap[key] = columnsMap;
}
for (uint32_t i = 0; i < columnList.size(); i++)
{
colType = systemCatalogPtr->colType(columnList[i].objnum);
tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key));
if (colType.ddn.dictOID > 0)
tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key));
}
/* for (uint32_t i=0; i < dictOidList.size(); i++)
{
tp.invoke(ColumnThread(dictOidList[i].dictOID));
} */
//check whether all threads finish
int sleepTime = 100; // sleep 100 milliseconds between checks
struct timespec rm_ts;
rm_ts.tv_sec = sleepTime / 1000;
rm_ts.tv_nsec = sleepTime % 1000 * 1000000;
uint32_t currentActiveThreads = 10;
while (currentActiveThreads > 0)
{
#ifdef _MSC_VER
Sleep(sleepTime);
#else
struct timespec abs_ts;
do
/* for (uint32_t i=0; i < dictOidList.size(); i++)
{
abs_ts.tv_sec = rm_ts.tv_sec;
abs_ts.tv_nsec = rm_ts.tv_nsec;
}
while (nanosleep(&abs_ts, &rm_ts) < 0);
tp.invoke(ColumnThread(dictOidList[i].dictOID));
} */
// check whether all threads finish
int sleepTime = 100; // sleep 100 milliseconds between checks
struct timespec rm_ts;
rm_ts.tv_sec = sleepTime / 1000;
rm_ts.tv_nsec = sleepTime % 1000 * 1000000;
uint32_t currentActiveThreads = 10;
while (currentActiveThreads > 0)
{
#ifdef _MSC_VER
Sleep(sleepTime);
#else
struct timespec abs_ts;
do
{
abs_ts.tv_sec = rm_ts.tv_sec;
abs_ts.tv_nsec = rm_ts.tv_nsec;
} while (nanosleep(&abs_ts, &rm_ts) < 0);
#endif
currentActiveThreads = activeThreadCounter->cur();
}
currentActiveThreads = activeThreadCounter->cur();
}
catch (std::exception& ex)
}
catch (std::exception& ex)
{
// cout << "WE_GetFileSizes got exception-" << ex.what() <<
// std::endl;
errMsg = ex.what();
rc = 1;
}
// Build the message to send to the caller
bs.reset();
boost::mutex::scoped_lock lk(columnMapLock);
allColumnMap::iterator colMapiter = wholeMap.find(key);
if (colMapiter != wholeMap.end())
{
columnMap::iterator iter = colMapiter->second->begin();
uint64_t size;
Files::iterator it;
while (iter != colMapiter->second->end())
{
//cout << "WE_GetFileSizes got exception-" << ex.what() <<
// std::endl;
errMsg = ex.what();
rc = 1;
bs << iter->first;
// cout << "processTable::coloid = " << iter->first << endl;
size = iter->second.size();
bs << size;
for (it = iter->second.begin(); it != iter->second.end(); it++)
it->serialize(bs);
// cout << "length now is " << bs.length() << endl;
iter++;
}
//Build the message to send to the caller
bs.reset();
boost::mutex::scoped_lock lk(columnMapLock);
allColumnMap::iterator colMapiter = wholeMap.find(key);
wholeMap.erase(colMapiter);
}
if (colMapiter != wholeMap.end())
{
columnMap::iterator iter = colMapiter->second->begin();
uint64_t size;
Files::iterator it;
while ( iter != colMapiter->second->end())
{
bs << iter->first;
//cout << "processTable::coloid = " << iter->first << endl;
size = iter->second.size();
bs << size;
for (it = iter->second.begin(); it != iter->second.end(); it++)
it->serialize(bs);
//cout << "length now is " << bs.length() << endl;
iter++;
}
wholeMap.erase(colMapiter);
}
return rc;
return rc;
}
}
} // namespace WriteEngine

69
writeengine/server/we_getfilesizes.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_getfilesizes.h 4450 2013-04-15 14:13:24Z chao $
*
*******************************************************************************/
* $Id: we_getfilesizes.h 4450 2013-04-15 14:13:24Z chao $
*
*******************************************************************************/
#pragma once
#include <unistd.h>
@ -30,53 +30,54 @@
namespace WriteEngine
{
/** @brief Get all file sizes for the given table
*/
class WE_GetFileSizes
{
public:
static int processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
static int processFileName(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
public:
static int processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
static int processFileName(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
};
class ActiveThreadCounter
{
public:
ActiveThreadCounter(int size) : factiveThreadCount(size) {}
virtual ~ActiveThreadCounter() {}
public:
ActiveThreadCounter(int size) : factiveThreadCount(size)
{
}
virtual ~ActiveThreadCounter()
{
}
void decr()
void decr()
{
int atc;
for (;;)
{
int atc;
atomicops::atomicMb();
atc = factiveThreadCount;
for (;;)
{
atomicops::atomicMb();
atc = factiveThreadCount;
if (atc <= 0) // hopefully atc will never be < 0!
return;
if (atc <= 0) //hopefully atc will never be < 0!
return;
if (atomicops::atomicCAS(&factiveThreadCount, atc, (atc - 1)))
return;
if (atomicops::atomicCAS(&factiveThreadCount, atc, (atc - 1)))
return;
atomicops::atomicYield();
}
atomicops::atomicYield();
}
}
uint32_t cur()
{
return factiveThreadCount;
}
uint32_t cur()
{
return factiveThreadCount;
}
private:
ActiveThreadCounter(const ActiveThreadCounter& rhs);
ActiveThreadCounter& operator=(const ActiveThreadCounter& rhs);
private:
ActiveThreadCounter(const ActiveThreadCounter& rhs);
ActiveThreadCounter& operator=(const ActiveThreadCounter& rhs);
volatile int32_t factiveThreadCount;
volatile int32_t factiveThreadCount;
};
}
} // namespace WriteEngine

11
writeengine/server/we_message_handlers.h
View File

@ -16,19 +16,16 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_message_handlers.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
* $Id: we_message_handlers.h 4450 2013-01-21 14:13:24Z rdempsey $
*
*******************************************************************************/
#pragma once
#include "bytestream.h"
namespace WriteEngine
{
extern messageqcpp::ByteStream::byte doMsg1(messageqcpp::ByteStream& bs, std::string err);
extern messageqcpp::ByteStream::byte doMsg2(messageqcpp::ByteStream& bs, std::string err);
extern messageqcpp::ByteStream::byte doUpdateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string err);
}
} // namespace WriteEngine

176
writeengine/server/we_messages.h
View File

@ -16,105 +16,99 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_messages.h 4609 2013-04-19 15:32:02Z chao $
*
*******************************************************************************/
* $Id: we_messages.h 4609 2013-04-19 15:32:02Z chao $
*
*******************************************************************************/
#pragma once
namespace WriteEngine
{
enum ServerMessages
{
WE_SVR_LOOPBACK,
WE_SVR_DDL_KEEPALIVE,
WE_SVR_DML_KEEPALIVE,
WE_UPDATE_NEXTVAL,
WE_SVR_WRITE_SYSTABLE,
WE_SVR_WRITE_SYSCOLUMN,
WE_SVR_WRITE_CREATETABLEFILES,
WE_SVR_COMMIT_VERSION,
WE_SVR_ROLLBACK_BLOCKS,
WE_SVR_ROLLBACK_VERSION,
WE_SVR_DELETE_SYSTABLE,
WE_SVR_DELETE_SYSTABLES,
WE_SVR_DELETE_SYSCOLUMN,
WE_SVR_DELETE_SYSCOLUMN_ROW,
WE_SVR_WRITE_DROPFILES,
WE_SVR_UPDATE_SYSTABLE_AUTO,
WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL,
WE_SVR_UPDATE_SYSCOLUMN_TABLENAME,
WE_SVR_UPDATE_SYSCOLUMN_AUTO,
WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL,
WE_SVR_UPDATE_SYSCOLUMN_COLPOS,
WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN,
WE_SVR_UPDATE_SYSTABLE_TABLENAME,
WE_SVR_UPDATE_SYSTABLES_TABLENAME,
WE_SVR_DROP_PARTITIONS,
WE_SVR_SINGLE_INSERT,
WE_SVR_BATCH_KEEPALIVE,
WE_SVR_BATCH_INSERT,
WE_SVR_BATCH_INSERT_END,
WE_SVR_COMMIT_BATCH_AUTO_ON,
WE_SVR_ROLLBACK_BATCH_AUTO_ON,
WE_SVR_COMMIT_BATCH_AUTO_OFF,
WE_SVR_ROLLBACK_BATCH_AUTO_OFF,
WE_SVR_BATCH_AUTOON_REMOVE_META,
WE_SVR_UPDATE,
WE_SVR_FLUSH_FILES, //35
WE_SVR_DELETE,
WE_SVR_DML_BULKROLLBACK,
WE_SVR_DML_BULKROLLBACK_CLEANUP,
WE_SVR_FILL_COLUMN,
WE_SVR_WRITE_TRUNCATE,
WE_SVR_WRITE_DROPPARTITION,
WE_SVR_WRITE_DROPTABLE,
WE_SVR_DELETE_DDLLOG,
WE_SVR_FETCH_DDL_LOGS,
WE_SVR_REMOVE_TABLEDATA,
WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL,
WE_SVR_REDISTRIBUTE,
WE_SVR_CLOSE_CONNECTION,
WE_SVR_GET_FILESIZES,
WE_SVR_GET_FILESIZE,
WE_SVR_PURGEFD,
WE_END_TRANSACTION,
WE_SRV_FIX_ROWS,
WE_SVR_WRITE_CREATE_SYSCOLUMN,
WE_SVR_BATCH_INSERT_BINARY,
WE_SVR_GET_WRITTEN_LBIDS,
WE_SVR_LOOPBACK,
WE_SVR_DDL_KEEPALIVE,
WE_SVR_DML_KEEPALIVE,
WE_UPDATE_NEXTVAL,
WE_SVR_WRITE_SYSTABLE,
WE_SVR_WRITE_SYSCOLUMN,
WE_SVR_WRITE_CREATETABLEFILES,
WE_SVR_COMMIT_VERSION,
WE_SVR_ROLLBACK_BLOCKS,
WE_SVR_ROLLBACK_VERSION,
WE_SVR_DELETE_SYSTABLE,
WE_SVR_DELETE_SYSTABLES,
WE_SVR_DELETE_SYSCOLUMN,
WE_SVR_DELETE_SYSCOLUMN_ROW,
WE_SVR_WRITE_DROPFILES,
WE_SVR_UPDATE_SYSTABLE_AUTO,
WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL,
WE_SVR_UPDATE_SYSCOLUMN_TABLENAME,
WE_SVR_UPDATE_SYSCOLUMN_AUTO,
WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL,
WE_SVR_UPDATE_SYSCOLUMN_COLPOS,
WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN,
WE_SVR_UPDATE_SYSTABLE_TABLENAME,
WE_SVR_UPDATE_SYSTABLES_TABLENAME,
WE_SVR_DROP_PARTITIONS,
WE_SVR_SINGLE_INSERT,
WE_SVR_BATCH_KEEPALIVE,
WE_SVR_BATCH_INSERT,
WE_SVR_BATCH_INSERT_END,
WE_SVR_COMMIT_BATCH_AUTO_ON,
WE_SVR_ROLLBACK_BATCH_AUTO_ON,
WE_SVR_COMMIT_BATCH_AUTO_OFF,
WE_SVR_ROLLBACK_BATCH_AUTO_OFF,
WE_SVR_BATCH_AUTOON_REMOVE_META,
WE_SVR_UPDATE,
WE_SVR_FLUSH_FILES, // 35
WE_SVR_DELETE,
WE_SVR_DML_BULKROLLBACK,
WE_SVR_DML_BULKROLLBACK_CLEANUP,
WE_SVR_FILL_COLUMN,
WE_SVR_WRITE_TRUNCATE,
WE_SVR_WRITE_DROPPARTITION,
WE_SVR_WRITE_DROPTABLE,
WE_SVR_DELETE_DDLLOG,
WE_SVR_FETCH_DDL_LOGS,
WE_SVR_REMOVE_TABLEDATA,
WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL,
WE_SVR_REDISTRIBUTE,
WE_SVR_CLOSE_CONNECTION,
WE_SVR_GET_FILESIZES,
WE_SVR_GET_FILESIZE,
WE_SVR_PURGEFD,
WE_END_TRANSACTION,
WE_SRV_FIX_ROWS,
WE_SVR_WRITE_CREATE_SYSCOLUMN,
WE_SVR_BATCH_INSERT_BINARY,
WE_SVR_GET_WRITTEN_LBIDS,
WE_CLT_SRV_DATA = 100,
WE_CLT_SRV_EOD,
WE_CLT_SRV_CMD,
WE_CLT_SRV_ACK,
WE_CLT_SRV_NAK,
WE_CLT_SRV_PM_ERROR,
WE_CLT_SRV_KEEPALIVE,
WE_CLT_SRV_IMPFILENAME,
WE_CLT_SRV_IMPFILEERROR,
WE_CLT_SRV_CMDLINEARGS,
WE_CLT_SRV_STARTCPI,
WE_CLT_SRV_CPIPASS,
WE_CLT_SRV_CPIFAIL,
WE_CLT_SRV_ROLLBACK,
WE_CLT_SRV_CLEANUP,
WE_CLT_SRV_DATARQST,
WE_CLT_SRV_MODE,
WE_CLT_SRV_DBRCNT,
WE_CLT_SRV_BRMRPT,
WE_CLT_SRV_JOBID,
WE_CLT_SRV_JOBDATA,
WE_CLT_SRV_ERRLOG,
WE_CLT_SRV_BADLOG,
WE_CLT_SRV_CLEAR_TABLE_LOCK,
WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP
WE_CLT_SRV_DATA = 100,
WE_CLT_SRV_EOD,
WE_CLT_SRV_CMD,
WE_CLT_SRV_ACK,
WE_CLT_SRV_NAK,
WE_CLT_SRV_PM_ERROR,
WE_CLT_SRV_KEEPALIVE,
WE_CLT_SRV_IMPFILENAME,
WE_CLT_SRV_IMPFILEERROR,
WE_CLT_SRV_CMDLINEARGS,
WE_CLT_SRV_STARTCPI,
WE_CLT_SRV_CPIPASS,
WE_CLT_SRV_CPIFAIL,
WE_CLT_SRV_ROLLBACK,
WE_CLT_SRV_CLEANUP,
WE_CLT_SRV_DATARQST,
WE_CLT_SRV_MODE,
WE_CLT_SRV_DBRCNT,
WE_CLT_SRV_BRMRPT,
WE_CLT_SRV_JOBID,
WE_CLT_SRV_JOBDATA,
WE_CLT_SRV_ERRLOG,
WE_CLT_SRV_BADLOG,
WE_CLT_SRV_CLEAR_TABLE_LOCK,
WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP
};
}

12
writeengine/server/we_msg1.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
#include <unistd.h>
@ -30,11 +30,9 @@ using namespace messageqcpp;
namespace WriteEngine
{
ByteStream::byte doMsg1(ByteStream& bs, std::string err)
{
return 0;
}
return 0;
}
} // namespace WriteEngine

12
writeengine/server/we_msg2.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
#include <unistd.h>
@ -30,11 +30,9 @@ using namespace messageqcpp;
namespace WriteEngine
{
ByteStream::byte doMsg2(ByteStream& bs, std::string err)
{
return 0;
}
return 0;
}
} // namespace WriteEngine

54
writeengine/server/we_observer.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
/*
* Observer.cpp
@ -31,77 +31,71 @@
#include "we_observer.h"
namespace WriteEngine
{
//-----------------------------------------------------------------------------
// ctor
Observer::Observer()
{
}
//-----------------------------------------------------------------------------
//dtor
// dtor
Observer::~Observer()
{
//
//
}
//-----------------------------------------------------------------------------
//ctor
// ctor
Subject::Subject()
{
}
//-----------------------------------------------------------------------------
//dtor
// dtor
Subject::~Subject()
{
}
//-----------------------------------------------------------------------------
void Subject::attach(Observer* Obs)
{
boost::mutex::scoped_lock aLstLock;
fObs.push_back(Obs);
boost::mutex::scoped_lock aLstLock;
fObs.push_back(Obs);
}
//-----------------------------------------------------------------------------
void Subject::detach(Observer* Obs)
{
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
while (aIt != fObs.end())
while (aIt != fObs.end())
{
if ((*aIt) == Obs)
{
if ((*aIt) == Obs)
{
fObs.erase(aIt);
break;
}
fObs.erase(aIt);
break;
}
}
}
//-----------------------------------------------------------------------------
void Subject::notify()
{
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
while (aIt != fObs.end())
{
(*aIt)->update(this);
}
while (aIt != fObs.end())
{
(*aIt)->update(this);
}
}
//-----------------------------------------------------------------------------
}// namespace WriteEngine
} // namespace WriteEngine

42
writeengine/server/we_observer.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
* $Id$
*
*******************************************************************************/
/*
* Observer.h
@ -31,37 +31,33 @@
#include <list>
namespace WriteEngine
{
class Subject; // forward deceleration
class Subject; // forward deceleration
class Observer
{
public:
virtual ~Observer();
virtual bool update(Subject* pSub) = 0;
public:
virtual ~Observer();
virtual bool update(Subject* pSub) = 0;
protected:
Observer();
protected:
Observer();
};
class Subject
{
public:
Subject();
virtual ~Subject();
public:
Subject();
virtual ~Subject();
virtual void attach(Observer* Obs);
virtual void detach(Observer* Obs);
virtual void notify();
virtual void attach(Observer* Obs);
virtual void detach(Observer* Obs);
virtual void notify();
private:
typedef std::list<Observer*> Observers;
Observers fObs;
private:
typedef std::list<Observer*> Observers;
Observers fObs;
};
}
} // namespace WriteEngine

1496
writeengine/server/we_readthread.cpp
View File

File diff suppressed because it is too large Load Diff

137
writeengine/server/we_readthread.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_readthread.h 4609 2013-04-19 15:32:02Z chao $
*
*******************************************************************************/
* $Id: we_readthread.h 4609 2013-04-19 15:32:02Z chao $
*
*******************************************************************************/
#pragma once
#include <unistd.h>
@ -36,92 +36,82 @@
namespace WriteEngine
{
class ReadThread
{
public:
explicit ReadThread(const messageqcpp::IOSocket& ios);
virtual ~ReadThread();
public:
explicit ReadThread(const messageqcpp::IOSocket& ios);
virtual ~ReadThread();
virtual void operator()();
virtual void operator()();
// protected:
public:
messageqcpp::IOSocket fIos;
messageqcpp::ByteStream fIbs;
//protected:
public:
messageqcpp::IOSocket fIos;
messageqcpp::ByteStream fIbs;
private:
//defaults okay
//ReadThread(const ReadThread& rhs);
//ReadThread& operator=(const ReadThread& rhs);
private:
// defaults okay
// ReadThread(const ReadThread& rhs);
// ReadThread& operator=(const ReadThread& rhs);
};
class DmlReadThread: public ReadThread
class DmlReadThread : public ReadThread
{
public:
explicit DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs );
virtual ~DmlReadThread();
public:
explicit DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~DmlReadThread();
virtual void operator()();
private:
boost::shared_ptr<WE_DMLCommandProc> fWeDMLprocessor;
boost::shared_ptr<WE_DDLCommandProc> fWeDDLprocessor;
virtual void operator()();
private:
boost::shared_ptr<WE_DMLCommandProc> fWeDMLprocessor;
boost::shared_ptr<WE_DDLCommandProc> fWeDDLprocessor;
};
class SplitterReadThread: public ReadThread
class SplitterReadThread : public ReadThread
{
public:
SplitterReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs);
SplitterReadThread(const SplitterReadThread& rhs);
virtual ~SplitterReadThread();
public:
SplitterReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs);
SplitterReadThread(const SplitterReadThread& rhs);
virtual ~SplitterReadThread();
virtual void operator()();
virtual void operator()();
private:
// WEDataLoader* fpWeDataLoader;
WEDataLoader fWeDataLoader;
private:
//WEDataLoader* fpWeDataLoader;
WEDataLoader fWeDataLoader;
friend class ReadThreadFactory;
//friend class WEDataLoader;
friend class ReadThreadFactory;
// friend class WEDataLoader;
};
//------------------------------------------------------------------------------
// Thread object that receives a cleartablelock tool command.
//------------------------------------------------------------------------------
class ClearTableLockReadThread : public ReadThread
{
public:
ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~ClearTableLockReadThread();
public:
ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~ClearTableLockReadThread();
virtual void operator()();
private:
boost::shared_ptr<WE_ClearTableLockCmd> fWeClearTableLockCmd;
virtual void operator()();
private:
boost::shared_ptr<WE_ClearTableLockCmd> fWeClearTableLockCmd;
};
//------------------------------------------------------------------------------
// Thread object that receives a redistributeDB tool command.
//------------------------------------------------------------------------------
class RedistributeReadThread : public ReadThread
{
public:
RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~RedistributeReadThread();
public:
RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~RedistributeReadThread();
virtual void operator()();
private:
virtual void operator()();
private:
};
//------------------------------------------------------------------------------
@ -129,30 +119,29 @@ private:
//------------------------------------------------------------------------------
class GetFileSizeThread : public ReadThread
{
public:
GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& ibs, BRM::DBRM& dbrm);
virtual ~GetFileSizeThread();
public:
GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& ibs, BRM::DBRM& dbrm);
virtual ~GetFileSizeThread();
virtual void operator()();
virtual void operator()();
private:
boost::shared_ptr<WE_GetFileSizes> fWeGetFileSizes;
int key;
private:
boost::shared_ptr<WE_GetFileSizes> fWeGetFileSizes;
int key;
};
class ReadThreadFactory
{
public:
ReadThreadFactory() {}
virtual ~ReadThreadFactory() {}
public:
static void CreateReadThread(threadpool::ThreadPool& Tp, IOSocket& ios, BRM::DBRM& dbrm);
public:
ReadThreadFactory()
{
}
virtual ~ReadThreadFactory()
{
}
public:
static void CreateReadThread(threadpool::ThreadPool& Tp, IOSocket& ios, BRM::DBRM& dbrm);
};
}
} // namespace WriteEngine

437
writeengine/server/we_server.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id: we_server.cpp 4700 2013-07-08 16:43:49Z bpaul $
*
*******************************************************************************/
* $Id: we_server.cpp 4700 2013-07-08 16:43:49Z bpaul $
*
*******************************************************************************/
#include <unistd.h>
#include <iostream>
@ -59,295 +59,280 @@ using namespace WriteEngine;
namespace
{
class Opt
{
public:
int m_debug;
bool m_fg;
Opt(int argc, char *argv[])
:m_debug(0),
m_fg(false)
public:
int m_debug;
bool m_fg;
Opt(int argc, char* argv[]) : m_debug(0), m_fg(false)
{
int c;
while ((c = getopt(argc, argv, "df")) != EOF)
{
int c;
while ((c = getopt(argc, argv, "df")) != EOF)
{
switch(c)
{
case 'd':
m_debug++;
break;
case 'f':
m_fg= true;
break;
case '?':
default:
break;
}
}
switch (c)
{
case 'd': m_debug++; break;
case 'f': m_fg = true; break;
case '?':
default: break;
}
}
}
};
class ServiceWriteEngine: public Service, public Opt
class ServiceWriteEngine : public Service, public Opt
{
void log(logging::LOG_TYPE type, const std::string &str)
{
logging::LoggingID logid(SUBSYSTEM_ID_WE_SRV);
logging::Message::Args args;
logging::Message msg(1);
args.add(str);
msg.format(args);
logging::Logger logger(logid.fSubsysID);
logger.logMessage(type, msg, logid);
}
int setupResources();
void setupChildSignalHandlers();
public:
ServiceWriteEngine(const Opt &opt)
:Service("WriteEngine"), Opt(opt)
{ }
void LogErrno() override
{
log(logging::LOG_TYPE_CRITICAL, strerror(errno));
}
void ParentLogChildMessage(const std::string &str) override
{
log(logging::LOG_TYPE_INFO, str);
}
int Child() override;
int Run() { return m_fg ? Child() : RunForking(); }
};
void log(logging::LOG_TYPE type, const std::string& str)
{
logging::LoggingID logid(SUBSYSTEM_ID_WE_SRV);
logging::Message::Args args;
logging::Message msg(1);
args.add(str);
msg.format(args);
logging::Logger logger(logid.fSubsysID);
logger.logMessage(type, msg, logid);
}
int setupResources();
void setupChildSignalHandlers();
public:
ServiceWriteEngine(const Opt& opt) : Service("WriteEngine"), Opt(opt)
{
}
void LogErrno() override
{
log(logging::LOG_TYPE_CRITICAL, strerror(errno));
}
void ParentLogChildMessage(const std::string& str) override
{
log(logging::LOG_TYPE_INFO, str);
}
int Child() override;
int Run()
{
return m_fg ? Child() : RunForking();
}
};
void added_a_pm(int)
{
logging::LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("we_server caught SIGHUP. Resetting connections");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid);
joblist::DistributedEngineComm::reset();
}
logging::LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("we_server caught SIGHUP. Resetting connections");
msg.format(args1);
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid);
joblist::DistributedEngineComm::reset();
}
} // namespace
int ServiceWriteEngine::setupResources()
{
#ifndef _MSC_VER
struct rlimit rlim;
struct rlimit rlim;
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -1;
}
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -1;
}
rlim.rlim_cur = rlim.rlim_max = 65536;
rlim.rlim_cur = rlim.rlim_max = 65536;
if (setrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -2;
}
if (setrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -2;
}
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -3;
}
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -3;
}
if (rlim.rlim_cur != 65536)
{
return -4;
}
if (rlim.rlim_cur != 65536)
{
return -4;
}
#endif
return 0;
return 0;
}
void ServiceWriteEngine::setupChildSignalHandlers()
{
#ifndef _MSC_VER
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = added_a_pm;
sigaction(SIGHUP, &sa, 0);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = added_a_pm;
sigaction(SIGHUP, &sa, 0);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
memset(&sa, 0, sizeof(sa));
sa.sa_handler = fatalHandler;
sigaction(SIGSEGV, &sa, 0);
sigaction(SIGABRT, &sa, 0);
sigaction(SIGFPE, &sa, 0);
memset(&sa, 0, sizeof(sa));
sa.sa_handler = fatalHandler;
sigaction(SIGSEGV, &sa, 0);
sigaction(SIGABRT, &sa, 0);
sigaction(SIGFPE, &sa, 0);
#endif
}
int ServiceWriteEngine::Child()
{
setupChildSignalHandlers();
setupChildSignalHandlers();
// Init WriteEngine Wrapper (including Config Columnstore.xml cache)
WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_WE_SRV );
// Init WriteEngine Wrapper (including Config Columnstore.xml cache)
WriteEngine::WriteEngineWrapper::init(WriteEngine::SUBSYSTEM_ID_WE_SRV);
#ifdef _MSC_VER
// In windows, initializing the wrapper (A dll) does not set the static variables
// in the main program
idbdatafile::IDBPolicy::configIDBPolicy();
// In windows, initializing the wrapper (A dll) does not set the static variables
// in the main program
idbdatafile::IDBPolicy::configIDBPolicy();
#endif
Config weConfig;
Config weConfig;
ostringstream serverParms;
serverParms << "pm" << weConfig.getLocalModuleID() << "_WriteEngineServer";
ostringstream serverParms;
serverParms << "pm" << weConfig.getLocalModuleID() << "_WriteEngineServer";
// Create MessageQueueServer, with one retry in case the call to bind the
// known port fails with "Address already in use".
boost::scoped_ptr<MessageQueueServer> mqs;
bool tellUser = true;
// Create MessageQueueServer, with one retry in case the call to bind the
// known port fails with "Address already in use".
boost::scoped_ptr<MessageQueueServer> mqs;
bool tellUser = true;
for (;;)
for (;;)
{
try
{
try
{
mqs.reset(new MessageQueueServer(serverParms.str()));
break;
}
// @bug4393 Error Handling for MessageQueueServer constructor exception
catch (runtime_error& re)
{
string what = re.what();
if (what.find("Address already in use") != string::npos)
{
if (tellUser)
{
cerr << "Address already in use, retrying..." << endl;
tellUser = false;
}
sleep(5);
}
else
{
// If/when a common logging class or function is added to the
// WriteEngineServer, we should use that. In the mean time,
// I will log this errmsg with inline calls to the logging.
logging::Message::Args args;
logging::Message message;
string errMsg("WriteEngineServer failed to initiate: ");
errMsg += what;
args.add( errMsg );
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
NotifyServiceInitializationFailed();
return 2;
}
}
mqs.reset(new MessageQueueServer(serverParms.str()));
break;
}
int err = 0;
if (!m_debug)
err = setupResources();
string errMsg;
switch (err)
// @bug4393 Error Handling for MessageQueueServer constructor exception
catch (runtime_error& re)
{
case -1:
case -3:
errMsg = "Error getting file limits, please see non-root install documentation";
break;
string what = re.what();
case -2:
errMsg = "Error setting file limits, please see non-root install documentation";
break;
if (what.find("Address already in use") != string::npos)
{
if (tellUser)
{
cerr << "Address already in use, retrying..." << endl;
tellUser = false;
}
case -4:
errMsg = "Could not install file limits to required value, please see non-root install documentation";
break;
default:
break;
}
if (err < 0)
{
Oam oam;
sleep(5);
}
else
{
// If/when a common logging class or function is added to the
// WriteEngineServer, we should use that. In the mean time,
// I will log this errmsg with inline calls to the logging.
logging::Message::Args args;
logging::Message message;
args.add( errMsg );
string errMsg("WriteEngineServer failed to initiate: ");
errMsg += what;
args.add(errMsg);
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
cerr << errMsg << endl;
ml.logCriticalMessage(message);
NotifyServiceInitializationFailed();
return 2;
}
}
}
int err = 0;
if (!m_debug)
err = setupResources();
string errMsg;
IOSocket ios;
size_t mt = 20;
size_t qs = mt * 100;
ThreadPool tp(mt, qs);
switch (err)
{
case -1:
case -3: errMsg = "Error getting file limits, please see non-root install documentation"; break;
cout << "WriteEngineServer is ready" << endl;
NotifyServiceStarted();
case -2: errMsg = "Error setting file limits, please see non-root install documentation"; break;
BRM::DBRM dbrm;
case -4:
errMsg = "Could not install file limits to required value, please see non-root install documentation";
break;
for (;;)
default: break;
}
if (err < 0)
{
Oam oam;
logging::Message::Args args;
logging::Message message;
args.add(errMsg);
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage(message);
cerr << errMsg << endl;
NotifyServiceInitializationFailed();
return 2;
}
IOSocket ios;
size_t mt = 20;
size_t qs = mt * 100;
ThreadPool tp(mt, qs);
cout << "WriteEngineServer is ready" << endl;
NotifyServiceStarted();
BRM::DBRM dbrm;
for (;;)
{
try // BUG 4834 -
{
try // BUG 4834 -
{
ios = mqs->accept();
//tp.invoke(ReadThread(ios));
ReadThreadFactory::CreateReadThread(tp, ios, dbrm);
{
/* logging::Message::Args args;
logging::Message message;
string aMsg("WriteEngineServer : New incoming connection");
args.add(aMsg);
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logInfoMessage( message ); */
}
}
catch (std::exception& ex) // BUG 4834 - log the exception
{
logging::Message::Args args;
logging::Message message;
string errMsg("WriteEngineServer : Exception caught on accept(): ");
errMsg += ex.what();
args.add( errMsg );
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
break;
}
ios = mqs->accept();
// tp.invoke(ReadThread(ios));
ReadThreadFactory::CreateReadThread(tp, ios, dbrm);
{
/* logging::Message::Args args;
logging::Message message;
string aMsg("WriteEngineServer : New incoming connection");
args.add(aMsg);
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logInfoMessage( message ); */
}
}
catch (std::exception& ex) // BUG 4834 - log the exception
{
logging::Message::Args args;
logging::Message message;
string errMsg("WriteEngineServer : Exception caught on accept(): ");
errMsg += ex.what();
args.add(errMsg);
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage(message);
break;
}
}
//It is an error to reach here...
return 1;
// It is an error to reach here...
return 1;
}
int main(int argc, char** argv)
{
Opt opt(argc, argv);
Opt opt(argc, argv);
// Set locale language
setlocale(LC_ALL, "");
setlocale(LC_NUMERIC, "C");
// This is unset due to the way we start it
program_invocation_short_name = const_cast<char*>("WriteEngineServ");
// Initialize the charset library
MY_INIT(argv[0]);
// Set locale language
setlocale(LC_ALL, "");
setlocale(LC_NUMERIC, "C");
// This is unset due to the way we start it
program_invocation_short_name = const_cast<char*>("WriteEngineServ");
// Initialize the charset library
MY_INIT(argv[0]);
return ServiceWriteEngine(opt).Run();
return ServiceWriteEngine(opt).Run();
}