database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
writeengine/server/resource.h
View File

@ -1,14 +1,14 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by cpimport.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by cpimport.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

358
writeengine/server/we_brmrprtparser.cpp
View File

@ -1,176 +1,182 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_brmrprtparser.cpp
*
* Created on: Dec 13, 2011
* Author: bpaul
*/
#include <fstream>
#include <iostream>
using namespace std;
#include "we_brmrprtparser.h"
namespace WriteEngine
{
BrmReportParser::BrmReportParser():fRptFile()
{
}
BrmReportParser::~BrmReportParser()
{
}
//------------------------------------------------------------------------------
// Serialize specified RptFileName into the ByteStream Bs.
// Serialization is done one record at a time. File should be an ASCII file
// with newlines markers, because getline() is used.
// Function is limited to records that are no longer than 255 bytes.
// If a record is longer than 255 bytes, the function misbehaves.
//------------------------------------------------------------------------------
bool BrmReportParser::serialize(std::string RptFileName,
messageqcpp::ByteStream& Bs)
{
try
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch(std::exception& ex)
{
cout <<"Failed to open BRMRptFile "<< RptFileName <<endl;
cout << ex.what() << endl;
throw runtime_error(ex.what());
}
if(fRptFile.good())
{
char aBuff[10240];
unsigned int aLen=0;
while(fRptFile.good() && !fRptFile.eof())
{
fRptFile.getline(aBuff, sizeof(aBuff)-1);
aLen=fRptFile.gcount();
if((aLen != (sizeof(aBuff)-1)) && (aLen>0))
{
//aBuff[aLen-1] = '\n';
//aBuff[aLen]=0;
//cout << "Data Read " << aBuff <<endl;
if(aBuff[0] != '#') // do not serialize comments
{
std::string strData = aBuff;
Bs << strData;
}
}
}// while
fRptFile.close();
cout << "Closed File : "<< RptFileName <<" "<< errno << endl;
}
else
{
cout << "Failed to open : "<< RptFileName <<" "<< errno << endl;
return false;
}
return true;
}
//------------------------------------------------------------------------------
// Serialize specified RptFileName into the ByteStream Bs.
// Serialization is done in 8192 byte blocks instead of by line or by record,
// so that this function will work for both character and binary files.
//------------------------------------------------------------------------------
bool BrmReportParser::serializeBlocks(std::string RptFileName,
messageqcpp::ByteStream& Bs)
{
try
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch (std::exception& ex)
{
cout <<"Failed to open Report File "<< RptFileName << endl;
cout << ex.what() << endl;
return false;
}
if (fRptFile.good())
{
char aBuff[8192];
unsigned int aLen=0;
std::string strBuff;
while ( fRptFile.good() )
{
fRptFile.read(aBuff, sizeof(aBuff));
aLen = fRptFile.gcount();
if (aLen > 0)
{
strBuff.assign( aBuff, aLen );
Bs << strBuff;
}
}
fRptFile.close();
cout << "Closed Report File : "<< RptFileName << endl;
}
else
{
std::ostringstream oss;
oss << "Failed to open Report File " << RptFileName << endl;
cout << oss.str() << endl;
return false;
}
return true;
}
void BrmReportParser::unserialize(messageqcpp::ByteStream& Bs)
{
//TODO to be changed. left it here to understand how to implement
/*
ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN);
ReturnedColumn::unserialize(b); // parent class unserialize
b >> (uint32_t&) fOid;
b >> fData;
b >> reinterpret_cast<ByteStream::doublebyte&>(fReturnAll);
b >> (uint32_t&) fSequence;
*/
std::string aStrLine;
while(Bs.length()>0)
{
Bs >> aStrLine;
cout << aStrLine;
}
}
} /* namespace WriteEngine */
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_brmrprtparser.cpp
*
* Created on: Dec 13, 2011
* Author: bpaul
*/
#include <fstream>
#include <iostream>
using namespace std;
#include "we_brmrprtparser.h"
namespace WriteEngine
{
BrmReportParser::BrmReportParser(): fRptFile()
{
}
BrmReportParser::~BrmReportParser()
{
}
//------------------------------------------------------------------------------
// Serialize specified RptFileName into the ByteStream Bs.
// Serialization is done one record at a time. File should be an ASCII file
// with newlines markers, because getline() is used.
// Function is limited to records that are no longer than 255 bytes.
// If a record is longer than 255 bytes, the function misbehaves.
//------------------------------------------------------------------------------
bool BrmReportParser::serialize(std::string RptFileName,
messageqcpp::ByteStream& Bs)
{
try
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch (std::exception& ex)
{
cout << "Failed to open BRMRptFile " << RptFileName << endl;
cout << ex.what() << endl;
throw runtime_error(ex.what());
}
if (fRptFile.good())
{
char aBuff[10240];
unsigned int aLen = 0;
while (fRptFile.good() && !fRptFile.eof())
{
fRptFile.getline(aBuff, sizeof(aBuff) - 1);
aLen = fRptFile.gcount();
if ((aLen != (sizeof(aBuff) - 1)) && (aLen > 0))
{
//aBuff[aLen-1] = '\n';
//aBuff[aLen]=0;
//cout << "Data Read " << aBuff <<endl;
if (aBuff[0] != '#') // do not serialize comments
{
std::string strData = aBuff;
Bs << strData;
}
}
}// while
fRptFile.close();
cout << "Closed File : " << RptFileName << " " << errno << endl;
}
else
{
cout << "Failed to open : " << RptFileName << " " << errno << endl;
return false;
}
return true;
}
//------------------------------------------------------------------------------
// Serialize specified RptFileName into the ByteStream Bs.
// Serialization is done in 8192 byte blocks instead of by line or by record,
// so that this function will work for both character and binary files.
//------------------------------------------------------------------------------
bool BrmReportParser::serializeBlocks(std::string RptFileName,
messageqcpp::ByteStream& Bs)
{
try
{
fRptFile.open(RptFileName.c_str(), ifstream::in);
}
catch (std::exception& ex)
{
cout << "Failed to open Report File " << RptFileName << endl;
cout << ex.what() << endl;
return false;
}
if (fRptFile.good())
{
char aBuff[8192];
unsigned int aLen = 0;
std::string strBuff;
while ( fRptFile.good() )
{
fRptFile.read(aBuff, sizeof(aBuff));
aLen = fRptFile.gcount();
if (aLen > 0)
{
strBuff.assign( aBuff, aLen );
Bs << strBuff;
}
}
fRptFile.close();
cout << "Closed Report File : " << RptFileName << endl;
}
else
{
std::ostringstream oss;
oss << "Failed to open Report File " << RptFileName << endl;
cout << oss.str() << endl;
return false;
}
return true;
}
void BrmReportParser::unserialize(messageqcpp::ByteStream& Bs)
{
//TODO to be changed. left it here to understand how to implement
/*
ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN);
ReturnedColumn::unserialize(b); // parent class unserialize
b >> (uint32_t&) fOid;
b >> fData;
b >> reinterpret_cast<ByteStream::doublebyte&>(fReturnAll);
b >> (uint32_t&) fSequence;
*/
std::string aStrLine;
while (Bs.length() > 0)
{
Bs >> aStrLine;
cout << aStrLine;
}
}
} /* namespace WriteEngine */

12
writeengine/server/we_brmrprtparser.h
View File

@ -59,16 +59,16 @@ namespace WriteEngine
class BrmReportParser
{
public:
BrmReportParser();
virtual ~BrmReportParser();
BrmReportParser();
virtual ~BrmReportParser();
public:
bool serialize(std::string RptFileName, messageqcpp::ByteStream& Bs);
bool serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs);
void unserialize(messageqcpp::ByteStream& Bs);
bool serialize(std::string RptFileName, messageqcpp::ByteStream& Bs);
bool serializeBlocks(std::string RptFileName, messageqcpp::ByteStream& Bs);
void unserialize(messageqcpp::ByteStream& Bs);
private:
ifstream fRptFile;
ifstream fRptFile;
};

163
writeengine/server/we_cleartablelockcmd.cpp
View File

@ -37,53 +37,54 @@ namespace WriteEngine
// bytestream object.
//------------------------------------------------------------------------------
int WE_ClearTableLockCmd::processRollback(
messageqcpp::ByteStream& bs,
std::string& errMsg)
messageqcpp::ByteStream& bs,
std::string& errMsg)
{
uint8_t rc = 0;
errMsg.clear();
uint8_t rc = 0;
errMsg.clear();
try {
uint32_t tableOID;
uint64_t tableLockID;
std::string tableName;
std::string appName;
try
{
uint32_t tableOID;
uint64_t tableLockID;
std::string tableName;
std::string appName;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processRollback for " << fUserDesc;
bs >> tableLockID;
std::cout << ": tableLock-"<< tableLockID;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processRollback for " << fUserDesc;
bs >> tableLockID;
std::cout << ": tableLock-" << tableLockID;
bs >> tableOID;
std::cout << "; tableOID-" << tableOID;
bs >> tableOID;
std::cout << "; tableOID-" << tableOID;
bs >> tableName;
std::cout << "; table-" << tableName;
bs >> tableName;
std::cout << "; table-" << tableName;
bs >> appName;
std::cout << "; app-" << appName << std::endl;
bs >> appName;
std::cout << "; app-" << appName << std::endl;
int we_rc = fWEWrapper.bulkRollback(
tableOID,
tableLockID,
tableName,
appName,
false, // no extra debug logging to the console
errMsg );
int we_rc = fWEWrapper.bulkRollback(
tableOID,
tableLockID,
tableName,
appName,
false, // no extra debug logging to the console
errMsg );
if (we_rc != NO_ERROR)
rc = 2;
}
catch(std::exception& ex)
{
std::cout << "ClearTableLockCmd::Rollback exception-" << ex.what() <<
std::endl;
errMsg = ex.what();
rc = 1;
}
if (we_rc != NO_ERROR)
rc = 2;
}
catch (std::exception& ex)
{
std::cout << "ClearTableLockCmd::Rollback exception-" << ex.what() <<
std::endl;
errMsg = ex.what();
rc = 1;
}
return rc;
return rc;
}
//------------------------------------------------------------------------------
@ -95,58 +96,60 @@ int WE_ClearTableLockCmd::processRollback(
// care of db file cleanup in the "unsuccessful" case.
//------------------------------------------------------------------------------
int WE_ClearTableLockCmd::processCleanup(
messageqcpp::ByteStream& bs,
std::string& errMsg)
messageqcpp::ByteStream& bs,
std::string& errMsg)
{
uint8_t rc = 0;
errMsg.clear();
uint8_t rc = 0;
errMsg.clear();
try {
uint32_t tableOID;
try
{
uint32_t tableOID;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processCleanup for " << fUserDesc;
bs >> tableOID;
std::cout << ": tableOID-" << tableOID << std::endl;
// May want to eventually comment out this logging to stdout,
// but it shouldn't hurt to keep in here.
std::cout << "ClearTableLockCmd::processCleanup for " << fUserDesc;
bs >> tableOID;
std::cout << ": tableOID-" << tableOID << std::endl;
// On an HDFS system, this is where we delete any DB files
// (ex: *.orig) that we no longer need.
if ((idbdatafile::IDBPolicy::useHdfs()) &&
(bs.length() >= sizeof(messageqcpp::ByteStream::byte)))
{
messageqcpp::ByteStream::byte deleteHdfsTempDbFiles;
bs >> deleteHdfsTempDbFiles;
if (deleteHdfsTempDbFiles)
{
std::string endDbErrMsg;
ConfirmHdfsDbFile confirmHdfs;
// On an HDFS system, this is where we delete any DB files
// (ex: *.orig) that we no longer need.
if ((idbdatafile::IDBPolicy::useHdfs()) &&
(bs.length() >= sizeof(messageqcpp::ByteStream::byte)))
{
messageqcpp::ByteStream::byte deleteHdfsTempDbFiles;
bs >> deleteHdfsTempDbFiles;
if (deleteHdfsTempDbFiles)
{
std::string endDbErrMsg;
ConfirmHdfsDbFile confirmHdfs;
// We always pass "true", as this only applies to the "success-
// ful" case. See comments that precede this function.
int endRc = confirmHdfs.endDbFileListFromMetaFile(
tableOID, true, endDbErrMsg);
int endRc = confirmHdfs.endDbFileListFromMetaFile(
tableOID, true, endDbErrMsg);
// In this case, a deletion error is not fatal.
// so we don't propagate a bad return code. We
// may want to add syslog msg (TBD)
if (endRc != NO_ERROR)
std::cout << "Orig db file deletion error: " <<
endDbErrMsg << std::endl;
}
}
// In this case, a deletion error is not fatal.
// so we don't propagate a bad return code. We
// may want to add syslog msg (TBD)
if (endRc != NO_ERROR)
std::cout << "Orig db file deletion error: " <<
endDbErrMsg << std::endl;
}
}
BulkRollbackMgr::deleteMetaFile( tableOID );
}
catch(std::exception& ex)
{
std::cout << "ClearTableLockCmd::Cleanup exception-" << ex.what() <<
std::endl;
errMsg = ex.what();
rc = 1;
}
BulkRollbackMgr::deleteMetaFile( tableOID );
}
catch (std::exception& ex)
{
std::cout << "ClearTableLockCmd::Cleanup exception-" << ex.what() <<
std::endl;
errMsg = ex.what();
rc = 1;
}
return rc;
return rc;
}
}

37
writeengine/server/we_cleartablelockcmd.h
View File

@ -28,34 +28,35 @@
#include "writeengine.h"
namespace WriteEngine {
namespace WriteEngine
{
/** @brief Process messages from a cleartablelock or filesplitter client
*/
class WE_ClearTableLockCmd
{
public:
/** @brief WE_ClearTableLockCmd constructor
*/
WE_ClearTableLockCmd(const char* userDesc) : fUserDesc(userDesc) { }
/** @brief WE_ClearTableLockCmd constructor
*/
WE_ClearTableLockCmd(const char* userDesc) : fUserDesc(userDesc) { }
/** @brief Process bulk rollback request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processRollback(messageqcpp::ByteStream& ibs,
std::string& errMsg);
/** @brief Process bulk rollback request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processRollback(messageqcpp::ByteStream& ibs,
std::string& errMsg);
/** @brief Process bulk rollback cleanup request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processCleanup (messageqcpp::ByteStream& ibs,
std::string& errMsg);
/** @brief Process bulk rollback cleanup request
* @param ibs Input byte stream
* @param errMsg Return error message
*/
int processCleanup (messageqcpp::ByteStream& ibs,
std::string& errMsg);
private:
WriteEngineWrapper fWEWrapper; // WriteEngineWrapper object
std::string fUserDesc;
WriteEngineWrapper fWEWrapper; // WriteEngineWrapper object
std::string fUserDesc;
};
}

139
writeengine/server/we_cpifeederthread.cpp
View File

@ -54,37 +54,38 @@ namespace WriteEngine
void WECpiFeederRunner::operator()()
{
fOwner.feedData2Cpi();
cout << "Finished running Feeder Thread!!" << endl;
fOwner.feedData2Cpi();
cout << "Finished running Feeder Thread!!" << endl;
}
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
WECpiFeederThread::WECpiFeederThread(WEDataLoader& Ref):
fOwner(Ref),
fpThread(0),
fContinue(true),
fStopped(true)
fOwner(Ref),
fpThread(0),
fContinue(true),
fStopped(true)
{
cout << "Inside WECpiFeederThread constructor"<< endl;
cout << "Inside WECpiFeederThread constructor" << endl;
}
//------------------------------------------------------------------------------
WECpiFeederThread::~WECpiFeederThread()
{
if (fpThread)
{
delete fpThread;
}
fpThread=0;
if (fpThread)
{
delete fpThread;
}
fpThread = 0;
}
//------------------------------------------------------------------------------
void WECpiFeederThread::startFeederThread()
{
fStopped = false;
cout << "Starting Feeder Thread!!" << endl;
fStopped = false;
cout << "Starting Feeder Thread!!" << endl;
fpThread = new boost::thread(WECpiFeederRunner(*this));
}
//------------------------------------------------------------------------------
@ -92,14 +93,14 @@ void WECpiFeederThread::startFeederThread()
void WECpiFeederThread::add2MsgQueue(ByteStream& Ibs)
{
//TODO creating copy is NOT good; later read from socket using a SBS
messageqcpp::SBS aSbs(new messageqcpp::ByteStream(Ibs));
Ibs.reset(); //forcefully clearing it
mutex::scoped_lock aLock(fMsgQMutex);
//cout << "pushing to the MsgQueue" << endl;
fMsgQueue.push(aSbs);
fFeederCond.notify_one(); // as per preference of Damon
aLock.unlock();
//TODO creating copy is NOT good; later read from socket using a SBS
messageqcpp::SBS aSbs(new messageqcpp::ByteStream(Ibs));
Ibs.reset(); //forcefully clearing it
mutex::scoped_lock aLock(fMsgQMutex);
//cout << "pushing to the MsgQueue" << endl;
fMsgQueue.push(aSbs);
fFeederCond.notify_one(); // as per preference of Damon
aLock.unlock();
}
@ -107,41 +108,53 @@ void WECpiFeederThread::add2MsgQueue(ByteStream& Ibs)
void WECpiFeederThread::feedData2Cpi()
{
while(isContinue())
{
while (isContinue())
{
mutex::scoped_lock aLock(fMsgQMutex);
if(fMsgQueue.empty())
{
bool aTimedOut = fFeederCond.timed_wait(aLock, boost::posix_time::milliseconds(3000));
if(!isContinue()) { aLock.unlock(); break; }
// to handle spurious wake ups and timeout wake ups
if((fMsgQueue.empty())||(!aTimedOut)) { aLock.unlock(); continue; }
}
mutex::scoped_lock aLock(fMsgQMutex);
messageqcpp::SBS aSbs = fMsgQueue.front();
fMsgQueue.pop();
if (fMsgQueue.empty())
{
bool aTimedOut = fFeederCond.timed_wait(aLock, boost::posix_time::milliseconds(3000));
aLock.unlock();
if (!isContinue())
{
aLock.unlock();
break;
}
try
{
fOwner.pushData2Cpimport((*aSbs));
//cout << "Finished PUSHING data " << endl;
}
catch(runtime_error&)
{
//cout << "Caught exception : " << e.what() << endl;
//break;
}
// to handle spurious wake ups and timeout wake ups
if ((fMsgQueue.empty()) || (!aTimedOut))
{
aLock.unlock();
continue;
}
}
aSbs.reset(); //forcefully clearing it
// We start sending data request from here ONLY
if(getQueueSize() == WEDataLoader::MAX_QSIZE) fOwner.sendDataRequest();
}
messageqcpp::SBS aSbs = fMsgQueue.front();
fMsgQueue.pop();
cout << "CpiFeedThread Stopped!! " << endl;
fStopped = true;
aLock.unlock();
try
{
fOwner.pushData2Cpimport((*aSbs));
//cout << "Finished PUSHING data " << endl;
}
catch (runtime_error&)
{
//cout << "Caught exception : " << e.what() << endl;
//break;
}
aSbs.reset(); //forcefully clearing it
// We start sending data request from here ONLY
if (getQueueSize() == WEDataLoader::MAX_QSIZE) fOwner.sendDataRequest();
}
cout << "CpiFeedThread Stopped!! " << endl;
fStopped = true;
}
@ -149,30 +162,30 @@ void WECpiFeederThread::feedData2Cpi()
bool WECpiFeederThread::isMsgQueueEmpty()
{
bool aRet = false;
mutex::scoped_lock aLock(fMsgQMutex);
aRet = fMsgQueue.empty();
aLock.unlock();
return aRet;
bool aRet = false;
mutex::scoped_lock aLock(fMsgQMutex);
aRet = fMsgQueue.empty();
aLock.unlock();
return aRet;
}
//------------------------------------------------------------------------------
void WECpiFeederThread::stopThread()
{
mutex::scoped_lock aCondLock(fContMutex);
fContinue = false;
aCondLock.unlock();
fFeederCond.notify_all();
cout << "Notified all" << endl;
mutex::scoped_lock aCondLock(fContMutex);
fContinue = false;
aCondLock.unlock();
fFeederCond.notify_all();
cout << "Notified all" << endl;
}
//------------------------------------------------------------------------------
bool WECpiFeederThread::isContinue()
{
mutex::scoped_lock aCondLock(fContMutex);
return fContinue;
mutex::scoped_lock aCondLock(fContMutex);
return fContinue;
}
//------------------------------------------------------------------------------

154
writeengine/server/we_cpifeederthread.h
View File

@ -14,77 +14,83 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_cpifeederthread.h
*
* Created on: Mar 29, 2012
* Author: bpaul
*/
#ifndef WE_CPIFEEDERTHREAD_H_
#define WE_CPIFEEDERTHREAD_H_
#include <queue>
namespace WriteEngine
{
class WEDataLoader;
class WECpiFeederThread;
class WECpiFeederRunner
{
public:
WECpiFeederRunner(WECpiFeederThread& Ref): fOwner(Ref){ /* ctor */ }
virtual ~WECpiFeederRunner(){/* dtor */}
void operator()();
public:
WECpiFeederThread& fOwner;
};
class WECpiFeederThread
{
public:
WECpiFeederThread(WEDataLoader& Ref);
virtual ~WECpiFeederThread();
public:
void startFeederThread();
void add2MsgQueue(messageqcpp::ByteStream& Ibs);
void feedData2Cpi();
void stopThread();
bool isMsgQueueEmpty();
//bool isPushing() { return fPushing; }
bool isStopped() { return fStopped; }
int getQueueSize() { return fMsgQueue.size(); }
bool isContinue();
private:
WEDataLoader& fOwner;
boost::condition fFeederCond;
boost::mutex fMsgQMutex;
typedef std::queue<messageqcpp::SBS> WEMsgQueue;
WEMsgQueue fMsgQueue;
boost::thread *fpThread;
bool fContinue;
boost::mutex fContMutex;
//bool fPushing;
bool fStopped;
friend class WEDataLoader;
};
} /* namespace WriteEngine */
#endif /* WE_CPIFEEDERTHREAD_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_cpifeederthread.h
*
* Created on: Mar 29, 2012
* Author: bpaul
*/
#ifndef WE_CPIFEEDERTHREAD_H_
#define WE_CPIFEEDERTHREAD_H_
#include <queue>
namespace WriteEngine
{
class WEDataLoader;
class WECpiFeederThread;
class WECpiFeederRunner
{
public:
WECpiFeederRunner(WECpiFeederThread& Ref): fOwner(Ref) { /* ctor */ }
virtual ~WECpiFeederRunner() {/* dtor */}
void operator()();
public:
WECpiFeederThread& fOwner;
};
class WECpiFeederThread
{
public:
WECpiFeederThread(WEDataLoader& Ref);
virtual ~WECpiFeederThread();
public:
void startFeederThread();
void add2MsgQueue(messageqcpp::ByteStream& Ibs);
void feedData2Cpi();
void stopThread();
bool isMsgQueueEmpty();
//bool isPushing() { return fPushing; }
bool isStopped()
{
return fStopped;
}
int getQueueSize()
{
return fMsgQueue.size();
}
bool isContinue();
private:
WEDataLoader& fOwner;
boost::condition fFeederCond;
boost::mutex fMsgQMutex;
typedef std::queue<messageqcpp::SBS> WEMsgQueue;
WEMsgQueue fMsgQueue;
boost::thread* fpThread;
bool fContinue;
boost::mutex fContMutex;
//bool fPushing;
bool fStopped;
friend class WEDataLoader;
};
} /* namespace WriteEngine */
#endif /* WE_CPIFEEDERTHREAD_H_ */

2331
writeengine/server/we_dataloader.cpp
View File

File diff suppressed because it is too large Load Diff

183
writeengine/server/we_dataloader.h
View File

@ -43,7 +43,8 @@
#include "we_cpifeederthread.h"
namespace WriteEngine {
namespace WriteEngine
{
class SplitterReadThread;
@ -53,35 +54,35 @@ class SplitterReadThread;
class WEDataLoader : public Observer
{
public:
explicit WEDataLoader(SplitterReadThread& pSrt );
virtual ~WEDataLoader();
explicit WEDataLoader(SplitterReadThread& pSrt );
virtual ~WEDataLoader();
virtual bool update(Subject* pSub);
virtual bool update(Subject* pSub);
public:
bool setupCpimport(); // fork the cpimport
void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267
void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue
void closeWritePipe();
void str2Argv(std::string CmdLine, std::vector<char*>& V);
std::string getCalpontHome();
std::string getPrgmPath(std::string& PrgmName);
void updateCmdLineWithPath(string& CmdLine);
bool setupCpimport(); // fork the cpimport
void teardownCpimport(bool useStoredWaitPidStatus); // @bug 4267
void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue
void closeWritePipe();
void str2Argv(std::string CmdLine, std::vector<char*>& V);
std::string getCalpontHome();
std::string getPrgmPath(std::string& PrgmName);
void updateCmdLineWithPath(string& CmdLine);
public:
void onReceiveKeepAlive(ByteStream& Ibs);
void onReceiveData(ByteStream& Ibs);
void onReceiveEod(ByteStream& Ibs); //end of data
void onReceiveMode(ByteStream& Ibs);
//void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs);
void onReceiveCmd(ByteStream& bs);// {(ByteStream& Ibs);
void onReceiveAck(ByteStream& Ibs);
void onReceiveNak(ByteStream& Ibs);
void onReceiveError(ByteStream& Ibs);
void onReceiveJobId(ByteStream& Ibs);
void onReceiveJobData(ByteStream& Ibs);
void onReceiveKeepAlive(ByteStream& Ibs);
void onReceiveData(ByteStream& Ibs);
void onReceiveEod(ByteStream& Ibs); //end of data
void onReceiveMode(ByteStream& Ibs);
//void onReceiveCmd(messageqcpp::SBS bs);// {(ByteStream& Ibs);
void onReceiveCmd(ByteStream& bs);// {(ByteStream& Ibs);
void onReceiveAck(ByteStream& Ibs);
void onReceiveNak(ByteStream& Ibs);
void onReceiveError(ByteStream& Ibs);
void onReceiveJobId(ByteStream& Ibs);
void onReceiveJobData(ByteStream& Ibs);
void onReceiveImportFileName(ByteStream& Ibs);
void onReceiveCmdLineArgs(ByteStream& Ibs);
void onReceiveStartCpimport();
@ -98,8 +99,8 @@ public:
void sendDataRequest();
void sendCpimportFailureNotice();
void serialize(messageqcpp::ByteStream& b) const;
void unserialize(messageqcpp::ByteStream& b);
void serialize(messageqcpp::ByteStream& b) const;
void unserialize(messageqcpp::ByteStream& b);
// setup the signal handlers for the main app
@ -107,63 +108,111 @@ public:
static void onSigChild(int aInt);
public:
void setMode(int Mode){ fMode = Mode; }
void updateTxBytes(unsigned int Tx){ fTxBytes += Tx; }
void updateRxBytes(unsigned int Rx){ fRxBytes += Rx; }
void setChPid(pid_t pid){ fCh_pid = pid; }
void setPid(pid_t pid){ fThis_pid = pid; }
void setPPid(pid_t pid){ fP_pid = pid; }
void setCmdLineStr(std::string& Str){ fCmdLineStr = Str; }
void setObjId(int ObjId){ fObjId = ObjId; }
void setMode(int Mode)
{
fMode = Mode;
}
void updateTxBytes(unsigned int Tx)
{
fTxBytes += Tx;
}
void updateRxBytes(unsigned int Rx)
{
fRxBytes += Rx;
}
void setChPid(pid_t pid)
{
fCh_pid = pid;
}
void setPid(pid_t pid)
{
fThis_pid = pid;
}
void setPPid(pid_t pid)
{
fP_pid = pid;
}
void setCmdLineStr(std::string& Str)
{
fCmdLineStr = Str;
}
void setObjId(int ObjId)
{
fObjId = ObjId;
}
unsigned int getTxBytes(){ return fTxBytes; }
unsigned int getRxBytes(){ return fRxBytes; }
unsigned int getTxBytes()
{
return fTxBytes;
}
unsigned int getRxBytes()
{
return fRxBytes;
}
int getObjId(){ return fObjId; }
int getMode(){ return fMode; }
pid_t getChPid(){ return fCh_pid; }
pid_t getPid(){ return fThis_pid; }
pid_t getPPid(){ return fP_pid; }
std::string getCmdLineStr(){ return fCmdLineStr; }
int getObjId()
{
return fObjId;
}
int getMode()
{
return fMode;
}
pid_t getChPid()
{
return fCh_pid;
}
pid_t getPid()
{
return fThis_pid;
}
pid_t getPPid()
{
return fP_pid;
}
std::string getCmdLineStr()
{
return fCmdLineStr;
}
private:
SplitterReadThread& fRef;
SplitterReadThread& fRef;
int fMode;
ofstream fDataDumpFile;
ofstream fJobFile;
int fMode;
ofstream fDataDumpFile;
ofstream fJobFile;
unsigned int fTxBytes;
unsigned int fRxBytes;
char fPmId;
int fObjId; // Object Identifier for logging
char fPmId;
int fObjId; // Object Identifier for logging
// CpImport related Member variables
int fFIFO[2]; //I/O Pipes
pid_t fCh_pid;
pid_t fThis_pid;
pid_t fP_pid;
bool fCpIStarted;
std::string fCmdLineStr;
std::string fBrmRptFileName;
// CpImport related Member variables
int fFIFO[2]; //I/O Pipes
pid_t fCh_pid;
pid_t fThis_pid;
pid_t fP_pid;
bool fCpIStarted;
std::string fCmdLineStr;
std::string fBrmRptFileName;
//CPI Feeder Thread
WECpiFeederThread* fpCfThread;
//CPI Feeder Thread
WECpiFeederThread* fpCfThread;
boost::mutex fClntMsgMutex; //mutex in sending messages to client.
boost::mutex fClntMsgMutex; //mutex in sending messages to client.
//static bool fTearDownCpimport; // @bug 4267
bool fTearDownCpimport; // @bug 4267
pid_t fWaitPidRc; // @bug 4267
int fWaitPidStatus; // @bug 4267
pid_t fWaitPidRc; // @bug 4267
int fWaitPidStatus; // @bug 4267
bool fForceKill;
bool fPipeErr; // Err Flag to restrict err msgs logging.
bool fForceKill;
bool fPipeErr; // Err Flag to restrict err msgs logging.
private:
// more enums follow
enum CmdId { BULKFILENAME };
// more enums follow
enum CmdId { BULKFILENAME };
public:
enum { MIN_QSIZE=25, MAX_QSIZE=250};
enum { MIN_QSIZE = 25, MAX_QSIZE = 250};
public:
SimpleSysLog* fpSysLog;

8572
writeengine/server/we_ddlcommandproc.cpp
View File

File diff suppressed because it is too large Load Diff

140
writeengine/server/we_ddlcommandproc.h
View File

@ -42,77 +42,77 @@ namespace WriteEngine
class WE_DDLCommandProc
{
public:
enum LogFileType { DROPTABLE_LOG, DROPPART_LOG, TRUNCATE_LOG};
EXPORT WE_DDLCommandProc();
EXPORT WE_DDLCommandProc(const WE_DDLCommandProc& rhs);
EXPORT ~WE_DDLCommandProc();
/** @brief Update SYSCOLUMN nextval column for the columnoid with nextVal.
*
* Update SYSCOLUMN nextval column for the columnoid with nexValue.
* @param columnOid (in) The column OID
* @param nextVal (in) The partition number
* @return 0 on success, non-0 on error.
*/
EXPORT uint8_t updateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeSystable(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeCreateSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t createtablefiles(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t deleteSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t deleteSyscolumnRow(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t deleteSystable(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t deleteSystables(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t dropFiles(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSyscolumnAuto(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSyscolumnNextvalCol(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSyscolumnTablename(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSystableAuto(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSystableTablename(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSystablesTablename(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSyscolumnColumnposCol(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t fillNewColumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSyscolumnRenameColumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSyscolumnSetDefault(messageqcpp::ByteStream& bs, std::string & err);
// EXPORT uint8_t updateSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeTruncateLog(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeDropPartitionLog(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeDropTableLog(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t deleteDDLLog(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t fetchDDLLog(messageqcpp::ByteStream& bs, std::string & err);
void purgeFDCache();
/** @brief drop a set of partitions
*
* Do many VSS lookups under one lock.
* @param bs (in) bytestream carry command info
* @param err (out) error message when error occurs
* @return 0 on success, otherwise error.
*/
EXPORT uint8_t dropPartitions(messageqcpp::ByteStream& bs, std::string& err);
inline void convertRidToColumn (uint64_t& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, const int32_t oid )
{
fDbrm.getSysCatDBRoot(oid, dbRoot);
partition = rid / ( filesPerColumnPartition * extentsPerSegmentFile * extentRows );
segment =( ( ( rid % ( filesPerColumnPartition * extentsPerSegmentFile * extentRows )) / extentRows ) ) % filesPerColumnPartition;
//Calculate the relative rid for this segment file
uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
assert ( relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows );
uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows;
}
public:
enum LogFileType { DROPTABLE_LOG, DROPPART_LOG, TRUNCATE_LOG};
EXPORT WE_DDLCommandProc();
EXPORT WE_DDLCommandProc(const WE_DDLCommandProc& rhs);
EXPORT ~WE_DDLCommandProc();
/** @brief Update SYSCOLUMN nextval column for the columnoid with nextVal.
*
* Update SYSCOLUMN nextval column for the columnoid with nexValue.
* @param columnOid (in) The column OID
* @param nextVal (in) The partition number
* @return 0 on success, non-0 on error.
*/
EXPORT uint8_t updateSyscolumnNextval(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeSystable(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeCreateSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t createtablefiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSyscolumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSyscolumnRow(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSystable(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteSystables(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t dropFiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnAuto(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnNextvalCol(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystableAuto(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystableTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSystablesTablename(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnColumnposCol(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t fillNewColumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnRenameColumn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnSetDefault(messageqcpp::ByteStream& bs, std::string& err);
// EXPORT uint8_t updateSyscolumn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t writeTruncateLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeDropPartitionLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t writeDropTableLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t deleteDDLLog(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t fetchDDLLog(messageqcpp::ByteStream& bs, std::string& err);
void purgeFDCache();
/** @brief drop a set of partitions
*
* Do many VSS lookups under one lock.
* @param bs (in) bytestream carry command info
* @param err (out) error message when error occurs
* @return 0 on success, otherwise error.
*/
EXPORT uint8_t dropPartitions(messageqcpp::ByteStream& bs, std::string& err);
inline void convertRidToColumn (uint64_t& rid, uint16_t& dbRoot, uint32_t& partition, uint16_t& segment, const int32_t oid )
{
fDbrm.getSysCatDBRoot(oid, dbRoot);
partition = rid / ( filesPerColumnPartition * extentsPerSegmentFile * extentRows );
segment = ( ( ( rid % ( filesPerColumnPartition * extentsPerSegmentFile * extentRows )) / extentRows ) ) % filesPerColumnPartition;
//Calculate the relative rid for this segment file
uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
assert ( relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows );
uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
rid = relRidInThisExtent + numExtentsInThisSegPart * extentRows;
}
private:
WriteEngineWrapper fWEWrapper;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
private:
WriteEngineWrapper fWEWrapper;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
};
}
#undef EXPORT

1072
writeengine/server/we_ddlcommon.h
View File

File diff suppressed because it is too large Load Diff

7971
writeengine/server/we_dmlcommandproc.cpp
View File

File diff suppressed because it is too large Load Diff

154
writeengine/server/we_dmlcommandproc.h
View File

@ -52,86 +52,86 @@ namespace WriteEngine
class WE_DMLCommandProc
{
public:
typedef std::vector<std::string> ColValues;
EXPORT WE_DMLCommandProc();
EXPORT WE_DMLCommandProc(const WE_DMLCommandProc& rhs);
EXPORT ~WE_DMLCommandProc();
inline void isFirstBatchPm (bool firstBatch)
{
fIsFirstBatchPm = firstBatch;
}
inline bool isFirstBatchPm ()
{
return fIsFirstBatchPm;
}
//Convert rid from logical block rid to file relative rid
inline void convertToRelativeRid (uint64_t& rid, const uint8_t extentNum, const uint16_t blockNum)
{
rid = rid + extentNum*extentRows + blockNum * 8192;
}
EXPORT uint8_t processSingleInsert(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId);
EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId);
EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t processUpdate(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId, uint64_t& blocksChanged);
EXPORT uint8_t processUpdate1(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t processFlushFiles(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t processDelete(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId, uint64_t& blocksChanged);
EXPORT uint8_t processRemoveMeta(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t processBulkRollback(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string & err);
EXPORT uint8_t updateSyscolumnNextval(ByteStream& bs, std::string & err);
EXPORT uint8_t processPurgeFDCache(ByteStream& bs, std::string & err);
EXPORT uint8_t processEndTransaction(ByteStream& bs, std::string & err);
EXPORT uint8_t processFixRows(ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId);
EXPORT uint8_t getWrittenLbids(messageqcpp::ByteStream& bs, std::string & err, ByteStream::quadbyte & PMId);
int validateColumnHWMs(
execplan::CalpontSystemCatalog::RIDList& ridList,
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr,
const std::vector<DBRootExtentInfo>& segFileInfo,
const char* stage );
private:
WriteEngineWrapper fWEWrapper;
boost::scoped_ptr<RBMetaWriter> fRBMetaWriter;
std::vector<boost::shared_ptr<DBRootExtentTracker> > dbRootExtTrackerVec;
inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType colType )
{
if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18))
|| (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)
public:
typedef std::vector<std::string> ColValues;
EXPORT WE_DMLCommandProc();
EXPORT WE_DMLCommandProc(const WE_DMLCommandProc& rhs);
EXPORT ~WE_DMLCommandProc();
inline void isFirstBatchPm (bool firstBatch)
{
fIsFirstBatchPm = firstBatch;
}
inline bool isFirstBatchPm ()
{
return fIsFirstBatchPm;
}
//Convert rid from logical block rid to file relative rid
inline void convertToRelativeRid (uint64_t& rid, const uint8_t extentNum, const uint16_t blockNum)
{
rid = rid + extentNum * extentRows + blockNum * 8192;
}
EXPORT uint8_t processSingleInsert(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBlocks(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsertHwm(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processUpdate(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, uint64_t& blocksChanged);
EXPORT uint8_t processUpdate1(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processFlushFiles(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processDelete(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, uint64_t& blocksChanged);
EXPORT uint8_t processRemoveMeta(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBulkRollback(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBulkRollbackCleanup(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t updateSyscolumnNextval(ByteStream& bs, std::string& err);
EXPORT uint8_t processPurgeFDCache(ByteStream& bs, std::string& err);
EXPORT uint8_t processEndTransaction(ByteStream& bs, std::string& err);
EXPORT uint8_t processFixRows(ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
EXPORT uint8_t getWrittenLbids(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId);
int validateColumnHWMs(
execplan::CalpontSystemCatalog::RIDList& ridList,
boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr,
const std::vector<DBRootExtentInfo>& segFileInfo,
const char* stage );
private:
WriteEngineWrapper fWEWrapper;
boost::scoped_ptr<RBMetaWriter> fRBMetaWriter;
std::vector<boost::shared_ptr<DBRootExtentTracker> > dbRootExtTrackerVec;
inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType colType )
{
if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18))
|| ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18))
|| (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY)
|| (colType.colDataType == execplan::CalpontSystemCatalog::BLOB)
|| (colType.colDataType == execplan::CalpontSystemCatalog::TEXT))
{
return true;
}
else
return false;
}
uint8_t processBatchInsertHwmFlushChunks(uint32_t tableOID, int txnID,
const std::vector<BRM::FileInfo>& files,
const std::vector<BRM::OID_t>& oidsToFlush,
std::string& err);
{
return true;
}
else
return false;
}
uint8_t processBatchInsertHwmFlushChunks(uint32_t tableOID, int txnID,
const std::vector<BRM::FileInfo>& files,
const std::vector<BRM::OID_t>& oidsToFlush,
std::string& err);
bool fIsFirstBatchPm;
std::map<uint32_t,rowgroup::RowGroup *> rowGroups;
std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
Log fLog;
bool fIsFirstBatchPm;
std::map<uint32_t, rowgroup::RowGroup*> rowGroups;
std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages;
BRM::DBRM fDbrm;
unsigned extentsPerSegmentFile, extentRows, filesPerColumnPartition, dbrootCnt;
Log fLog;
};
}

576
writeengine/server/we_getfilesizes.cpp
View File

@ -44,206 +44,221 @@ using namespace idbdatafile;
namespace WriteEngine
{
struct FileInfo
struct FileInfo
{
uint32_t partition; /** @brief Partition for a file*/
uint16_t segment; /** @brief Segment for a file */
uint16_t dbRoot; /** @brief DbRoot for a file */
std::string segFileName; /** @brief seg file path */
double fileSize; /** @brief seg file size in giga bytes */
void serialize(messageqcpp::ByteStream& bs)
{
bs << partition;
bs << segment;
bs << dbRoot;
bs << segFileName;
bs << (*(uint64_t*)(&fileSize));
}
uint32_t partition; /** @brief Partition for a file*/
uint16_t segment; /** @brief Segment for a file */
uint16_t dbRoot; /** @brief DbRoot for a file */
std::string segFileName; /** @brief seg file path */
double fileSize; /** @brief seg file size in giga bytes */
void serialize(messageqcpp::ByteStream& bs)
{
bs << partition;
bs << segment;
bs << dbRoot;
bs << segFileName;
bs << (*(uint64_t*)(&fileSize));
}
};
typedef std::vector<FileInfo> Files;
typedef std::map<uint32_t, Files> columnMap;
typedef std::map<int, columnMap*> allColumnMap;
allColumnMap wholeMap;
boost::mutex columnMapLock;
ActiveThreadCounter *activeThreadCounter;
boost::mutex columnMapLock;
ActiveThreadCounter* activeThreadCounter;
size_t readFillBuffer(
idbdatafile::IDBDataFile* pFile,
char* buffer,
size_t bytesReq)
idbdatafile::IDBDataFile* pFile,
char* buffer,
size_t bytesReq)
{
char* pBuf = buffer;
ssize_t nBytes;
size_t bytesToRead = bytesReq;
size_t totalBytesRead = 0;
char* pBuf = buffer;
ssize_t nBytes;
size_t bytesToRead = bytesReq;
size_t totalBytesRead = 0;
while (1)
{
nBytes = pFile->read(pBuf, bytesToRead);
if (nBytes > 0)
totalBytesRead += nBytes;
else
break;
while (1)
{
nBytes = pFile->read(pBuf, bytesToRead);
if ((size_t)nBytes == bytesToRead)
break;
if (nBytes > 0)
totalBytesRead += nBytes;
else
break;
pBuf += nBytes;
bytesToRead = bytesToRead - (size_t)nBytes;
}
if ((size_t)nBytes == bytesToRead)
break;
return totalBytesRead;
pBuf += nBytes;
bytesToRead = bytesToRead - (size_t)nBytes;
}
return totalBytesRead;
}
off64_t getCompressedDataSize(string& fileName)
{
off64_t dataSize = 0;
IDBDataFile* pFile = 0;
size_t nBytes;
// Some IDBPolicy functions can throw exceptions, caller will catch it
IDBPolicy::configIDBPolicy();
bool bHdfsFile = IDBPolicy::useHdfs();
if (bHdfsFile)
pFile = IDBDataFile::open(IDBDataFile::HDFS,
fileName.c_str(), "r", 0);
else
pFile = IDBDataFile::open(IDBDataFile::BUFFERED,
fileName.c_str(), "r", 0);
if (!pFile)
{
std::ostringstream oss;
oss << "Cannot open file " << fileName << " for read.";
throw std::runtime_error(oss.str());
}
IDBCompressInterface decompressor;
//--------------------------------------------------------------------------
// Read headers and extract compression pointers
//--------------------------------------------------------------------------
char hdr1[IDBCompressInterface::HDR_BUF_LEN];
nBytes = readFillBuffer( pFile,hdr1,IDBCompressInterface::HDR_BUF_LEN);
if ( nBytes != IDBCompressInterface::HDR_BUF_LEN )
{
std::ostringstream oss;
oss << "Error reading first header from file " << fileName;
throw std::runtime_error(oss.str());
}
off64_t dataSize = 0;
IDBDataFile* pFile = 0;
size_t nBytes;
// Some IDBPolicy functions can throw exceptions, caller will catch it
IDBPolicy::configIDBPolicy();
bool bHdfsFile = IDBPolicy::useHdfs();
int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - IDBCompressInterface::HDR_BUF_LEN;
char* hdr2 = new char[ptrSecSize];
nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
if ( (int64_t)nBytes != ptrSecSize )
{
std::ostringstream oss;
oss << "Error reading second header from file " << fileName;
throw std::runtime_error(oss.str());
}
if (bHdfsFile)
pFile = IDBDataFile::open(IDBDataFile::HDFS,
fileName.c_str(), "r", 0);
else
pFile = IDBDataFile::open(IDBDataFile::BUFFERED,
fileName.c_str(), "r", 0);
CompChunkPtrList chunkPtrs;
int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
delete[] hdr2;
if (rc != 0)
{
std::ostringstream oss;
oss << "Error decompressing second header from file " << fileName;
throw std::runtime_error(oss.str());
}
if (!pFile)
{
std::ostringstream oss;
oss << "Cannot open file " << fileName << " for read.";
throw std::runtime_error(oss.str());
}
unsigned k = chunkPtrs.size();
// last header's offset + length will be the data bytes
dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
delete pFile;
return dataSize;
IDBCompressInterface decompressor;
//--------------------------------------------------------------------------
// Read headers and extract compression pointers
//--------------------------------------------------------------------------
char hdr1[IDBCompressInterface::HDR_BUF_LEN];
nBytes = readFillBuffer( pFile, hdr1, IDBCompressInterface::HDR_BUF_LEN);
if ( nBytes != IDBCompressInterface::HDR_BUF_LEN )
{
std::ostringstream oss;
oss << "Error reading first header from file " << fileName;
throw std::runtime_error(oss.str());
}
int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - IDBCompressInterface::HDR_BUF_LEN;
char* hdr2 = new char[ptrSecSize];
nBytes = readFillBuffer( pFile, hdr2, ptrSecSize);
if ( (int64_t)nBytes != ptrSecSize )
{
std::ostringstream oss;
oss << "Error reading second header from file " << fileName;
throw std::runtime_error(oss.str());
}
CompChunkPtrList chunkPtrs;
int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
delete[] hdr2;
if (rc != 0)
{
std::ostringstream oss;
oss << "Error decompressing second header from file " << fileName;
throw std::runtime_error(oss.str());
}
unsigned k = chunkPtrs.size();
// last header's offset + length will be the data bytes
dataSize = chunkPtrs[k - 1].first + chunkPtrs[k - 1].second;
delete pFile;
return dataSize;
}
struct ColumnThread
{
ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key)
: fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key)
ColumnThread(uint32_t oid, int32_t compressionType, bool reportRealUse, int key)
: fOid(oid), fCompressionType(compressionType), fReportRealUse(reportRealUse), fKey(key)
{}
void operator()()
{
Config config;
config.initConfigCache();
std::vector<uint16_t> rootList;
config.getRootIdList( rootList );
FileOp fileOp;
Files aFiles;
Config config;
config.initConfigCache();
std::vector<uint16_t> rootList;
config.getRootIdList( rootList );
FileOp fileOp;
Files aFiles;
// This function relies on IDBPolicy being initialized by
// IDBPolicy::init(). This is done when WriteEngineServer main() calls
// IDBPolicy::configIDBPolicy();
IDBDataFile::Types fileType;
bool bUsingHdfs = IDBPolicy::useHdfs();
if (bUsingHdfs)
fileType = IDBDataFile::HDFS;
else
fileType = IDBDataFile::UNBUFFERED;
IDBFileSystem& fs = IDBFileSystem::getFs( fileType );
for (uint32_t i=0; i < rootList.size(); i++)
{
std::vector<struct BRM::EMEntry> entries;
(void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]);
std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
while ( iter != entries.end() ) //organize extents into files
{
//Find the size of this file
//string fileName;
char fileName[200];
(void)fileOp.getFileName( fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum);
string aFile(fileName); //convert between char* and string
off64_t fileSize = 0;
if (fReportRealUse && (fCompressionType > 0))
{
try {
fileSize = getCompressedDataSize(aFile);
}
catch (std::exception& ex)
{
cerr << ex.what();
}
}
else
fileSize = fs.size( fileName );
if (fileSize > 0) // File exists, add to list
{
FileInfo aFileInfo;
aFileInfo.partition = entries[0].partitionNum;
aFileInfo.segment = entries[0].segmentNum;
aFileInfo.dbRoot = rootList[i];
aFileInfo.segFileName = aFile;
aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024);
aFiles.push_back(aFileInfo);
//cout.precision(15);
//cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize << "GB" << endl;
}
//erase the entries from this dbroot.
std::vector<struct BRM::EMEntry> entriesTrimed;
for (uint32_t m=0; m<entries.size(); m++)
{
if ((entries[0].partitionNum != entries[m].partitionNum) || (entries[0].segmentNum != entries[m].segmentNum))
entriesTrimed.push_back(entries[m]);
}
entriesTrimed.swap(entries);
iter = entries.begin();
}
}
boost::mutex::scoped_lock lk(columnMapLock);
//cout << "Current size of columnsMap is " << columnsMap.size() << endl;
allColumnMap::iterator colMapiter = wholeMap.find(fKey);
if (colMapiter != wholeMap.end())
{
(colMapiter->second)->insert(make_pair(fOid,aFiles));
activeThreadCounter->decr();
//cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl;
}
}
uint32_t fOid;
int32_t fCompressionType;
bool fReportRealUse;
int fKey;
// This function relies on IDBPolicy being initialized by
// IDBPolicy::init(). This is done when WriteEngineServer main() calls
// IDBPolicy::configIDBPolicy();
IDBDataFile::Types fileType;
bool bUsingHdfs = IDBPolicy::useHdfs();
if (bUsingHdfs)
fileType = IDBDataFile::HDFS;
else
fileType = IDBDataFile::UNBUFFERED;
IDBFileSystem& fs = IDBFileSystem::getFs( fileType );
for (uint32_t i = 0; i < rootList.size(); i++)
{
std::vector<struct BRM::EMEntry> entries;
(void)BRMWrapper::getInstance()->getExtents_dbroot(fOid, entries, rootList[i]);
std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
while ( iter != entries.end() ) //organize extents into files
{
//Find the size of this file
//string fileName;
char fileName[200];
(void)fileOp.getFileName( fOid, fileName, rootList[i], entries[0].partitionNum, entries[0].segmentNum);
string aFile(fileName); //convert between char* and string
off64_t fileSize = 0;
if (fReportRealUse && (fCompressionType > 0))
{
try
{
fileSize = getCompressedDataSize(aFile);
}
catch (std::exception& ex)
{
cerr << ex.what();
}
}
else
fileSize = fs.size( fileName );
if (fileSize > 0) // File exists, add to list
{
FileInfo aFileInfo;
aFileInfo.partition = entries[0].partitionNum;
aFileInfo.segment = entries[0].segmentNum;
aFileInfo.dbRoot = rootList[i];
aFileInfo.segFileName = aFile;
aFileInfo.fileSize = (double)fileSize / (1024 * 1024 * 1024);
aFiles.push_back(aFileInfo);
//cout.precision(15);
//cout << "The file " << aFileInfo.segFileName << " has size " << fixed << aFileInfo.fileSize << "GB" << endl;
}
//erase the entries from this dbroot.
std::vector<struct BRM::EMEntry> entriesTrimed;
for (uint32_t m = 0; m < entries.size(); m++)
{
if ((entries[0].partitionNum != entries[m].partitionNum) || (entries[0].segmentNum != entries[m].segmentNum))
entriesTrimed.push_back(entries[m]);
}
entriesTrimed.swap(entries);
iter = entries.begin();
}
}
boost::mutex::scoped_lock lk(columnMapLock);
//cout << "Current size of columnsMap is " << columnsMap.size() << endl;
allColumnMap::iterator colMapiter = wholeMap.find(fKey);
if (colMapiter != wholeMap.end())
{
(colMapiter->second)->insert(make_pair(fOid, aFiles));
activeThreadCounter->decr();
//cout << "Added to columnsMap aFiles with size " << aFiles.size() << " for oid " << fOid << endl;
}
}
uint32_t fOid;
int32_t fCompressionType;
bool fReportRealUse;
int fKey;
};
//------------------------------------------------------------------------------
@ -266,11 +281,12 @@ int WE_GetFileSizes::processFileName(
fileSize = IDBPolicy::size(fileName.c_str());
compressedFileSize = IDBPolicy::compressedSize(fileName.c_str());
}
catch(std::exception& ex)
{
errMsg = ex.what();
rc = 1;
}
catch (std::exception& ex)
{
errMsg = ex.what();
rc = 1;
}
bs.reset();
bs << fileSize;
bs << compressedFileSize;
@ -282,109 +298,123 @@ int WE_GetFileSizes::processFileName(
// bytestream object.
//------------------------------------------------------------------------------
int WE_GetFileSizes::processTable(
messageqcpp::ByteStream& bs,
std::string& errMsg, int key)
messageqcpp::ByteStream& bs,
std::string& errMsg, int key)
{
uint8_t rc = 0;
errMsg.clear();
uint8_t rc = 0;
errMsg.clear();
try {
std::string aTableName;
std::string schemaName;
bool reportRealUse = false;
ByteStream::byte tmp8;
bs >> schemaName;
//cout << "schema: "<< schemaName << endl;
try
{
std::string aTableName;
std::string schemaName;
bool reportRealUse = false;
ByteStream::byte tmp8;
bs >> schemaName;
//cout << "schema: "<< schemaName << endl;
bs >> aTableName;
//cout << "tableName: " << aTableName << endl;
bs >> tmp8;
reportRealUse = (tmp8 != 0);
//get column oids
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(0);
CalpontSystemCatalog::TableName tableName;
tableName.schema = schemaName;
tableName.table = aTableName;
CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName);
CalpontSystemCatalog::ColType colType;
CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName);
int serverThreads = 20;
int serverQueueSize = serverThreads * 100;
threadpool::ThreadPool tp(serverThreads,serverQueueSize);
int totalSize = columnList.size() + dictOidList.size();
activeThreadCounter = new ActiveThreadCounter(totalSize);
columnMap *columnsMap = new columnMap();
{
boost::mutex::scoped_lock lk(columnMapLock);
wholeMap[key] = columnsMap;
}
for (uint32_t i=0; i < columnList.size(); i++)
{
colType = systemCatalogPtr->colType(columnList[i].objnum);
tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key));
if (colType.ddn.dictOID > 0)
tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key));
}
/* for (uint32_t i=0; i < dictOidList.size(); i++)
{
tp.invoke(ColumnThread(dictOidList[i].dictOID));
} */
//check whether all threads finish
int sleepTime = 100; // sleep 100 milliseconds between checks
struct timespec rm_ts;
bs >> aTableName;
//cout << "tableName: " << aTableName << endl;
bs >> tmp8;
reportRealUse = (tmp8 != 0);
rm_ts.tv_sec = sleepTime/1000;
rm_ts.tv_nsec = sleepTime%1000 *1000000;
uint32_t currentActiveThreads = 10;
while (currentActiveThreads > 0)
{
//get column oids
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(0);
CalpontSystemCatalog::TableName tableName;
tableName.schema = schemaName;
tableName.table = aTableName;
CalpontSystemCatalog::RIDList columnList = systemCatalogPtr->columnRIDs(tableName);
CalpontSystemCatalog::ColType colType;
CalpontSystemCatalog::DictOIDList dictOidList = systemCatalogPtr->dictOIDs(tableName);
int serverThreads = 20;
int serverQueueSize = serverThreads * 100;
threadpool::ThreadPool tp(serverThreads, serverQueueSize);
int totalSize = columnList.size() + dictOidList.size();
activeThreadCounter = new ActiveThreadCounter(totalSize);
columnMap* columnsMap = new columnMap();
{
boost::mutex::scoped_lock lk(columnMapLock);
wholeMap[key] = columnsMap;
}
for (uint32_t i = 0; i < columnList.size(); i++)
{
colType = systemCatalogPtr->colType(columnList[i].objnum);
tp.invoke(ColumnThread(columnList[i].objnum, colType.compressionType, reportRealUse, key));
if (colType.ddn.dictOID > 0)
tp.invoke(ColumnThread(colType.ddn.dictOID, colType.compressionType, reportRealUse, key));
}
/* for (uint32_t i=0; i < dictOidList.size(); i++)
{
tp.invoke(ColumnThread(dictOidList[i].dictOID));
} */
//check whether all threads finish
int sleepTime = 100; // sleep 100 milliseconds between checks
struct timespec rm_ts;
rm_ts.tv_sec = sleepTime / 1000;
rm_ts.tv_nsec = sleepTime % 1000 * 1000000;
uint32_t currentActiveThreads = 10;
while (currentActiveThreads > 0)
{
#ifdef _MSC_VER
Sleep(sleepTime);
Sleep(sleepTime);
#else
struct timespec abs_ts;
do
{
abs_ts.tv_sec = rm_ts.tv_sec;
abs_ts.tv_nsec = rm_ts.tv_nsec;
}
while(nanosleep(&abs_ts,&rm_ts) < 0);
struct timespec abs_ts;
do
{
abs_ts.tv_sec = rm_ts.tv_sec;
abs_ts.tv_nsec = rm_ts.tv_nsec;
}
while (nanosleep(&abs_ts, &rm_ts) < 0);
#endif
currentActiveThreads = activeThreadCounter->cur();
}
}
catch(std::exception& ex)
{
//cout << "WE_GetFileSizes got exception-" << ex.what() <<
// std::endl;
errMsg = ex.what();
rc = 1;
}
//Build the message to send to the caller
bs.reset();
boost::mutex::scoped_lock lk(columnMapLock);
allColumnMap::iterator colMapiter = wholeMap.find(key);
if (colMapiter != wholeMap.end())
{
columnMap::iterator iter = colMapiter->second->begin();
uint64_t size;
Files::iterator it;
while ( iter != colMapiter->second->end())
{
bs << iter->first;
//cout << "processTable::coloid = " << iter->first << endl;
currentActiveThreads = activeThreadCounter->cur();
}
}
catch (std::exception& ex)
{
//cout << "WE_GetFileSizes got exception-" << ex.what() <<
// std::endl;
errMsg = ex.what();
rc = 1;
}
size = iter->second.size();
bs << size;
for (it = iter->second.begin(); it != iter->second.end(); it++)
it->serialize(bs);
//cout << "length now is " << bs.length() << endl;
iter++;
}
wholeMap.erase(colMapiter);
}
return rc;
//Build the message to send to the caller
bs.reset();
boost::mutex::scoped_lock lk(columnMapLock);
allColumnMap::iterator colMapiter = wholeMap.find(key);
if (colMapiter != wholeMap.end())
{
columnMap::iterator iter = colMapiter->second->begin();
uint64_t size;
Files::iterator it;
while ( iter != colMapiter->second->end())
{
bs << iter->first;
//cout << "processTable::coloid = " << iter->first << endl;
size = iter->second.size();
bs << size;
for (it = iter->second.begin(); it != iter->second.end(); it++)
it->serialize(bs);
//cout << "length now is " << bs.length() << endl;
iter++;
}
wholeMap.erase(colMapiter);
}
return rc;
}

56
writeengine/server/we_getfilesizes.h
View File

@ -29,48 +29,54 @@
#include "atomicops.h"
namespace WriteEngine {
namespace WriteEngine
{
/** @brief Get all file sizes for the given table
*/
class WE_GetFileSizes
{
public:
static int processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
static int processTable(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
static int processFileName(messageqcpp::ByteStream& bs, std::string& errMsg, int key);
};
class ActiveThreadCounter
{
public:
ActiveThreadCounter(int size) : factiveThreadCount(size){}
virtual ~ActiveThreadCounter() {}
ActiveThreadCounter(int size) : factiveThreadCount(size) {}
virtual ~ActiveThreadCounter() {}
void decr()
{
int atc;
for (;;) {
atomicops::atomicMb();
atc = factiveThreadCount;
if (atc <= 0) //hopefully atc will never be < 0!
return;
if (atomicops::atomicCAS(&factiveThreadCount, atc, (atc - 1)))
return;
atomicops::atomicYield();
}
}
void decr()
{
int atc;
uint32_t cur()
{
return factiveThreadCount;
}
for (;;)
{
atomicops::atomicMb();
atc = factiveThreadCount;
if (atc <= 0) //hopefully atc will never be < 0!
return;
if (atomicops::atomicCAS(&factiveThreadCount, atc, (atc - 1)))
return;
atomicops::atomicYield();
}
}
uint32_t cur()
{
return factiveThreadCount;
}
private:
ActiveThreadCounter(const ActiveThreadCounter& rhs);
ActiveThreadCounter& operator=(const ActiveThreadCounter& rhs);
ActiveThreadCounter(const ActiveThreadCounter& rhs);
ActiveThreadCounter& operator=(const ActiveThreadCounter& rhs);
volatile int32_t factiveThreadCount;
volatile int32_t factiveThreadCount;
};
}

158
writeengine/server/we_messages.h
View File

@ -27,89 +27,89 @@ namespace WriteEngine
enum ServerMessages
{
WE_SVR_LOOPBACK,
WE_SVR_DDL_KEEPALIVE,
WE_SVR_DML_KEEPALIVE,
WE_UPDATE_NEXTVAL,
WE_SVR_WRITE_SYSTABLE,
WE_SVR_WRITE_SYSCOLUMN,
WE_SVR_WRITE_CREATETABLEFILES,
WE_SVR_COMMIT_VERSION,
WE_SVR_ROLLBACK_BLOCKS,
WE_SVR_ROLLBACK_VERSION,
WE_SVR_DELETE_SYSTABLE,
WE_SVR_DELETE_SYSTABLES,
WE_SVR_DELETE_SYSCOLUMN,
WE_SVR_DELETE_SYSCOLUMN_ROW,
WE_SVR_WRITE_DROPFILES,
WE_SVR_UPDATE_SYSTABLE_AUTO,
WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL,
WE_SVR_UPDATE_SYSCOLUMN_TABLENAME,
WE_SVR_UPDATE_SYSCOLUMN_AUTO,
WE_SVR_LOOPBACK,
WE_SVR_DDL_KEEPALIVE,
WE_SVR_DML_KEEPALIVE,
WE_UPDATE_NEXTVAL,
WE_SVR_WRITE_SYSTABLE,
WE_SVR_WRITE_SYSCOLUMN,
WE_SVR_WRITE_CREATETABLEFILES,
WE_SVR_COMMIT_VERSION,
WE_SVR_ROLLBACK_BLOCKS,
WE_SVR_ROLLBACK_VERSION,
WE_SVR_DELETE_SYSTABLE,
WE_SVR_DELETE_SYSTABLES,
WE_SVR_DELETE_SYSCOLUMN,
WE_SVR_DELETE_SYSCOLUMN_ROW,
WE_SVR_WRITE_DROPFILES,
WE_SVR_UPDATE_SYSTABLE_AUTO,
WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL,
WE_SVR_UPDATE_SYSCOLUMN_TABLENAME,
WE_SVR_UPDATE_SYSCOLUMN_AUTO,
WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL,
WE_SVR_UPDATE_SYSCOLUMN_COLPOS,
WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN,
WE_SVR_UPDATE_SYSTABLE_TABLENAME,
WE_SVR_UPDATE_SYSTABLES_TABLENAME,
WE_SVR_DROP_PARTITIONS,
WE_SVR_SINGLE_INSERT,
WE_SVR_BATCH_KEEPALIVE,
WE_SVR_BATCH_INSERT,
WE_SVR_BATCH_INSERT_END,
WE_SVR_COMMIT_BATCH_AUTO_ON,
WE_SVR_ROLLBACK_BATCH_AUTO_ON,
WE_SVR_COMMIT_BATCH_AUTO_OFF,
WE_SVR_ROLLBACK_BATCH_AUTO_OFF,
WE_SVR_BATCH_AUTOON_REMOVE_META,
WE_SVR_UPDATE,
WE_SVR_FLUSH_FILES, //35
WE_SVR_DELETE,
WE_SVR_DML_BULKROLLBACK,
WE_SVR_DML_BULKROLLBACK_CLEANUP,
WE_SVR_FILL_COLUMN,
WE_SVR_WRITE_TRUNCATE,
WE_SVR_WRITE_DROPPARTITION,
WE_SVR_WRITE_DROPTABLE,
WE_SVR_DELETE_DDLLOG,
WE_SVR_FETCH_DDL_LOGS,
WE_SVR_REMOVE_TABLEDATA,
WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL,
WE_SVR_REDISTRIBUTE,
WE_SVR_CLOSE_CONNECTION,
WE_SVR_GET_FILESIZES,
WE_SVR_UPDATE_SYSCOLUMN_COLPOS,
WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN,
WE_SVR_UPDATE_SYSTABLE_TABLENAME,
WE_SVR_UPDATE_SYSTABLES_TABLENAME,
WE_SVR_DROP_PARTITIONS,
WE_SVR_SINGLE_INSERT,
WE_SVR_BATCH_KEEPALIVE,
WE_SVR_BATCH_INSERT,
WE_SVR_BATCH_INSERT_END,
WE_SVR_COMMIT_BATCH_AUTO_ON,
WE_SVR_ROLLBACK_BATCH_AUTO_ON,
WE_SVR_COMMIT_BATCH_AUTO_OFF,
WE_SVR_ROLLBACK_BATCH_AUTO_OFF,
WE_SVR_BATCH_AUTOON_REMOVE_META,
WE_SVR_UPDATE,
WE_SVR_FLUSH_FILES, //35
WE_SVR_DELETE,
WE_SVR_DML_BULKROLLBACK,
WE_SVR_DML_BULKROLLBACK_CLEANUP,
WE_SVR_FILL_COLUMN,
WE_SVR_WRITE_TRUNCATE,
WE_SVR_WRITE_DROPPARTITION,
WE_SVR_WRITE_DROPTABLE,
WE_SVR_DELETE_DDLLOG,
WE_SVR_FETCH_DDL_LOGS,
WE_SVR_REMOVE_TABLEDATA,
WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL,
WE_SVR_REDISTRIBUTE,
WE_SVR_CLOSE_CONNECTION,
WE_SVR_GET_FILESIZES,
WE_SVR_GET_FILESIZE,
WE_SVR_PURGEFD,
WE_END_TRANSACTION,
WE_SRV_FIX_ROWS,
WE_SVR_WRITE_CREATE_SYSCOLUMN,
WE_SVR_BATCH_INSERT_BINARY,
WE_SVR_PURGEFD,
WE_END_TRANSACTION,
WE_SRV_FIX_ROWS,
WE_SVR_WRITE_CREATE_SYSCOLUMN,
WE_SVR_BATCH_INSERT_BINARY,
WE_SVR_GET_WRITTEN_LBIDS,
WE_CLT_SRV_DATA=100,
WE_CLT_SRV_EOD,
WE_CLT_SRV_CMD,
WE_CLT_SRV_ACK,
WE_CLT_SRV_NAK,
WE_CLT_SRV_PM_ERROR,
WE_CLT_SRV_KEEPALIVE,
WE_CLT_SRV_IMPFILENAME,
WE_CLT_SRV_IMPFILEERROR,
WE_CLT_SRV_CMDLINEARGS,
WE_CLT_SRV_STARTCPI,
WE_CLT_SRV_CPIPASS,
WE_CLT_SRV_CPIFAIL,
WE_CLT_SRV_ROLLBACK,
WE_CLT_SRV_CLEANUP,
WE_CLT_SRV_DATARQST,
WE_CLT_SRV_MODE,
WE_CLT_SRV_DBRCNT,
WE_CLT_SRV_BRMRPT,
WE_CLT_SRV_JOBID,
WE_CLT_SRV_JOBDATA,
WE_CLT_SRV_ERRLOG,
WE_CLT_SRV_BADLOG,
WE_CLT_SRV_CLEAR_TABLE_LOCK,
WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP
WE_CLT_SRV_DATA = 100,
WE_CLT_SRV_EOD,
WE_CLT_SRV_CMD,
WE_CLT_SRV_ACK,
WE_CLT_SRV_NAK,
WE_CLT_SRV_PM_ERROR,
WE_CLT_SRV_KEEPALIVE,
WE_CLT_SRV_IMPFILENAME,
WE_CLT_SRV_IMPFILEERROR,
WE_CLT_SRV_CMDLINEARGS,
WE_CLT_SRV_STARTCPI,
WE_CLT_SRV_CPIPASS,
WE_CLT_SRV_CPIFAIL,
WE_CLT_SRV_ROLLBACK,
WE_CLT_SRV_CLEANUP,
WE_CLT_SRV_DATARQST,
WE_CLT_SRV_MODE,
WE_CLT_SRV_DBRCNT,
WE_CLT_SRV_BRMRPT,
WE_CLT_SRV_JOBID,
WE_CLT_SRV_JOBDATA,
WE_CLT_SRV_ERRLOG,
WE_CLT_SRV_BADLOG,
WE_CLT_SRV_CLEAR_TABLE_LOCK,
WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP
};

2
writeengine/server/we_msg1.cpp
View File

@ -33,7 +33,7 @@ namespace WriteEngine
ByteStream::byte doMsg1(ByteStream& bs, std::string err)
{
return 0;
return 0;
}
}

2
writeengine/server/we_msg2.cpp
View File

@ -33,7 +33,7 @@ namespace WriteEngine
ByteStream::byte doMsg2(ByteStream& bs, std::string err)
{
return 0;
return 0;
}
}

169
writeengine/server/we_observer.cpp
View File

@ -14,81 +14,94 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* Observer.cpp
*
* Created on: Oct 5, 2011
* Author: bpaul@calpont.com
*/
#include <boost/thread/mutex.hpp>
#include "we_observer.h"
namespace WriteEngine {
//-----------------------------------------------------------------------------
// ctor
Observer::Observer() {
}
//-----------------------------------------------------------------------------
//dtor
Observer::~Observer() {
//
}
//-----------------------------------------------------------------------------
//ctor
Subject::Subject(){
}
//-----------------------------------------------------------------------------
//dtor
Subject::~Subject(){
}
//-----------------------------------------------------------------------------
void Subject::attach(Observer* Obs) {
boost::mutex::scoped_lock aLstLock;
fObs.push_back(Obs);
}
//-----------------------------------------------------------------------------
void Subject::detach(Observer* Obs) {
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
while (aIt != fObs.end()) {
if ((*aIt) == Obs) {
fObs.erase(aIt);
break;
}
}
}
//-----------------------------------------------------------------------------
void Subject::notify() {
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
while (aIt != fObs.end()) {
(*aIt)->update(this);
}
}
//-----------------------------------------------------------------------------
}// namespace WriteEngine
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* Observer.cpp
*
* Created on: Oct 5, 2011
* Author: bpaul@calpont.com
*/
#include <boost/thread/mutex.hpp>
#include "we_observer.h"
namespace WriteEngine
{
//-----------------------------------------------------------------------------
// ctor
Observer::Observer()
{
}
//-----------------------------------------------------------------------------
//dtor
Observer::~Observer()
{
//
}
//-----------------------------------------------------------------------------
//ctor
Subject::Subject()
{
}
//-----------------------------------------------------------------------------
//dtor
Subject::~Subject()
{
}
//-----------------------------------------------------------------------------
void Subject::attach(Observer* Obs)
{
boost::mutex::scoped_lock aLstLock;
fObs.push_back(Obs);
}
//-----------------------------------------------------------------------------
void Subject::detach(Observer* Obs)
{
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
while (aIt != fObs.end())
{
if ((*aIt) == Obs)
{
fObs.erase(aIt);
break;
}
}
}
//-----------------------------------------------------------------------------
void Subject::notify()
{
boost::mutex::scoped_lock aLstLock;
Observers::iterator aIt = fObs.begin();
while (aIt != fObs.end())
{
(*aIt)->update(this);
}
}
//-----------------------------------------------------------------------------
}// namespace WriteEngine

106
writeengine/server/we_observer.h
View File

@ -14,55 +14,57 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* Observer.h
*
* Created on: Oct 5, 2011
* Author: bpaul@calpont.com
*/
#ifndef OBSERVER_H_
#define OBSERVER_H_
#include <list>
using namespace std;
namespace WriteEngine
{
class Subject; // forward deceleration
class Observer {
public:
virtual ~Observer();
virtual bool update(Subject* pSub)=0;
protected:
Observer();
};
class Subject {
public:
Subject();
virtual ~Subject();
virtual void attach(Observer* Obs);
virtual void detach(Observer* Obs);
virtual void notify();
private:
typedef std::list<Observer*> Observers;
Observers fObs;
};
}
#endif /* OBSERVER_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* Observer.h
*
* Created on: Oct 5, 2011
* Author: bpaul@calpont.com
*/
#ifndef OBSERVER_H_
#define OBSERVER_H_
#include <list>
using namespace std;
namespace WriteEngine
{
class Subject; // forward deceleration
class Observer
{
public:
virtual ~Observer();
virtual bool update(Subject* pSub) = 0;
protected:
Observer();
};
class Subject
{
public:
Subject();
virtual ~Subject();
virtual void attach(Observer* Obs);
virtual void detach(Observer* Obs);
virtual void notify();
private:
typedef std::list<Observer*> Observers;
Observers fObs;
};
}
#endif /* OBSERVER_H_ */

443
writeengine/server/we_readthread.cpp
View File

@ -44,12 +44,13 @@ using namespace WriteEngine;
namespace WriteEngine
{
ReadThread::ReadThread(const IOSocket& ios): fIos(ios)
ReadThread::ReadThread(const IOSocket& ios): fIos(ios)
{
}
ReadThread::~ReadThread(){
ReadThread::~ReadThread()
{
}
@ -87,8 +88,8 @@ void DmlReadThread::operator()()
//cout << "DmlReadThread created ..." << endl;
// queryStats.blocksChanged for delete/update
uint64_t blocksChanged = 0;
while (ibs.length()>0)
while (ibs.length() > 0)
{
try
{
@ -96,261 +97,312 @@ void DmlReadThread::operator()()
//do work here...
ibs >> msgId;
if (msgId != WE_SVR_CLOSE_CONNECTION)
ibs >> uniqueID;
if (msgId != WE_SVR_CLOSE_CONNECTION)
ibs >> uniqueID;
//cout << "DmlReadThread " << pthread_self () << " received message id " << (uint32_t)msgId << " and bytestream length " << ibs.length() << endl;
switch (msgId)
{
case WE_SVR_SINGLE_INSERT:
case WE_SVR_SINGLE_INSERT:
{
rc = fWeDMLprocessor->processSingleInsert(ibs, errMsg);
break;
}
case WE_SVR_COMMIT_VERSION:
case WE_SVR_COMMIT_VERSION:
{
rc = fWeDMLprocessor->commitVersion(ibs, errMsg);
break;
}
case WE_SVR_ROLLBACK_BLOCKS:
case WE_SVR_ROLLBACK_BLOCKS:
{
rc = fWeDMLprocessor->rollbackBlocks(ibs, errMsg);
break;
}
case WE_SVR_ROLLBACK_VERSION:
case WE_SVR_ROLLBACK_VERSION:
{
rc = fWeDMLprocessor->rollbackVersion(ibs, errMsg);
break;
}
case WE_SVR_COMMIT_BATCH_AUTO_ON:
case WE_SVR_COMMIT_BATCH_AUTO_ON:
{
rc = fWeDMLprocessor->commitBatchAutoOn(ibs, errMsg);
break;
}
case WE_SVR_ROLLBACK_BATCH_AUTO_ON:
case WE_SVR_ROLLBACK_BATCH_AUTO_ON:
{
rc = fWeDMLprocessor->rollbackBatchAutoOn(ibs, errMsg);
break;
}
case WE_SVR_COMMIT_BATCH_AUTO_OFF:
case WE_SVR_COMMIT_BATCH_AUTO_OFF:
{
rc = fWeDMLprocessor->commitBatchAutoOn(ibs, errMsg);
break;
}
case WE_SVR_ROLLBACK_BATCH_AUTO_OFF:
case WE_SVR_ROLLBACK_BATCH_AUTO_OFF:
{
rc = fWeDMLprocessor->rollbackBatchAutoOff(ibs, errMsg);
break;
}
case WE_SVR_BATCH_INSERT:
case WE_SVR_BATCH_INSERT:
{
//timer.start("processBatchInsert");
//timer.start("processBatchInsert");
rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId);
//timer.stop("processBatchInsert");
//cout << "fWeDMLprocessor " << fWeDMLprocessor << " is processing batchinsert ..." << endl;
//timer.stop("processBatchInsert");
//cout << "fWeDMLprocessor " << fWeDMLprocessor << " is processing batchinsert ..." << endl;
break;
}
case WE_SVR_BATCH_INSERT_BINARY:
{
rc = fWeDMLprocessor->processBatchInsertBinary(ibs, errMsg, PMId);
break;
}
case WE_SVR_GET_WRITTEN_LBIDS:
{
rc = fWeDMLprocessor->getWrittenLbids(ibs, errMsg, PMId);
break;
}
case WE_SVR_BATCH_INSERT_END:
case WE_SVR_BATCH_INSERT_BINARY:
{
rc = fWeDMLprocessor->processBatchInsertBinary(ibs, errMsg, PMId);
break;
}
case WE_SVR_GET_WRITTEN_LBIDS:
{
rc = fWeDMLprocessor->getWrittenLbids(ibs, errMsg, PMId);
break;
}
case WE_SVR_BATCH_INSERT_END:
{
rc = fWeDMLprocessor->processBatchInsertHwm(ibs, errMsg);
//timer.finish();
//timer.finish();
break;
}
case WE_SVR_UPDATE:
case WE_SVR_UPDATE:
{
rc = fWeDMLprocessor->processUpdate(ibs, errMsg, PMId, blocksChanged);
break;
}
case WE_SVR_FLUSH_FILES:
case WE_SVR_FLUSH_FILES:
{
rc = fWeDMLprocessor->processFlushFiles(ibs, errMsg);
break;
break;
}
case WE_SVR_DELETE:
case WE_SVR_DELETE:
{
rc = fWeDMLprocessor->processDelete(ibs, errMsg, PMId, blocksChanged);
break;
}
case WE_SVR_BATCH_AUTOON_REMOVE_META:
case WE_SVR_BATCH_AUTOON_REMOVE_META:
{
rc = fWeDMLprocessor->processRemoveMeta(ibs, errMsg);
break;
}
case WE_SVR_DML_BULKROLLBACK:
case WE_SVR_DML_BULKROLLBACK:
{
rc = fWeDMLprocessor->processBulkRollback(ibs, errMsg);
break;
}
case WE_SVR_DML_BULKROLLBACK_CLEANUP:
case WE_SVR_DML_BULKROLLBACK_CLEANUP:
{
rc = fWeDMLprocessor->processBulkRollbackCleanup(ibs, errMsg);
break;
}
case WE_UPDATE_NEXTVAL:
case WE_UPDATE_NEXTVAL:
{
rc = fWeDMLprocessor->updateSyscolumnNextval(ibs, errMsg);
break;
}
case WE_SVR_WRITE_SYSTABLE:
case WE_SVR_WRITE_SYSTABLE:
{
rc = fWeDDLprocessor->writeSystable(ibs, errMsg);
break;
}
case WE_SVR_WRITE_SYSCOLUMN:
case WE_SVR_WRITE_SYSCOLUMN:
{
rc = fWeDDLprocessor->writeSyscolumn(ibs, errMsg);
break;
}
case WE_SVR_WRITE_CREATE_SYSCOLUMN:
case WE_SVR_WRITE_CREATE_SYSCOLUMN:
{
rc = fWeDDLprocessor->writeCreateSyscolumn(ibs, errMsg);
break;
}
case WE_SVR_WRITE_CREATETABLEFILES:
case WE_SVR_WRITE_CREATETABLEFILES:
{
rc = fWeDDLprocessor->createtablefiles(ibs, errMsg);
break;
}
case WE_SVR_DELETE_SYSCOLUMN:
case WE_SVR_DELETE_SYSCOLUMN:
{
rc = fWeDDLprocessor->deleteSyscolumn(ibs, errMsg);
break;
}
case WE_SVR_DELETE_SYSCOLUMN_ROW:
case WE_SVR_DELETE_SYSCOLUMN_ROW:
{
rc = fWeDDLprocessor->deleteSyscolumnRow(ibs, errMsg);
break;
}
case WE_SVR_DELETE_SYSTABLE:
case WE_SVR_DELETE_SYSTABLE:
{
rc = fWeDDLprocessor->deleteSystable(ibs, errMsg);
break;
}
case WE_SVR_DELETE_SYSTABLES:
case WE_SVR_DELETE_SYSTABLES:
{
rc = fWeDDLprocessor->deleteSystables(ibs, errMsg);
break;
}
case WE_SVR_WRITE_DROPFILES:
case WE_SVR_WRITE_DROPFILES:
{
rc = fWeDDLprocessor->dropFiles(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSCOLUMN_AUTO:
case WE_SVR_UPDATE_SYSCOLUMN_AUTO:
{
rc = fWeDDLprocessor->updateSyscolumnAuto(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL:
case WE_SVR_UPDATE_SYSCOLUMN_NEXTVAL:
{
rc = fWeDDLprocessor->updateSyscolumnNextvalCol(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL:
case WE_SVR_UPDATE_SYSCOLUMN_AUTOVAL:
{
rc = fWeDDLprocessor->updateSyscolumnNextval(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL:
case WE_SVR_UPDATE_SYSCOLUMN_DEFAULTVAL:
{
rc = fWeDDLprocessor->updateSyscolumnSetDefault(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSCOLUMN_TABLENAME:
case WE_SVR_UPDATE_SYSCOLUMN_TABLENAME:
{
rc = fWeDDLprocessor->updateSyscolumnTablename(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN:
case WE_SVR_UPDATE_SYSCOLUMN_RENAMECOLUMN:
{
rc = fWeDDLprocessor->updateSyscolumnRenameColumn(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSCOLUMN_COLPOS:
case WE_SVR_UPDATE_SYSCOLUMN_COLPOS:
{
rc = fWeDDLprocessor->updateSyscolumnColumnposCol(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSTABLE_AUTO:
case WE_SVR_UPDATE_SYSTABLE_AUTO:
{
rc = fWeDDLprocessor->updateSystableAuto(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSTABLE_TABLENAME:
case WE_SVR_UPDATE_SYSTABLE_TABLENAME:
{
rc = fWeDDLprocessor->updateSystableTablename(ibs, errMsg);
break;
}
case WE_SVR_UPDATE_SYSTABLES_TABLENAME:
case WE_SVR_UPDATE_SYSTABLES_TABLENAME:
{
rc = fWeDDLprocessor->updateSystablesTablename(ibs, errMsg);
break;
}
case WE_SVR_FILL_COLUMN:
case WE_SVR_FILL_COLUMN:
{
rc = fWeDDLprocessor->fillNewColumn(ibs, errMsg);
break;
}
case WE_SVR_DROP_PARTITIONS:
case WE_SVR_DROP_PARTITIONS:
{
rc = fWeDDLprocessor->dropPartitions(ibs, errMsg);
break;
}
case WE_SVR_WRITE_TRUNCATE:
case WE_SVR_WRITE_TRUNCATE:
{
rc = fWeDDLprocessor->writeTruncateLog(ibs, errMsg);
break;
}
case WE_SVR_WRITE_DROPPARTITION:
case WE_SVR_WRITE_DROPPARTITION:
{
rc = fWeDDLprocessor->writeDropPartitionLog(ibs, errMsg);
break;
}
case WE_SVR_WRITE_DROPTABLE:
case WE_SVR_WRITE_DROPTABLE:
{
rc = fWeDDLprocessor->writeDropTableLog(ibs, errMsg);
break;
}
case WE_SVR_DELETE_DDLLOG:
case WE_SVR_DELETE_DDLLOG:
{
rc = fWeDDLprocessor->deleteDDLLog(ibs, errMsg);
break;
}
case WE_SVR_FETCH_DDL_LOGS:
case WE_SVR_FETCH_DDL_LOGS:
{
rc = fWeDDLprocessor->fetchDDLLog(ibs, errMsg);
break;
}
case WE_SVR_PURGEFD:
{
rc = fWeDMLprocessor->processPurgeFDCache(ibs, errMsg);
break;
}
case WE_END_TRANSACTION:
{
rc = fWeDMLprocessor->processEndTransaction(ibs, errMsg);
break;
}
case WE_SRV_FIX_ROWS:
{
rc = fWeDMLprocessor->processFixRows(ibs, errMsg, PMId);
break;
}
case WE_SVR_CLOSE_CONNECTION:
{
break;
}
default:
break;
case WE_SVR_PURGEFD:
{
rc = fWeDMLprocessor->processPurgeFDCache(ibs, errMsg);
break;
}
case WE_END_TRANSACTION:
{
rc = fWeDMLprocessor->processEndTransaction(ibs, errMsg);
break;
}
case WE_SRV_FIX_ROWS:
{
rc = fWeDMLprocessor->processFixRows(ibs, errMsg, PMId);
break;
}
case WE_SVR_CLOSE_CONNECTION:
{
break;
}
default:
break;
}
}
catch (std::exception& ex)
@ -377,18 +429,18 @@ void DmlReadThread::operator()()
logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
rc = 1;
errMsg = msg.msg();
}
if (msgId != WE_SVR_CLOSE_CONNECTION)
{
//send response
obs.restart();
obs << uniqueID;
obs << rc;
obs << errMsg;
}
}
if ((msgId == WE_SVR_COMMIT_BATCH_AUTO_ON) || (msgId ==WE_SVR_BATCH_INSERT_END) || (msgId == WE_SVR_FETCH_DDL_LOGS) || (msgId == WE_SVR_GET_WRITTEN_LBIDS))
if (msgId != WE_SVR_CLOSE_CONNECTION)
{
//send response
obs.restart();
obs << uniqueID;
obs << rc;
obs << errMsg;
}
if ((msgId == WE_SVR_COMMIT_BATCH_AUTO_ON) || (msgId == WE_SVR_BATCH_INSERT_END) || (msgId == WE_SVR_FETCH_DDL_LOGS) || (msgId == WE_SVR_GET_WRITTEN_LBIDS))
{
obs += ibs;
//cout << " sending back hwm info with ibs length " << endl;
@ -405,29 +457,32 @@ void DmlReadThread::operator()()
if (msgId == WE_SVR_UPDATE || msgId == WE_SVR_DELETE)
obs << blocksChanged; // send stats back to DMLProc
blocksChanged = 0; // reset
if (msgId == WE_SVR_CLOSE_CONNECTION)
{
//cout << "received request. closing connection ..." << endl;
break;
}
else
{
try
{
fIos.write(obs);
//cout << "dmlthread sent back response for msgid " << (uint32_t)msgId << " with uniqueID:rc= "
//<< (uint32_t)uniqueID<<":"<< (uint32_t)rc<<" and error message is " << errMsg <<endl;
//get next message
ibs = fIos.read();
}
catch (...)
{
break;
}
}
}
//cout << "closing connection for thread " << pthread_self () << endl;
if (msgId == WE_SVR_CLOSE_CONNECTION)
{
//cout << "received request. closing connection ..." << endl;
break;
}
else
{
try
{
fIos.write(obs);
//cout << "dmlthread sent back response for msgid " << (uint32_t)msgId << " with uniqueID:rc= "
//<< (uint32_t)uniqueID<<":"<< (uint32_t)rc<<" and error message is " << errMsg <<endl;
//get next message
ibs = fIos.read();
}
catch (...)
{
break;
}
}
}
//cout << "closing connection for thread " << pthread_self () << endl;
fIos.close();
}
@ -435,15 +490,15 @@ void DmlReadThread::operator()()
//-----------------------------------------------------------------------------
//ctor
SplitterReadThread::SplitterReadThread(const messageqcpp::IOSocket& ios,
ByteStream& Ibs):ReadThread(ios), fWeDataLoader(*this)
ByteStream& Ibs): ReadThread(ios), fWeDataLoader(*this)
{
fIbs = Ibs;
}
//-----------------------------------------------------------------------------
//copy ctor
SplitterReadThread::SplitterReadThread(const SplitterReadThread& rhs):
ReadThread(rhs.fIos), fWeDataLoader(*this)
SplitterReadThread::SplitterReadThread(const SplitterReadThread& rhs):
ReadThread(rhs.fIos), fWeDataLoader(*this)
{
fIbs = rhs.fIbs;
@ -461,7 +516,8 @@ SplitterReadThread::~SplitterReadThread()
void SplitterReadThread::operator()()
{
ByteStream::byte msgId;
while (fIbs.length()>0)
while (fIbs.length() > 0)
{
fWeDataLoader.updateRxBytes(fIbs.length());
@ -472,104 +528,124 @@ void SplitterReadThread::operator()()
switch (msgId)
{
case WE_CLT_SRV_KEEPALIVE:
case WE_CLT_SRV_KEEPALIVE:
{
fWeDataLoader.onReceiveKeepAlive(fIbs);
break;
}
case WE_CLT_SRV_DATA:
case WE_CLT_SRV_DATA:
{
fWeDataLoader.onReceiveData(fIbs);
break;
}
case WE_CLT_SRV_EOD:
case WE_CLT_SRV_EOD:
{
fWeDataLoader.onReceiveEod(fIbs);
break;
}
case WE_CLT_SRV_MODE:
case WE_CLT_SRV_MODE:
{
fWeDataLoader.onReceiveMode(fIbs);
break;
}
case WE_CLT_SRV_IMPFILENAME:
case WE_CLT_SRV_IMPFILENAME:
{
fWeDataLoader.onReceiveImportFileName(fIbs);
break;
}
case WE_CLT_SRV_CMDLINEARGS:
case WE_CLT_SRV_CMDLINEARGS:
{
fWeDataLoader.onReceiveCmdLineArgs(fIbs);
break;
}
case WE_CLT_SRV_CMD:
case WE_CLT_SRV_CMD:
{
fWeDataLoader.onReceiveCmd(fIbs); //fig out share_ptr on BS& is better
break;
}
case WE_CLT_SRV_ACK:
case WE_CLT_SRV_ACK:
{
fWeDataLoader.onReceiveAck(fIbs);
break;
}
case WE_CLT_SRV_NAK:
case WE_CLT_SRV_NAK:
{
fWeDataLoader.onReceiveNak(fIbs);
break;
}
case WE_CLT_SRV_PM_ERROR:
case WE_CLT_SRV_PM_ERROR:
{
fWeDataLoader.onReceiveError(fIbs);
break;
}
case WE_CLT_SRV_STARTCPI:
case WE_CLT_SRV_STARTCPI:
{
fWeDataLoader.onReceiveStartCpimport();
break;
}
case WE_CLT_SRV_BRMRPT:
case WE_CLT_SRV_BRMRPT:
{
fWeDataLoader.onReceiveBrmRptFileName(fIbs);
break;
}
case WE_CLT_SRV_CLEANUP:
case WE_CLT_SRV_CLEANUP:
{
fWeDataLoader.onReceiveCleanup(fIbs);
break;
}
case WE_CLT_SRV_ROLLBACK:
case WE_CLT_SRV_ROLLBACK:
{
fWeDataLoader.onReceiveRollback(fIbs);
break;
}
case WE_CLT_SRV_JOBID:
case WE_CLT_SRV_JOBID:
{
fWeDataLoader.onReceiveJobId(fIbs);
break;
}
case WE_CLT_SRV_JOBDATA:
case WE_CLT_SRV_JOBDATA:
{
fWeDataLoader.onReceiveJobData(fIbs);
break;
}
case WE_CLT_SRV_ERRLOG:
case WE_CLT_SRV_ERRLOG:
{
fWeDataLoader.onReceiveErrFileRqst(fIbs);
break;
}
case WE_CLT_SRV_BADLOG:
case WE_CLT_SRV_BADLOG:
{
fWeDataLoader.onReceiveBadFileRqst(fIbs);
break;
}
default:
break;
default:
break;
}
fIbs.restart();
try
{ //get next message
{
//get next message
fIbs = fIos.read();
}
catch (...)
@ -595,9 +671,9 @@ void SplitterReadThread::operator()()
// ClearTableLockReadThread constructor.
//------------------------------------------------------------------------------
ClearTableLockReadThread::ClearTableLockReadThread(
const messageqcpp::IOSocket& ios,
ByteStream& Ibs ): ReadThread(ios),
fWeClearTableLockCmd(new WE_ClearTableLockCmd("ClearTableLockTool"))
const messageqcpp::IOSocket& ios,
ByteStream& Ibs ): ReadThread(ios),
fWeClearTableLockCmd(new WE_ClearTableLockCmd("ClearTableLockTool"))
{
fIbs = Ibs;
}
@ -624,19 +700,22 @@ void ClearTableLockReadThread::operator()()
while (fIbs.length() > 0)
{
fIbs >> msgId;
switch (msgId)
{
case WE_CLT_SRV_CLEAR_TABLE_LOCK:
case WE_CLT_SRV_CLEAR_TABLE_LOCK:
{
rc = fWeClearTableLockCmd->processRollback(fIbs, errMsg);
break;
}
case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP:
case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP:
{
rc = fWeClearTableLockCmd->processCleanup(fIbs, errMsg);
break;
}
default:
default:
{
break;
}
@ -675,7 +754,7 @@ void ClearTableLockReadThread::operator()()
// RedistributeReadThread constructor.
//------------------------------------------------------------------------------
RedistributeReadThread::RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs)
: ReadThread(ios)
: ReadThread(ios)
{
fIbs = Ibs;
}
@ -714,7 +793,7 @@ void RedistributeReadThread::operator()()
// GetFileSizeThread constructor.
//------------------------------------------------------------------------------
GetFileSizeThread::GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs, BRM::DBRM& dbrm)
: ReadThread(ios), fWeGetFileSizes(new WE_GetFileSizes())
: ReadThread(ios), fWeGetFileSizes(new WE_GetFileSizes())
{
fIbs = Ibs;
key = dbrm.getUnique32();
@ -741,19 +820,22 @@ void GetFileSizeThread::operator()()
while (fIbs.length() > 0)
{
fIbs >> msgId;
switch (msgId)
{
case WE_SVR_GET_FILESIZES:
case WE_SVR_GET_FILESIZES:
{
rc = fWeGetFileSizes->processTable(fIbs, errMsg, key);
break;
}
case WE_SVR_GET_FILESIZE:
case WE_SVR_GET_FILESIZE:
{
rc = fWeGetFileSizes->processFileName(fIbs, errMsg, key);
break;
}
default:
default:
{
break;
}
@ -762,7 +844,7 @@ void GetFileSizeThread::operator()()
// Send response
obs.restart();
obs << rc;
obs << errMsg;
obs << errMsg;
obs += fIbs;
try
@ -794,19 +876,20 @@ void ReadThreadFactory::CreateReadThread(ThreadPool& Tp, IOSocket& Ios, BRM::DBR
{
struct timespec rm_ts;
int sleepTime = 20000; // wait for 20 seconds
rm_ts.tv_sec = sleepTime/1000;
rm_ts.tv_nsec = sleepTime%1000 *1000000;
rm_ts.tv_sec = sleepTime / 1000;
rm_ts.tv_nsec = sleepTime % 1000 * 1000000;
bool isTimeOut = false;
ByteStream::byte msgId;
ByteStream aBs;
try
{
aBs = Ios.read(&rm_ts, &isTimeOut);
}
catch (std::exception& ex)
{
cout<< "Handled : " << ex.what() <<endl;
cout << "Handled : " << ex.what() << endl;
logging::LoggingID logid(19, 0, 0);
logging::Message::Args args;
logging::Message msg(1);
@ -816,7 +899,8 @@ void ReadThreadFactory::CreateReadThread(ThreadPool& Tp, IOSocket& Ios, BRM::DBR
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_ERROR, msg, logid);
}
if ((aBs.length()<=0)||(isTimeOut))
if ((aBs.length() <= 0) || (isTimeOut))
{
Ios.close();
return;
@ -826,52 +910,55 @@ void ReadThreadFactory::CreateReadThread(ThreadPool& Tp, IOSocket& Ios, BRM::DBR
switch (msgId)
{
case WE_SVR_DDL_KEEPALIVE:
case WE_SVR_DML_KEEPALIVE:
case WE_SVR_DDL_KEEPALIVE:
case WE_SVR_DML_KEEPALIVE:
{
DmlReadThread dmlReadThread(Ios, aBs);
boost::thread t(dmlReadThread);
//cout << "starting DML thread id " << t.get_id() << endl;
boost::thread t(dmlReadThread);
//cout << "starting DML thread id " << t.get_id() << endl;
}
break;
case WE_CLT_SRV_KEEPALIVE:
case WE_CLT_SRV_MODE:
case WE_CLT_SRV_DATA:
case WE_CLT_SRV_CMD:
case WE_CLT_SRV_ACK:
case WE_CLT_SRV_NAK:
case WE_CLT_SRV_PM_ERROR:
case WE_CLT_SRV_CMDLINEARGS:
case WE_CLT_SRV_KEEPALIVE:
case WE_CLT_SRV_MODE:
case WE_CLT_SRV_DATA:
case WE_CLT_SRV_CMD:
case WE_CLT_SRV_ACK:
case WE_CLT_SRV_NAK:
case WE_CLT_SRV_PM_ERROR:
case WE_CLT_SRV_CMDLINEARGS:
{
//SplitterReadThread aSpReadThread(Ios, aBs);
//fOwner.attach(reinterpret_cast<Observer*>(&(aSpReadThread.fWeDataLoader)));
//Tp.invoke(aSpReadThread);
Tp.invoke(SplitterReadThread(Ios,aBs));
Tp.invoke(SplitterReadThread(Ios, aBs));
}
break;
case WE_CLT_SRV_CLEAR_TABLE_LOCK:
case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP:
case WE_CLT_SRV_CLEAR_TABLE_LOCK:
case WE_CLT_SRV_CLEAR_TABLE_LOCK_CLEANUP:
{
ClearTableLockReadThread clearTableLockThread(Ios, aBs);
Tp.invoke( clearTableLockThread );
}
break;
case WE_SVR_REDISTRIBUTE:
case WE_SVR_REDISTRIBUTE:
{
RedistributeReadThread RedistributeReadThread(Ios, aBs);
Tp.invoke(RedistributeReadThread);
}
break;
case WE_SVR_GET_FILESIZES:
case WE_SVR_GET_FILESIZE:
case WE_SVR_GET_FILESIZES:
case WE_SVR_GET_FILESIZE:
{
GetFileSizeThread getFileSizeThread(Ios, aBs, dbrm);
Tp.invoke(getFileSizeThread);
}
break;
default:
default:
{
Ios.close(); // don't know who is this
}

64
writeengine/server/we_readthread.h
View File

@ -43,21 +43,21 @@ namespace WriteEngine
class ReadThread
{
public:
explicit ReadThread(const messageqcpp::IOSocket& ios);
virtual ~ReadThread();
explicit ReadThread(const messageqcpp::IOSocket& ios);
virtual ~ReadThread();
virtual void operator()();
virtual void operator()();
//protected:
public:
messageqcpp::IOSocket fIos;
messageqcpp::IOSocket fIos;
messageqcpp::ByteStream fIbs;
private:
//defaults okay
//ReadThread(const ReadThread& rhs);
//ReadThread& operator=(const ReadThread& rhs);
//defaults okay
//ReadThread(const ReadThread& rhs);
//ReadThread& operator=(const ReadThread& rhs);
};
@ -65,33 +65,33 @@ private:
class DmlReadThread: public ReadThread
{
public:
explicit DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs );
virtual ~DmlReadThread();
explicit DmlReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs );
virtual ~DmlReadThread();
virtual void operator()();
virtual void operator()();
private:
boost::shared_ptr<WE_DMLCommandProc> fWeDMLprocessor;
boost::shared_ptr<WE_DDLCommandProc> fWeDDLprocessor;
boost::shared_ptr<WE_DMLCommandProc> fWeDMLprocessor;
boost::shared_ptr<WE_DDLCommandProc> fWeDDLprocessor;
};
class SplitterReadThread: public ReadThread
{
public:
SplitterReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs);
SplitterReadThread(const messageqcpp::IOSocket& ios, ByteStream& Ibs);
SplitterReadThread(const SplitterReadThread& rhs);
virtual ~SplitterReadThread();
virtual ~SplitterReadThread();
virtual void operator()();
virtual void operator()();
private:
//WEDataLoader* fpWeDataLoader;
WEDataLoader fWeDataLoader;
friend class ReadThreadFactory;
friend class ReadThreadFactory;
//friend class WEDataLoader;
};
@ -102,12 +102,12 @@ friend class ReadThreadFactory;
class ClearTableLockReadThread : public ReadThread
{
public:
ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~ClearTableLockReadThread();
ClearTableLockReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~ClearTableLockReadThread();
virtual void operator()();
virtual void operator()();
private:
boost::shared_ptr<WE_ClearTableLockCmd> fWeClearTableLockCmd;
boost::shared_ptr<WE_ClearTableLockCmd> fWeClearTableLockCmd;
};
@ -117,13 +117,13 @@ private:
class RedistributeReadThread : public ReadThread
{
public:
RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~RedistributeReadThread();
RedistributeReadThread(const messageqcpp::IOSocket& ios, ByteStream& ibs);
virtual ~RedistributeReadThread();
virtual void operator()();
virtual void operator()();
private:
};
//------------------------------------------------------------------------------
@ -132,24 +132,24 @@ private:
class GetFileSizeThread : public ReadThread
{
public:
GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& ibs, BRM::DBRM &dbrm);
virtual ~GetFileSizeThread();
GetFileSizeThread(const messageqcpp::IOSocket& ios, ByteStream& ibs, BRM::DBRM& dbrm);
virtual ~GetFileSizeThread();
virtual void operator()();
virtual void operator()();
private:
boost::shared_ptr<WE_GetFileSizes> fWeGetFileSizes;
int key;
boost::shared_ptr<WE_GetFileSizes> fWeGetFileSizes;
int key;
};
class ReadThreadFactory
{
public:
ReadThreadFactory(){}
virtual ~ReadThreadFactory(){}
ReadThreadFactory() {}
virtual ~ReadThreadFactory() {}
public:
static void CreateReadThread(ThreadPool& Tp, IOSocket& ios, BRM::DBRM &dbrm);
static void CreateReadThread(ThreadPool& Tp, IOSocket& ios, BRM::DBRM& dbrm);
};

325
writeengine/server/we_server.cpp
View File

@ -53,155 +53,174 @@ using namespace oam;
namespace
{
void added_a_pm(int)
{
logging::LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("we_server caught SIGHUP. Resetting connections");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid);
joblist::DistributedEngineComm::reset();
}
void added_a_pm(int)
{
logging::LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("we_server caught SIGHUP. Resetting connections");
msg.format( args1 );
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid);
joblist::DistributedEngineComm::reset();
}
}
int setupResources()
{
#ifndef _MSC_VER
struct rlimit rlim;
struct rlimit rlim;
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
return -1;
}
rlim.rlim_cur = rlim.rlim_max = 65536;
if (setrlimit(RLIMIT_NOFILE, &rlim) != 0) {
return -2;
}
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -1;
}
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
return -3;
}
rlim.rlim_cur = rlim.rlim_max = 65536;
if (setrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -2;
}
if (getrlimit(RLIMIT_NOFILE, &rlim) != 0)
{
return -3;
}
if (rlim.rlim_cur != 65536)
{
return -4;
}
if (rlim.rlim_cur != 65536) {
return -4;
}
#endif
return 0;
return 0;
}
int main(int argc, char** argv)
{
// get and set locale language
// get and set locale language
string systemLang = "C";
systemLang = funcexp::utf8::idb_setlocale();
systemLang = funcexp::utf8::idb_setlocale();
printf ("Locale is : %s\n", systemLang.c_str() );
//set BUSY_INIT state
{
// Is there a reason to have a seperate Oam instance for this?
Oam oam;
try
{
oam.processInitComplete("WriteEngineServer", oam::BUSY_INIT);
}
catch (...)
{
}
}
//BUG 2991
setlocale(LC_NUMERIC, "C");
//set BUSY_INIT state
{
// Is there a reason to have a seperate Oam instance for this?
Oam oam;
try
{
oam.processInitComplete("WriteEngineServer", oam::BUSY_INIT);
}
catch (...)
{
}
}
//BUG 2991
setlocale(LC_NUMERIC, "C");
#ifndef _MSC_VER
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = added_a_pm;
sigaction(SIGHUP, &sa, 0);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
sigaction(SIGHUP, &sa, 0);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
#endif
// Init WriteEngine Wrapper (including Config Columnstore.xml cache)
WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_WE_SRV );
// Init WriteEngine Wrapper (including Config Columnstore.xml cache)
WriteEngine::WriteEngineWrapper::init( WriteEngine::SUBSYSTEM_ID_WE_SRV );
#ifdef _MSC_VER
// In windows, initializing the wrapper (A dll) does not set the static variables
// in the main program
idbdatafile::IDBPolicy::configIDBPolicy();
#endif
Config weConfig;
Config weConfig;
ostringstream serverParms;
serverParms << "pm" << weConfig.getLocalModuleID() << "_WriteEngineServer";
ostringstream serverParms;
serverParms << "pm" << weConfig.getLocalModuleID() << "_WriteEngineServer";
// Create MessageQueueServer, with one retry in case the call to bind the
// known port fails with "Address already in use".
boost::scoped_ptr<MessageQueueServer> mqs;
bool tellUser = true;
for (;;)
{
try {
mqs.reset(new MessageQueueServer(serverParms.str()));
break;
}
// @bug4393 Error Handling for MessageQueueServer constructor exception
catch (runtime_error& re) {
string what = re.what();
if (what.find("Address already in use") != string::npos)
{
if (tellUser)
{
cerr << "Address already in use, retrying..." << endl;
tellUser = false;
}
sleep(5);
}
else
{
Oam oam;
try // Get out of BUSYINIT state; else OAM will not retry
{
oam.processInitComplete("WriteEngineServer");
}
catch (...)
{
}
// Create MessageQueueServer, with one retry in case the call to bind the
// known port fails with "Address already in use".
boost::scoped_ptr<MessageQueueServer> mqs;
bool tellUser = true;
// If/when a common logging class or function is added to the
// WriteEngineServer, we should use that. In the mean time,
// I will log this errmsg with inline calls to the logging.
logging::Message::Args args;
logging::Message message;
string errMsg("WriteEngineServer failed to initiate: ");
errMsg += what;
args.add( errMsg );
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
for (;;)
{
try
{
mqs.reset(new MessageQueueServer(serverParms.str()));
break;
}
// @bug4393 Error Handling for MessageQueueServer constructor exception
catch (runtime_error& re)
{
string what = re.what();
return 2;
}
}
}
if (what.find("Address already in use") != string::npos)
{
if (tellUser)
{
cerr << "Address already in use, retrying..." << endl;
tellUser = false;
}
sleep(5);
}
else
{
Oam oam;
try // Get out of BUSYINIT state; else OAM will not retry
{
oam.processInitComplete("WriteEngineServer");
}
catch (...)
{
}
// If/when a common logging class or function is added to the
// WriteEngineServer, we should use that. In the mean time,
// I will log this errmsg with inline calls to the logging.
logging::Message::Args args;
logging::Message message;
string errMsg("WriteEngineServer failed to initiate: ");
errMsg += what;
args.add( errMsg );
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
return 2;
}
}
}
int err = setupResources();
string errMsg;
switch (err)
{
case -1:
case -3:
errMsg = "Error getting file limits, please see non-root install documentation";
break;
case -2:
errMsg = "Error setting file limits, please see non-root install documentation";
break;
case -4:
errMsg = "Could not install file limits to required value, please see non-root install documentation";
break;
default:
break;
}
if (err < 0)
{
Oam oam;
@ -213,6 +232,7 @@ int main(int argc, char** argv)
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
cerr << errMsg << endl;
try
{
oam.processInitFailure();
@ -220,62 +240,65 @@ int main(int argc, char** argv)
catch (...)
{
}
return 2;
}
IOSocket ios;
size_t mt = 20;
size_t qs = mt * 100;
ThreadPool tp(mt, qs);
IOSocket ios;
size_t mt = 20;
size_t qs = mt * 100;
ThreadPool tp(mt, qs);
//set ACTIVE state
{
Oam oam;
try
{
oam.processInitComplete("WriteEngineServer", ACTIVE);
}
catch (...)
{
}
}
cout << "WriteEngineServer is ready" << endl;
BRM::DBRM dbrm;
for (;;)
{
try // BUG 4834 -
{
ios = mqs->accept();
//tp.invoke(ReadThread(ios));
ReadThreadFactory::CreateReadThread(tp,ios, dbrm);
{
/* logging::Message::Args args;
logging::Message message;
string aMsg("WriteEngineServer : New incoming connection");
args.add(aMsg);
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logInfoMessage( message ); */
}
}
catch(std::exception& ex) // BUG 4834 - log the exception
{
logging::Message::Args args;
logging::Message message;
string errMsg("WriteEngineServer : Exception caught on accept(): ");
errMsg += ex.what();
args.add( errMsg );
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
break;
}
}
//set ACTIVE state
{
Oam oam;
//It is an error to reach here...
return 1;
try
{
oam.processInitComplete("WriteEngineServer", ACTIVE);
}
catch (...)
{
}
}
cout << "WriteEngineServer is ready" << endl;
BRM::DBRM dbrm;
for (;;)
{
try // BUG 4834 -
{
ios = mqs->accept();
//tp.invoke(ReadThread(ios));
ReadThreadFactory::CreateReadThread(tp, ios, dbrm);
{
/* logging::Message::Args args;
logging::Message message;
string aMsg("WriteEngineServer : New incoming connection");
args.add(aMsg);
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logInfoMessage( message ); */
}
}
catch (std::exception& ex) // BUG 4834 - log the exception
{
logging::Message::Args args;
logging::Message message;
string errMsg("WriteEngineServer : Exception caught on accept(): ");
errMsg += ex.what();
args.add( errMsg );
message.format(args);
logging::LoggingID lid(SUBSYSTEM_ID_WE_SRV);
logging::MessageLog ml(lid);
ml.logCriticalMessage( message );
break;
}
}
//It is an error to reach here...
return 1;
}