database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
writeengine/splitter/resource.h
View File

@ -1,14 +1,14 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by cpimport.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by cpimport.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

1287
writeengine/splitter/we_brmupdater.cpp
View File

File diff suppressed because it is too large Load Diff

148
writeengine/splitter/we_brmupdater.h
View File

@ -14,77 +14,77 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_brmupdater.h
*
* Created on: Dec 13, 2011
* Author: bpaul
*/
#ifndef WE_BRMUPDATER_H_
#define WE_BRMUPDATER_H_
namespace WriteEngine
{
class WESDHandler; // forward deceleration
class WEBrmUpdater
{
public:
WEBrmUpdater(WESDHandler& Ref): fRef(Ref), fpBrm(0) {}
virtual ~WEBrmUpdater() {}
public:
bool updateCasualPartitionAndHighWaterMarkInBRM();
int updateCPAndHWMInBRM();
int updateCasualPartitionInBRM();
int updateHighWaterMarkInBRM();
bool prepareCasualPartitionInfo();
bool prepareHighWaterMarkInfo();
bool createBrmConnection()
{
fpBrm = new BRM::DBRM();
return (fpBrm)?true:false;
}
void releaseBrmConnection()
{
delete fpBrm;
fpBrm = 0;
}
public:
static bool prepareRowsInsertedInfo(std::string Entry, int64_t& TotRows,
int64_t& InsRows);
static bool prepareColumnOutOfRangeInfo(std::string Entry, int& ColNum,
CalpontSystemCatalog::ColDataType& ColType,
std::string& ColName, int& OorValues);
static bool prepareErrorFileInfo(std::string Entry, std::string& ErrFileName);
static bool prepareBadDataFileInfo(std::string Entry, std::string& BadFileName);
private:
WESDHandler& fRef;
BRM::DBRM* fpBrm;
//BRM::CPInfoMergeList_t fCPInfo;
std::vector<BRM::CPInfoMerge> fCPInfo;
std::vector<BRM::BulkSetHWMArg> fHWMInfo;
std::vector<BRM::CPInfo> fCPInfoData;
};
} /* namespace WriteEngine */
#endif /* WE_BRMUPDATER_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_brmupdater.h
*
* Created on: Dec 13, 2011
* Author: bpaul
*/
#ifndef WE_BRMUPDATER_H_
#define WE_BRMUPDATER_H_
namespace WriteEngine
{
class WESDHandler; // forward deceleration
class WEBrmUpdater
{
public:
WEBrmUpdater(WESDHandler& Ref): fRef(Ref), fpBrm(0) {}
virtual ~WEBrmUpdater() {}
public:
bool updateCasualPartitionAndHighWaterMarkInBRM();
int updateCPAndHWMInBRM();
int updateCasualPartitionInBRM();
int updateHighWaterMarkInBRM();
bool prepareCasualPartitionInfo();
bool prepareHighWaterMarkInfo();
bool createBrmConnection()
{
fpBrm = new BRM::DBRM();
return (fpBrm) ? true : false;
}
void releaseBrmConnection()
{
delete fpBrm;
fpBrm = 0;
}
public:
static bool prepareRowsInsertedInfo(std::string Entry, int64_t& TotRows,
int64_t& InsRows);
static bool prepareColumnOutOfRangeInfo(std::string Entry, int& ColNum,
CalpontSystemCatalog::ColDataType& ColType,
std::string& ColName, int& OorValues);
static bool prepareErrorFileInfo(std::string Entry, std::string& ErrFileName);
static bool prepareBadDataFileInfo(std::string Entry, std::string& BadFileName);
private:
WESDHandler& fRef;
BRM::DBRM* fpBrm;
//BRM::CPInfoMergeList_t fCPInfo;
std::vector<BRM::CPInfoMerge> fCPInfo;
std::vector<BRM::BulkSetHWMArg> fHWMInfo;
std::vector<BRM::CPInfo> fCPInfoData;
};
} /* namespace WriteEngine */
#endif /* WE_BRMUPDATER_H_ */

2872
writeengine/splitter/we_cmdargs.cpp
View File

File diff suppressed because it is too large Load Diff

368
writeengine/splitter/we_cmdargs.h
View File

@ -16,7 +16,7 @@
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
* $Id$
*
*******************************************************************************/
#ifndef WE_CMDARGS_H_
@ -34,144 +34,262 @@ namespace WriteEngine
class WECmdArgs
{
public:
WECmdArgs(int argc, char** argv);
virtual ~WECmdArgs(){}
public:
void appTestFunction();
void parseCmdLineArgs(int argc, char** argv);
std::string getCpImportCmdLine();
void setSchemaAndTableFromJobFile(std::string& JobName);
void setEnclByAndEscCharFromJobFile(std::string& JobName);
void usage();
void usageMode3();
bool checkForCornerCases();
void checkForBulkLogDir(const std::string& BulkRoot);
public:
WECmdArgs(int argc, char** argv);
virtual ~WECmdArgs() {}
public:
void appTestFunction();
void parseCmdLineArgs(int argc, char** argv);
std::string getCpImportCmdLine();
void setSchemaAndTableFromJobFile(std::string& JobName);
void setEnclByAndEscCharFromJobFile(std::string& JobName);
void usage();
void usageMode3();
bool checkForCornerCases();
void checkForBulkLogDir(const std::string& BulkRoot);
void addJobFilesToVector(std::string& JobName);
void splitConfigFilePerTable(std::string& ConfigName, int tblCount);
void write2ConfigFiles(std::vector<std::ofstream*>& Files,
char*pBuff, int FileIdx);
void updateWithJobFile(int Idx);
void addJobFilesToVector(std::string& JobName);
void splitConfigFilePerTable(std::string& ConfigName, int tblCount);
void write2ConfigFiles(std::vector<std::ofstream*>& Files,
char* pBuff, int FileIdx);
void updateWithJobFile(int Idx);
public:
std::string getJobFileName();
std::string getBrmRptFileName();
std::string getTmpFileDir();
std::string getBulkRootDir();
unsigned int getBatchQuantity();
void checkJobIdCase();
std::string getFileNameFromPath(const std::string& Path) const;
std::string getModuleID();
std::string replaceCharInStr(const std::string& Str,char C,char R);
public:
std::string getJobFileName();
std::string getBrmRptFileName();
std::string getTmpFileDir();
std::string getBulkRootDir();
unsigned int getBatchQuantity();
void checkJobIdCase();
std::string getFileNameFromPath(const std::string& Path) const;
std::string getModuleID();
std::string replaceCharInStr(const std::string& Str, char C, char R);
std::string getJobId() const { return fJobId; }
std::string getOrigJobId() const { return fOrigJobId; }
std::string getLocFile() const { return fLocFile; }
int getMode() const { return fMode; }
int getArgMode() const { return fArgMode; }
bool isHelpMode() { return fHelp; }
int getDebugLvl() { return fDebugLvl; }
char getEnclChar() { return fEnclosedChar; }
char getEscChar() { return fEscChar; }
char getDelimChar() { return fColDelim; }
ImportDataMode getImportDataMode() const { return fImportDataMode; }
bool getConsoleLog() { return fConsoleLog; }
std::string getJobId() const
{
return fJobId;
}
std::string getOrigJobId() const
{
return fOrigJobId;
}
std::string getLocFile() const
{
return fLocFile;
}
int getMode() const
{
return fMode;
}
int getArgMode() const
{
return fArgMode;
}
bool isHelpMode()
{
return fHelp;
}
int getDebugLvl()
{
return fDebugLvl;
}
char getEnclChar()
{
return fEnclosedChar;
}
char getEscChar()
{
return fEscChar;
}
char getDelimChar()
{
return fColDelim;
}
ImportDataMode getImportDataMode() const
{
return fImportDataMode;
}
bool getConsoleLog()
{
return fConsoleLog;
}
bool isCpimportInvokeMode(){return (fBlockMode3)? false : fCpiInvoke;}
bool isQuiteMode() const { return fQuiteMode; }
void setJobId(std::string fJobId) { this->fJobId = fJobId; }
void setOrigJobId(std::string fOrigJobId){ this->fOrigJobId = fJobId; }
void setLocFile(std::string fLocFile) { this->fLocFile = fLocFile; }
void setMode(int fMode) { this->fMode = fMode; }
void setArgMode(int ArgMode) { this->fArgMode = ArgMode; }
void setPmFile(std::string fPmFile) { this->fPmFile = fPmFile; }
void setQuiteMode(bool fQuiteMode) { this->fQuiteMode = fQuiteMode; }
void setVerbose(int fVerbose) { this->fVerbose = fVerbose; }
void setJobFileName(std::string& JobFileName){fJobFile = JobFileName;}
void setBrmRptFileName(std::string& BrmFileName)
{ this->fBrmRptFile = BrmFileName; }
void setCpiInvoke(bool CpiInvoke=true) { this->fCpiInvoke = CpiInvoke; }
void setBlockMode3(bool Block) { this->fBlockMode3 = Block; }
void setTruncationAsError(bool bTruncationAsError)
{ fbTruncationAsError = bTruncationAsError; }
bool isJobLogOnly() const { return fJobLogOnly; }
void setJobUUID(const boost::uuids::uuid& jobUUID) { fUUID = jobUUID; }
bool getConsoleOutput( ) {return fConsoleOutput; }
bool isCpimportInvokeMode()
{
return (fBlockMode3) ? false : fCpiInvoke;
}
bool isQuiteMode() const
{
return fQuiteMode;
}
void setJobId(std::string fJobId)
{
this->fJobId = fJobId;
}
void setOrigJobId(std::string fOrigJobId)
{
this->fOrigJobId = fJobId;
}
void setLocFile(std::string fLocFile)
{
this->fLocFile = fLocFile;
}
void setMode(int fMode)
{
this->fMode = fMode;
}
void setArgMode(int ArgMode)
{
this->fArgMode = ArgMode;
}
void setPmFile(std::string fPmFile)
{
this->fPmFile = fPmFile;
}
void setQuiteMode(bool fQuiteMode)
{
this->fQuiteMode = fQuiteMode;
}
void setVerbose(int fVerbose)
{
this->fVerbose = fVerbose;
}
void setJobFileName(std::string& JobFileName)
{
fJobFile = JobFileName;
}
void setBrmRptFileName(std::string& BrmFileName)
{
this->fBrmRptFile = BrmFileName;
}
void setCpiInvoke(bool CpiInvoke = true)
{
this->fCpiInvoke = CpiInvoke;
}
void setBlockMode3(bool Block)
{
this->fBlockMode3 = Block;
}
void setTruncationAsError(bool bTruncationAsError)
{
fbTruncationAsError = bTruncationAsError;
}
bool isJobLogOnly() const
{
return fJobLogOnly;
}
void setJobUUID(const boost::uuids::uuid& jobUUID)
{
fUUID = jobUUID;
}
bool getConsoleOutput( )
{
return fConsoleOutput;
}
private: // variables for SplitterApp
typedef std::vector<std::string> VecArgs;
VecArgs fVecArgs;
typedef std::vector<unsigned int> VecInts;
VecInts fPmVec;
private: // variables for SplitterApp
typedef std::vector<std::string> VecArgs;
VecArgs fVecArgs;
typedef std::vector<unsigned int> VecInts;
VecInts fPmVec;
VecArgs fVecJobFiles; //JobFiles splitter from master JobFile
int fMultiTableCount; //MultiTable count
VecArgs fColFldsFromJobFile;//List of columns from any job file, that
// represent fields in the import data
VecArgs fVecJobFiles; //JobFiles splitter from master JobFile
int fMultiTableCount; //MultiTable count
VecArgs fColFldsFromJobFile;//List of columns from any job file, that
// represent fields in the import data
public:
bool getPmStatus(int Id);
bool str2PmList(std::string& PmList, VecInts& V);
int getPmVecSize(){ return fPmVec.size(); }
void add2PmVec(int PmId){ fPmVec.push_back(PmId); }
public:
bool getPmStatus(int Id);
bool str2PmList(std::string& PmList, VecInts& V);
int getPmVecSize()
{
return fPmVec.size();
}
void add2PmVec(int PmId)
{
fPmVec.push_back(PmId);
}
WECmdArgs::VecInts& getPmVec() { return fPmVec; }
std::string getPmFile() const { return fPmFile; }
int getVerbose() const { return fVerbose; }
std::string getTableName() const { return fTable; }
std::string getSchemaName() const { return fSchema; }
bool getTruncationAsError() const {return fbTruncationAsError; }
WECmdArgs::VecInts& getPmVec()
{
return fPmVec;
}
std::string getPmFile() const
{
return fPmFile;
}
int getVerbose() const
{
return fVerbose;
}
std::string getTableName() const
{
return fTable;
}
std::string getSchemaName() const
{
return fSchema;
}
bool getTruncationAsError() const
{
return fbTruncationAsError;
}
int getMultiTableCount() const { return fMultiTableCount; }
void setMultiTableCount(int Count) { fMultiTableCount = Count; }
int getMultiTableCount() const
{
return fMultiTableCount;
}
void setMultiTableCount(int Count)
{
fMultiTableCount = Count;
}
std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342
void getColumnList( std::set<std::string>& columnList ) const;
std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342
void getColumnList( std::set<std::string>& columnList ) const;
private: // variables for SplitterApp
std::string fJobId; // JobID
std::string fOrigJobId; // Original JobID, in case we have to split it
bool fJobLogOnly; // Job number is only for log filename only
bool fHelp; // Help mode
int fMode; // splitter Mode
int fArgMode; // Argument mode, dep. on this fMode is decided.
bool fQuiteMode; // in quite mode or not
bool fConsoleLog; // Log everything to console - w.r.t cpimport
int fVerbose; // how many v's
std::string fPmFile; // FileName at PM
std::string fPmFilePath;// Path of input file in PM
std::string fLocFile; // Local file name
std::string fBrmRptFile;// BRM report file
std::string fJobPath; // Path to Job File
std::string fTmpFileDir;// Temp file directory.
std::string fBulkRoot; // Bulk Root path
std::string fJobFile; // Job File Name
private: // variables for SplitterApp
std::string fJobId; // JobID
std::string fOrigJobId; // Original JobID, in case we have to split it
bool fJobLogOnly; // Job number is only for log filename only
bool fHelp; // Help mode
int fMode; // splitter Mode
int fArgMode; // Argument mode, dep. on this fMode is decided.
bool fQuiteMode; // in quite mode or not
bool fConsoleLog; // Log everything to console - w.r.t cpimport
int fVerbose; // how many v's
std::string fPmFile; // FileName at PM
std::string fPmFilePath;// Path of input file in PM
std::string fLocFile; // Local file name
std::string fBrmRptFile;// BRM report file
std::string fJobPath; // Path to Job File
std::string fTmpFileDir;// Temp file directory.
std::string fBulkRoot; // Bulk Root path
std::string fJobFile; // Job File Name
unsigned int fBatchQty; // No. of batch Qty.
int fNoOfReadThrds; // No. of read buffers
//std::string fConfig; // config filename
int fDebugLvl; // Debug level
int fMaxErrors; // Max allowable errors
int fReadBufSize; // Read buffer size
int fIOReadBufSize; // I/O read buffer size
int fSetBufSize; // Buff size w/setvbuf
char fColDelim; // column delimiter
char fEnclosedChar; // enclosed by char
char fEscChar; // esc char
int fNoOfWriteThrds; // No. of write threads
bool fNullStrMode; // set null string mode - treat null as null
ImportDataMode fImportDataMode; // Importing text or binary data
std::string fPrgmName; // argv[0]
std::string fSchema; // Schema name - positional parmater
std::string fTable; // Table name - table name parameter
unsigned int fBatchQty; // No. of batch Qty.
int fNoOfReadThrds; // No. of read buffers
//std::string fConfig; // config filename
int fDebugLvl; // Debug level
int fMaxErrors; // Max allowable errors
int fReadBufSize; // Read buffer size
int fIOReadBufSize; // I/O read buffer size
int fSetBufSize; // Buff size w/setvbuf
char fColDelim; // column delimiter
char fEnclosedChar; // enclosed by char
char fEscChar; // esc char
int fNoOfWriteThrds; // No. of write threads
bool fNullStrMode; // set null string mode - treat null as null
ImportDataMode fImportDataMode; // Importing text or binary data
std::string fPrgmName; // argv[0]
std::string fSchema; // Schema name - positional parmater
std::string fTable; // Table name - table name parameter
bool fCpiInvoke; // invoke cpimport in mode 3
bool fBlockMode3; // Do not allow Mode 3
bool fbTruncationAsError; // Treat string truncation as error
boost::uuids::uuid fUUID;
bool fConsoleOutput; // If false, no output to console.
bool fCpiInvoke; // invoke cpimport in mode 3
bool fBlockMode3; // Do not allow Mode 3
bool fbTruncationAsError; // Treat string truncation as error
boost::uuids::uuid fUUID;
bool fConsoleOutput; // If false, no output to console.
};
//----------------------------------------------------------------------

1099
writeengine/splitter/we_filereadthread.cpp
View File

File diff suppressed because it is too large Load Diff

253
writeengine/splitter/we_filereadthread.h
View File

@ -14,114 +14,145 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_filereadthread.h
*
* Created on: Oct 25, 2011
* Author: bpaul
*/
#ifndef WE_FILEREADTHREAD_H_
#define WE_FILEREADTHREAD_H_
namespace WriteEngine
{
class WESDHandler;
class WEFileReadThread;
class WEReadThreadRunner
{
public:
WEReadThreadRunner(WEFileReadThread& Owner):fRef(Owner)
{ // ctor
}
~WEReadThreadRunner()
{
}
void operator()(); // Thread function
private:
WEFileReadThread& fRef;
};
//------------------------------------------------------------------------------
class WEFileReadThread
{
public:
WEFileReadThread(WESDHandler& aSdh);
virtual ~WEFileReadThread();
void reset();
void setup(std::string FileName);
void shutdown();
void feedData();
unsigned int readDataFile(messageqcpp::SBS& Sbs);
unsigned int readBinaryDataFile(messageqcpp::SBS& Sbs, unsigned int recLen);
void openInFile();
int getNextRow(std::istream& ifs, char*pBuf, int MaxLen);
boost::thread* getFpThread() const { return fpThread; }
bool isContinue() const { return fContinue; }
void setContinue(bool fContinue) { this->fContinue = fContinue; }
std::string getInFileName() const { return fInFileName; }
unsigned int getTgtPmId() const { return fTgtPmId; }
unsigned int getBatchQty() const { return fBatchQty; }
void setFpThread(boost::thread* fpThread) { this->fpThread = fpThread; }
void setInFileName(std::string fInFileName)
{
if((0==fInFileName.compare("STDIN"))||(0==fInFileName.compare("stdin")))
this->fInFileName = "/dev/stdin";
else
this->fInFileName = fInFileName;
}
//@BUG 4326
const std::istream& getInFile() const { return fInFile; }
void setBatchQty(unsigned int BatchQty) { fBatchQty = BatchQty; }
bool chkForListOfFiles(std::string& FileName);
std::string getNextInputDataFile();
void add2InputDataFileList(std::string& FileName);
private:
enum { MAXBUFFSIZE=1024*1024 };
// don't allow anyone else to set
void setTgtPmId(unsigned int fTgtPmId) { this->fTgtPmId = fTgtPmId; }
WESDHandler & fSdh;
boost::thread *fpThread;
boost::mutex fFileMutex;
bool fContinue;
std::string fInFileName;
std::istream fInFile; //@BUG 4326
std::ifstream fIfFile; //@BUG 4326
typedef std::list<std::string> strList;
strList fInfileList;
unsigned int fTgtPmId;
unsigned int fBatchQty;
bool fEnclEsc; // Encl/Esc char is set
char fEncl; // Encl char
char fEsc; // Esc char
char fDelim; // Column Delimit char
char fBuff[MAXBUFFSIZE]; // main data buffer
};
} /* namespace WriteEngine */
#endif /* WE_FILEREADTHREAD_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_filereadthread.h
*
* Created on: Oct 25, 2011
* Author: bpaul
*/
#ifndef WE_FILEREADTHREAD_H_
#define WE_FILEREADTHREAD_H_
namespace WriteEngine
{
class WESDHandler;
class WEFileReadThread;
class WEReadThreadRunner
{
public:
WEReadThreadRunner(WEFileReadThread& Owner): fRef(Owner)
{
// ctor
}
~WEReadThreadRunner()
{
}
void operator()(); // Thread function
private:
WEFileReadThread& fRef;
};
//------------------------------------------------------------------------------
class WEFileReadThread
{
public:
WEFileReadThread(WESDHandler& aSdh);
virtual ~WEFileReadThread();
void reset();
void setup(std::string FileName);
void shutdown();
void feedData();
unsigned int readDataFile(messageqcpp::SBS& Sbs);
unsigned int readBinaryDataFile(messageqcpp::SBS& Sbs, unsigned int recLen);
void openInFile();
int getNextRow(std::istream& ifs, char* pBuf, int MaxLen);
boost::thread* getFpThread() const
{
return fpThread;
}
bool isContinue() const
{
return fContinue;
}
void setContinue(bool fContinue)
{
this->fContinue = fContinue;
}
std::string getInFileName() const
{
return fInFileName;
}
unsigned int getTgtPmId() const
{
return fTgtPmId;
}
unsigned int getBatchQty() const
{
return fBatchQty;
}
void setFpThread(boost::thread* fpThread)
{
this->fpThread = fpThread;
}
void setInFileName(std::string fInFileName)
{
if ((0 == fInFileName.compare("STDIN")) || (0 == fInFileName.compare("stdin")))
this->fInFileName = "/dev/stdin";
else
this->fInFileName = fInFileName;
}
//@BUG 4326
const std::istream& getInFile() const
{
return fInFile;
}
void setBatchQty(unsigned int BatchQty)
{
fBatchQty = BatchQty;
}
bool chkForListOfFiles(std::string& FileName);
std::string getNextInputDataFile();
void add2InputDataFileList(std::string& FileName);
private:
enum { MAXBUFFSIZE = 1024 * 1024 };
// don't allow anyone else to set
void setTgtPmId(unsigned int fTgtPmId)
{
this->fTgtPmId = fTgtPmId;
}
WESDHandler& fSdh;
boost::thread* fpThread;
boost::mutex fFileMutex;
bool fContinue;
std::string fInFileName;
std::istream fInFile; //@BUG 4326
std::ifstream fIfFile; //@BUG 4326
typedef std::list<std::string> strList;
strList fInfileList;
unsigned int fTgtPmId;
unsigned int fBatchQty;
bool fEnclEsc; // Encl/Esc char is set
char fEncl; // Encl char
char fEsc; // Esc char
char fDelim; // Column Delimit char
char fBuff[MAXBUFFSIZE]; // main data buffer
};
} /* namespace WriteEngine */
#endif /* WE_FILEREADTHREAD_H_ */

14
writeengine/splitter/we_respreadthread.cpp
View File

@ -39,24 +39,24 @@ namespace WriteEngine
{
WERespReadThread::WERespReadThread(WESDHandler& aSdh):
fSdh(aSdh)
fSdh(aSdh)
{
// ctor
// ctor
}
WERespReadThread::WERespReadThread(const WERespReadThread& rhs):
fSdh(rhs.fSdh)
fSdh(rhs.fSdh)
{
// copy ctor
// copy ctor
}
WERespReadThread::~WERespReadThread()
{
// dtor
// dtor
}
void WERespReadThread::operator()()
{
//call datahandler checkForAllPmMsgs()
fSdh.checkForRespMsgs();
//call datahandler checkForAllPmMsgs()
fSdh.checkForRespMsgs();
}

70
writeengine/splitter/we_respreadthread.h
View File

@ -14,38 +14,38 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_respreadthread.h
*
* Created on: Oct 18, 2011
* Author: bpaul
*/
#ifndef WE_RESPREADTHREAD_H_
#define WE_RESPREADTHREAD_H_
namespace WriteEngine
{
class WERespReadThread
{
public:
WERespReadThread(WESDHandler& aSdh);
WERespReadThread(const WERespReadThread& rhs);
virtual ~WERespReadThread();
void operator()();
private:
WESDHandler& fSdh;
};
} /* namespace WriteEngine */
#endif /* WE_RESPREADTHREAD_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_respreadthread.h
*
* Created on: Oct 18, 2011
* Author: bpaul
*/
#ifndef WE_RESPREADTHREAD_H_
#define WE_RESPREADTHREAD_H_
namespace WriteEngine
{
class WERespReadThread
{
public:
WERespReadThread(WESDHandler& aSdh);
WERespReadThread(const WERespReadThread& rhs);
virtual ~WERespReadThread();
void operator()();
private:
WESDHandler& fSdh;
};
} /* namespace WriteEngine */
#endif /* WE_RESPREADTHREAD_H_ */

4296
writeengine/splitter/we_sdhandler.cpp
View File

File diff suppressed because it is too large Load Diff

329
writeengine/splitter/we_sdhandler.h
View File

@ -57,19 +57,22 @@ class WEFileReadThread;
class WEPmList
{
public:
WEPmList():fPmList(),fListMutex(){}
virtual ~WEPmList(){ fPmList.clear(); }
WEPmList(): fPmList(), fListMutex() {}
virtual ~WEPmList()
{
fPmList.clear();
}
void addPm2List(int PmId);
void addPriorityPm2List(int PmId);
int getNextPm();
void clearPmList();
bool check4Pm(int PmId);
void addPm2List(int PmId);
void addPriorityPm2List(int PmId);
int getNextPm();
void clearPmList();
bool check4Pm(int PmId);
private:
typedef std::list<int> WePmList; // List to add in front
WePmList fPmList;
boost::mutex fListMutex; //mutex controls add/remove
typedef std::list<int> WePmList; // List to add in front
WePmList fPmList;
boost::mutex fListMutex; //mutex controls add/remove
};
@ -79,19 +82,19 @@ class WESDHandler
{
public:
WESDHandler(WESplitterApp& Ref);
WESDHandler(const WESDHandler& rhs);
virtual ~WESDHandler();
WESDHandler(WESplitterApp& Ref);
WESDHandler(const WESDHandler& rhs);
virtual ~WESDHandler();
void setup();
void shutdown();
void reset();
void send2Pm(messageqcpp::SBS& Sbs, unsigned int PmId=0);
void send2Pm(messageqcpp::ByteStream & Bs, unsigned int PmId=0);
void send2Pm(messageqcpp::SBS& Sbs, unsigned int PmId = 0);
void send2Pm(messageqcpp::ByteStream& Bs, unsigned int PmId = 0);
void sendEODMsg();
void checkForRespMsgs();
void add2RespQueue(const messageqcpp::SBS & Sbs);
void exportJobFile(std::string &JobId, std::string &JobFileName);
void add2RespQueue(const messageqcpp::SBS& Sbs);
void exportJobFile(std::string& JobId, std::string& JobFileName);
int leastDataSendPm();
bool check4AllBrmReports();
bool updateCPAndHWMInBRM();
@ -116,20 +119,20 @@ public:
void onAckResponse(int PmId);
void onNakResponse(int PmId);
void onEodResponse(int Pmid);
void onPmErrorResponse(int PmId);
void onKeepAliveMessage(int PmId);
void onCpimportPass(int PmId);
void onCpimportFail(int PmId, bool SigHandle=false);
void onImpFileError(int PmId);
void onBrmReport(int PmId, messageqcpp::SBS& Sbs);
void onErrorFile(int PmId, messageqcpp::SBS& Sbs);
void onBadFile(int PmId, messageqcpp::SBS& Sbs);
void onRollbackResult(int PmId, messageqcpp::SBS& Sbs);
void onCleanupResult(int PmId, messageqcpp::SBS& Sbs);
void onDBRootCount(int PmId, messageqcpp::SBS& Sbs);
void onHandlingSignal();
void onHandlingSigHup();
void onDisconnectFailure();
void onPmErrorResponse(int PmId);
void onKeepAliveMessage(int PmId);
void onCpimportPass(int PmId);
void onCpimportFail(int PmId, bool SigHandle = false);
void onImpFileError(int PmId);
void onBrmReport(int PmId, messageqcpp::SBS& Sbs);
void onErrorFile(int PmId, messageqcpp::SBS& Sbs);
void onBadFile(int PmId, messageqcpp::SBS& Sbs);
void onRollbackResult(int PmId, messageqcpp::SBS& Sbs);
void onCleanupResult(int PmId, messageqcpp::SBS& Sbs);
void onDBRootCount(int PmId, messageqcpp::SBS& Sbs);
void onHandlingSignal();
void onHandlingSigHup();
void onDisconnectFailure();
int getNextPm2Feed();
int getNextDbrPm2Send();
@ -148,44 +151,106 @@ public:
char getEnclChar();
char getEscChar();
char getDelimChar();
bool getConsoleLog();
bool getConsoleLog();
ImportDataMode getImportDataMode() const;
void sysLog(const logging::Message::Args& msgArgs,
logging::LOG_TYPE logType, logging::Message::MessageID msgId);
logging::LOG_TYPE logType, logging::Message::MessageID msgId);
boost::thread* getFpRespThread() const { return fpRespThread; }
unsigned int getQId() const { return fQId; }
void setFpRespThread(boost::thread *pRespThread)
{ fpRespThread = pRespThread; }
void setQId(unsigned int QId) { fQId = QId; }
bool isContinue() const { return fContinue; }
void setContinue(bool Continue) { fContinue = Continue; }
int getPmCount() const { return fPmCount; }
void setPmCount(int PmCount) { fPmCount = PmCount; }
int getNextPm2Send() { return fDataFeedList.getNextPm(); }
bool check4Ack(unsigned int PmId) { return fDataFeedList.check4Pm(PmId); }
int getTableOID() { return fTableOId; }
void setDebugLvl(int DebugLvl) { fDebugLvl = DebugLvl; }
int getDebugLvl() { return fDebugLvl; }
unsigned int getTableRecLen() const { return fFixedBinaryRecLen; }
boost::thread* getFpRespThread() const
{
return fpRespThread;
}
unsigned int getQId() const
{
return fQId;
}
void setFpRespThread(boost::thread* pRespThread)
{
fpRespThread = pRespThread;
}
void setQId(unsigned int QId)
{
fQId = QId;
}
bool isContinue() const
{
return fContinue;
}
void setContinue(bool Continue)
{
fContinue = Continue;
}
int getPmCount() const
{
return fPmCount;
}
void setPmCount(int PmCount)
{
fPmCount = PmCount;
}
int getNextPm2Send()
{
return fDataFeedList.getNextPm();
}
bool check4Ack(unsigned int PmId)
{
return fDataFeedList.check4Pm(PmId);
}
int getTableOID()
{
return fTableOId;
}
void setDebugLvl(int DebugLvl)
{
fDebugLvl = DebugLvl;
}
int getDebugLvl()
{
return fDebugLvl;
}
unsigned int getTableRecLen() const
{
return fFixedBinaryRecLen;
}
void updateRowTx(unsigned int RowCnt, int CIdx)
{ fWeSplClients[CIdx]->updateRowTx(RowCnt); }
void resetRowTx(){ for (int aCnt = 1; aCnt <= fPmCount; aCnt++)
{if (fWeSplClients[aCnt] != 0) {fWeSplClients[aCnt]->resetRowTx(); } } }
{
fWeSplClients[CIdx]->updateRowTx(RowCnt);
}
void resetRowTx()
{
for (int aCnt = 1; aCnt <= fPmCount; aCnt++)
{
if (fWeSplClients[aCnt] != 0)
{
fWeSplClients[aCnt]->resetRowTx();
}
}
}
void setRowsUploadInfo(int PmId, int64_t RowsRead, int64_t RowsInserted)
{ fWeSplClients[PmId]->setRowsUploadInfo(RowsRead, RowsInserted); }
void add2ColOutOfRangeInfo(int PmId, int ColNum,
{
fWeSplClients[PmId]->setRowsUploadInfo(RowsRead, RowsInserted);
}
void add2ColOutOfRangeInfo(int PmId, int ColNum,
CalpontSystemCatalog::ColDataType ColType,
std::string& ColName, int NoOfOors)
{ fWeSplClients[PmId]->add2ColOutOfRangeInfo(ColNum, ColType, ColName, NoOfOors); }
{
fWeSplClients[PmId]->add2ColOutOfRangeInfo(ColNum, ColType, ColName, NoOfOors);
}
void setErrorFileName(int PmId, const std::string& ErrFileName)
{ fWeSplClients[PmId]->setErrInfoFile(ErrFileName); }
{
fWeSplClients[PmId]->setErrInfoFile(ErrFileName);
}
void setBadFileName(int PmId, const std::string& BadFileName)
{ fWeSplClients[PmId]->setBadDataFile(BadFileName); }
{
fWeSplClients[PmId]->setBadDataFile(BadFileName);
}
void setDisconnectFailure(bool Flag);
bool getDisconnectFailure(){ return fDisconnectFailure; }
void setDisconnectFailure(bool Flag);
bool getDisconnectFailure()
{
return fDisconnectFailure;
}
public: // for multi-table support
WESplitterApp& fRef;
@ -193,7 +258,7 @@ public: // for multi-table support
private:
unsigned int fQId;
joblist::ResourceManager *fRm;
joblist::ResourceManager* fRm;
oam::Oam fOam;
oam::ModuleTypeConfig fModuleTypeConfig;
int fDebugLvl;
@ -212,12 +277,12 @@ private:
typedef std::list<messageqcpp::SBS> WESRespList;
WESRespList fRespList;
// Other member variables
boost::thread *fpRespThread;
boost::thread* fpRespThread;
WEPmList fDataFeedList;
WEFileReadThread fFileReadThread;
bool fDisconnectFailure; //Failure due to disconnect from PM
bool fDisconnectFailure; //Failure due to disconnect from PM
bool fForcedFailure;
bool fAllCpiStarted;
bool fFirstDataSent;
@ -227,7 +292,7 @@ private:
// set of PM specific vector entries
typedef std::vector<WESplClient*> WESplClients;
WESplClients fWeSplClients;
enum { MAX_PMS = 512, MAX_QSIZE=10, MAX_WES_QSIZE=100};
enum { MAX_PMS = 512, MAX_QSIZE = 10, MAX_WES_QSIZE = 100};
typedef std::vector<std::string> StrVec;
StrVec fBrmRptVec;
@ -242,69 +307,93 @@ private:
class WEImportRslt
{
public:
WEImportRslt():fRowsPro(0),fRowsIns(0),fStartTime(),fEndTime(),fTotTime(0){}
~WEImportRslt(){}
WEImportRslt(): fRowsPro(0), fRowsIns(0), fStartTime(), fEndTime(), fTotTime(0) {}
~WEImportRslt() {}
public:
void reset(){fRowsPro=0; fRowsIns=0; fTotTime=0; fColOorVec.clear();}
void updateRowsProcessed(int64_t Rows){ fRowsPro+=Rows; }
void updateRowsInserted(int64_t Rows){ fRowsIns+=Rows; }
void updateColOutOfRangeInfo(int aColNum, CalpontSystemCatalog::ColDataType aColType,
void reset()
{
fRowsPro = 0;
fRowsIns = 0;
fTotTime = 0;
fColOorVec.clear();
}
void updateRowsProcessed(int64_t Rows)
{
fRowsPro += Rows;
}
void updateRowsInserted(int64_t Rows)
{
fRowsIns += Rows;
}
void updateColOutOfRangeInfo(int aColNum, CalpontSystemCatalog::ColDataType aColType,
std::string aColName, int aNoOfOors)
{
WEColOorVec::iterator aIt = fColOorVec.begin();
while(aIt != fColOorVec.end())
{
if ((*aIt).fColNum == aColNum)
{
(*aIt).fNoOfOORs += aNoOfOors;
break;
}
aIt++;
}
if (aIt == fColOorVec.end())
{
// First time for aColNum to have out of range count
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = aColNum;
aColOorInfo.fColType = aColType;
aColOorInfo.fColName = aColName;
aColOorInfo.fNoOfOORs = aNoOfOors;
fColOorVec.push_back(aColOorInfo);
}
{
WEColOorVec::iterator aIt = fColOorVec.begin();
while (aIt != fColOorVec.end())
{
if ((*aIt).fColNum == aColNum)
{
(*aIt).fNoOfOORs += aNoOfOors;
break;
}
aIt++;
}
if (aIt == fColOorVec.end())
{
// First time for aColNum to have out of range count
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = aColNum;
aColOorInfo.fColType = aColType;
aColOorInfo.fColName = aColName;
aColOorInfo.fNoOfOORs = aNoOfOors;
fColOorVec.push_back(aColOorInfo);
}
#if 0
try
{
fColOorVec.at(aColNum).fNoOfOORs += aNoOfOors;
}
catch (out_of_range& e)
{
// First time for aColNum to have out of range count
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = aColNum;
aColOorInfo.fColName = aColName;
aColOorInfo.fNoOfOORs = aNoOfOors;
fColOorVec[aColNum] = aColOorInfo;
}
try
{
fColOorVec.at(aColNum).fNoOfOORs += aNoOfOors;
}
catch (out_of_range& e)
{
// First time for aColNum to have out of range count
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = aColNum;
aColOorInfo.fColName = aColName;
aColOorInfo.fNoOfOORs = aNoOfOors;
fColOorVec[aColNum] = aColOorInfo;
}
#endif
}
void startTimer(){ gettimeofday( &fStartTime, 0 ); }
void stopTimer(){ gettimeofday( &fEndTime, 0 ); }
float getTotalRunTime()
{
//fTotTime = (fEndTime>0)?(fEndTime-fStartTime):0;
fTotTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) -
(fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0));
return fTotTime;
}
}
void startTimer()
{
gettimeofday( &fStartTime, 0 );
}
void stopTimer()
{
gettimeofday( &fEndTime, 0 );
}
float getTotalRunTime()
{
//fTotTime = (fEndTime>0)?(fEndTime-fStartTime):0;
fTotTime = (fEndTime.tv_sec + (fEndTime.tv_usec / 1000000.0)) -
(fStartTime.tv_sec + (fStartTime.tv_usec / 1000000.0));
return fTotTime;
}
public:
int64_t fRowsPro; //Rows processed
int64_t fRowsIns; //Rows inserted
timeval fStartTime; //StartTime
timeval fEndTime; //EndTime
float fTotTime; //TotalTime
// A vector containing a list of rows and counts of Out of Range values
WEColOorVec fColOorVec;
int64_t fRowsPro; //Rows processed
int64_t fRowsIns; //Rows inserted
timeval fStartTime; //StartTime
timeval fEndTime; //EndTime
float fTotTime; //TotalTime
// A vector containing a list of rows and counts of Out of Range values
WEColOorVec fColOorVec;
};
WEImportRslt fImportRslt;
@ -312,7 +401,7 @@ private:
friend class WEBrmUpdater;
friend class WESplitterApp;
friend class WEFileReadThread;
friend class WETableLockGrabber;
friend class WETableLockGrabber;
};
//------------------------------------------------------------------------------

546
writeengine/splitter/we_splclient.cpp
View File

@ -64,7 +64,7 @@ namespace WriteEngine
//------------------------------------------------------------------------------
void WESplClientRunner::operator()()
{
fOwner.sendAndRecv();
fOwner.sendAndRecv();
}
//------------------------------------------------------------------------------
@ -73,74 +73,76 @@ void WESplClientRunner::operator()()
WESplClient::WESplClient(WESDHandler& Sdh, int PmId):
fContinue(true),
fConnected(false),
fPmId(PmId),
fDbrCnt(0),
fDbrVar(0),
fDataRqstCnt(0),
fRdSecTo(0),
fRowTx(0),
fBytesTx(0),
fBytesRcv(0),
fLastInTime(0),
fStartTime(time(0)),
fSend(true),
fCpiStarted(false),
fCpiPassed(false),
fCpiFailed(false),
fBrmRptRcvd(false),
fRollbackRslt(0),
fCleanupRslt(0),
fServer(),
fClnt(),
fpThread(0),
fOwner(Sdh)
fContinue(true),
fConnected(false),
fPmId(PmId),
fDbrCnt(0),
fDbrVar(0),
fDataRqstCnt(0),
fRdSecTo(0),
fRowTx(0),
fBytesTx(0),
fBytesRcv(0),
fLastInTime(0),
fStartTime(time(0)),
fSend(true),
fCpiStarted(false),
fCpiPassed(false),
fCpiFailed(false),
fBrmRptRcvd(false),
fRollbackRslt(0),
fCleanupRslt(0),
fServer(),
fClnt(),
fpThread(0),
fOwner(Sdh)
{
// TODO ctor
// TODO ctor
}
//------------------------------------------------------------------------------
WESplClient::~WESplClient()
{
delete fpThread;
fpThread=0;
delete fpThread;
fpThread = 0;
}
//------------------------------------------------------------------------------
void WESplClient::setup()
{
// do the setup stuffs here
if(fOwner.getDebugLvl())
cout << "setting connection to moduleid " << getPmId() << endl;
// do the setup stuffs here
if (fOwner.getDebugLvl())
cout << "setting connection to moduleid " << getPmId() << endl;
char buff[32];
snprintf(buff, sizeof(buff), "pm%u_WriteEngineServer", getPmId());
fServer = buff;
char buff[32];
snprintf(buff, sizeof(buff), "pm%u_WriteEngineServer", getPmId());
fServer = buff;
fClnt.reset(new MessageQueueClient(fServer));
fClnt.reset(new MessageQueueClient(fServer));
if(fOwner.getDebugLvl())
cout << "WEServer : " << fServer << " " << fClnt->addr2String() <<endl;
if (fOwner.getDebugLvl())
cout << "WEServer : " << fServer << " " << fClnt->addr2String() << endl;
try
{
if (fClnt->connect())
{
onConnect();
startClientThread();
}
else
{
throw runtime_error("Connection refused");
}
} catch (std::exception& ex)
{
cerr << "Could not connect to " << fServer << ": " << ex.what() << endl;
throw runtime_error("Problem in connecting to PM");
} catch (...)
{
cerr << "Could not connect to " << fServer << endl;
throw runtime_error("Problem in connecting to PM");
}
try
{
if (fClnt->connect())
{
onConnect();
startClientThread();
}
else
{
throw runtime_error("Connection refused");
}
}
catch (std::exception& ex)
{
cerr << "Could not connect to " << fServer << ": " << ex.what() << endl;
throw runtime_error("Problem in connecting to PM");
}
catch (...)
{
cerr << "Could not connect to " << fServer << endl;
throw runtime_error("Problem in connecting to PM");
}
}
@ -152,301 +154,321 @@ void WESplClient::startClientThread()
//------------------------------------------------------------------------------
void WESplClient::sendAndRecv()
{
while(fContinue)
{
try
{
// Send messages if out queue has something
send();
// Recv messages if there is something in socket or timeout
recv();
}
catch (runtime_error&)
{
//setCpiFailed(true); - done in onDisconnect() BUG
setConnected(false);
if(fOwner.getDebugLvl())
cout <<"Disconnect from PM - " << getPmId() << endl;
onDisconnect();
}
}
while (fContinue)
{
try
{
// Send messages if out queue has something
send();
// Recv messages if there is something in socket or timeout
recv();
}
catch (runtime_error&)
{
//setCpiFailed(true); - done in onDisconnect() BUG
setConnected(false);
if(this->fCpiFailed)
{
// NOTE : commented out to avoid sending rollback twice.
//fOwner.onCpimportFail(this->getPmId());
char aDefCon[16], aRedCol[16];
snprintf(aDefCon, sizeof(aDefCon), "\033[0m");
snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m");
if(fOwner.getDebugLvl())
cout << aRedCol << "Bulk load FAILED on PM "
<< getPmId()<< aDefCon << endl;
}
else if(this->fCpiPassed)
{
//if(fOwner.getDebugLvl())
//BUG 4195
char aDefCon[16], aGreenCol[16];
snprintf(aDefCon, sizeof(aDefCon), "\033[0m");
snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m");
if(fOwner.getDebugLvl())
cout << aGreenCol << "Bulk load Finished Successfully on PM "
<< getPmId()<< aDefCon << endl;
}
else if(!this->fCpiStarted)
{
if(fOwner.getDebugLvl())
cout << "Cpimport Failed to Start!!!"<< this->getPmId() << endl;
}
if (fOwner.getDebugLvl())
cout << "Disconnect from PM - " << getPmId() << endl;
onDisconnect();
}
}
if (this->fCpiFailed)
{
// NOTE : commented out to avoid sending rollback twice.
//fOwner.onCpimportFail(this->getPmId());
char aDefCon[16], aRedCol[16];
snprintf(aDefCon, sizeof(aDefCon), "\033[0m");
snprintf(aRedCol, sizeof(aRedCol), "\033[0;31m");
if (fOwner.getDebugLvl())
cout << aRedCol << "Bulk load FAILED on PM "
<< getPmId() << aDefCon << endl;
}
else if (this->fCpiPassed)
{
//if(fOwner.getDebugLvl())
//BUG 4195
char aDefCon[16], aGreenCol[16];
snprintf(aDefCon, sizeof(aDefCon), "\033[0m");
snprintf(aGreenCol, sizeof(aGreenCol), "\033[0;32m");
if (fOwner.getDebugLvl())
cout << aGreenCol << "Bulk load Finished Successfully on PM "
<< getPmId() << aDefCon << endl;
}
else if (!this->fCpiStarted)
{
if (fOwner.getDebugLvl())
cout << "Cpimport Failed to Start!!!" << this->getPmId() << endl;
}
}
//------------------------------------------------------------------------------
void WESplClient::send()
{
if ((!fSendQueue.empty())&&(getDataRqstCount()>0))
{
if(fOwner.getDebugLvl()>2)
cout << "DataRqstCnt [" << getPmId() << "] = "
<< getDataRqstCount() << endl;
mutex::scoped_lock aLock(fSentQMutex);
messageqcpp::SBS aSbs = fSendQueue.front();
fSendQueue.pop();
aLock.unlock();
int aLen = (*aSbs).length();
if (aLen > 0)
{
mutex::scoped_lock aLock(fWriteMutex);
setBytesTx(getBytesTx() + aLen);
try
{
if(isConnected())
fClnt->write(aSbs);
}
catch(...)
{
}
aLock.unlock();
}
if ((!fSendQueue.empty()) && (getDataRqstCount() > 0))
{
if (fOwner.getDebugLvl() > 2)
cout << "DataRqstCnt [" << getPmId() << "] = "
<< getDataRqstCount() << endl;
decDataRqstCount();
//decDbRootVar();
}
//setSendFlag(fOwner.check4Ack(fPmId));
mutex::scoped_lock aLock(fSentQMutex);
messageqcpp::SBS aSbs = fSendQueue.front();
fSendQueue.pop();
aLock.unlock();
int aLen = (*aSbs).length();
if (aLen > 0)
{
mutex::scoped_lock aLock(fWriteMutex);
setBytesTx(getBytesTx() + aLen);
try
{
if (isConnected())
fClnt->write(aSbs);
}
catch (...)
{
}
aLock.unlock();
}
decDataRqstCount();
//decDbRootVar();
}
//setSendFlag(fOwner.check4Ack(fPmId));
}
//------------------------------------------------------------------------------
void WESplClient::recv()
{
messageqcpp::SBS aSbs;
messageqcpp::SBS aSbs;
struct timespec rm_ts;
rm_ts.tv_sec = fRdSecTo; //0 when data sending otherwise 1- second
rm_ts.tv_nsec = 20000000; // 20 milliSec
bool isTimeOut = false;
int aLen = 0;
int aLen = 0;
try
{
if(isConnected())
aSbs = fClnt->read(&rm_ts, &isTimeOut);
if (isConnected())
aSbs = fClnt->read(&rm_ts, &isTimeOut);
}
catch (std::exception& ex)
{
setConnected(false);
cout << ex.what() <<endl;
cout << "fClnt read error on " << getPmId() << endl;
throw runtime_error("fClnt read error");
setConnected(false);
cout << ex.what() << endl;
cout << "fClnt read error on " << getPmId() << endl;
throw runtime_error("fClnt read error");
}
// - aSbs->length()>0 add to the sdh.fWesMsgQueue
try
{
if(aSbs)
aLen = aSbs->length();
}
catch(...)
{
aLen = 0;
}
if(aLen > 0)
{
setLastInTime(time(0)); //added back for BUG 4535 / BUG 4195
setBytesRcv( getBytesRcv()+ aLen);
fOwner.add2RespQueue(aSbs);
}
else if ((aLen <= 0) && (!isTimeOut)) //disconnect
{
cout <<"Disconnect from PM - " << getPmId() << " IP " << endl;
onDisconnect();
}
// - aSbs->length()>0 add to the sdh.fWesMsgQueue
try
{
if (aSbs)
aLen = aSbs->length();
}
catch (...)
{
aLen = 0;
}
if (aLen > 0)
{
setLastInTime(time(0)); //added back for BUG 4535 / BUG 4195
setBytesRcv( getBytesRcv() + aLen);
fOwner.add2RespQueue(aSbs);
}
else if ((aLen <= 0) && (!isTimeOut)) //disconnect
{
cout << "Disconnect from PM - " << getPmId() << " IP " << endl;
onDisconnect();
}
}
//------------------------------------------------------------------------------
void WESplClient::write(const messageqcpp::ByteStream& Msg)
{
setBytesTx(getBytesTx() + Msg.length());
try
{
if(Msg.length()>0)
fClnt->write(Msg);
}
catch(...)
{
//ignore it
}
setBytesTx(getBytesTx() + Msg.length());
try
{
if (Msg.length() > 0)
fClnt->write(Msg);
}
catch (...)
{
//ignore it
}
}
//------------------------------------------------------------------------------
void WESplClient::read(messageqcpp::SBS& Sbs)
{
// read from the WEServerMsgQueue
// if Key is needed give that constant here
// read from the WEServerMsgQueue
// if Key is needed give that constant here
}
//------------------------------------------------------------------------------
//TODO - We may need to make it much more efficient by incorporating file read
void WESplClient::add2SendQueue(const messageqcpp::SBS& Sbs)
{
this->fSendQueue.push(Sbs);
this->fSendQueue.push(Sbs);
}
void WESplClient::clearSendQueue()
{
mutex::scoped_lock aLock(fSentQMutex);
while(!fSendQueue.empty())
fSendQueue.pop();
aLock.unlock();
mutex::scoped_lock aLock(fSentQMutex);
while (!fSendQueue.empty())
fSendQueue.pop();
aLock.unlock();
}
int WESplClient::getSendQSize()
{
int aQSize=0;
mutex::scoped_lock aLock(fSentQMutex);
aQSize = fSendQueue.size();
aLock.unlock();
return aQSize;
int aQSize = 0;
mutex::scoped_lock aLock(fSentQMutex);
aQSize = fSendQueue.size();
aLock.unlock();
return aQSize;
}
//------------------------------------------------------------------------------
void WESplClient::printStats()
{
if(fOwner.getDebugLvl())
{
cout <<"\tPMid \t"<<getPmId()<<endl;
cout <<"\tTx Rows \t"<<getRowTx()<<endl;
//if(fOwner.getDebugLvl())
cout <<"\tTx Bytes \t"<<getBytesTx()<<endl;
//if(fOwner.getDebugLvl())
cout <<"\tRcv Bytes \t"<<getBytesRcv()<<endl;
cout <<"\tInserted/Read Rows "<<fRowsUploadInfo.fRowsInserted<<"/"
<<fRowsUploadInfo.fRowsRead<< endl;
if(fColOorVec.size()>0)
cout <<"\tCol Id\tColName\t\t\tout-of-range count" <<endl;
WEColOorVec::iterator aIt = fColOorVec.begin();
while(aIt != fColOorVec.end())
{
cout <<"\t"<<(*aIt).fColNum <<"\t"<<(*aIt).fColName <<"\t\t" << (*aIt).fNoOfOORs <<endl;
aIt++;
}
if(!fBadDataFile.empty())cout<<"\tBad Data Filename "<<fBadDataFile<<endl;
if(!fErrInfoFile.empty())cout<<"\tError Filename "<<fErrInfoFile<<endl;
cout <<"\t("<<getLastInTime()-getStartTime()<<"sec)"<<endl;
cout <<"\t"<<endl;
}
if (fOwner.getDebugLvl())
{
cout << "\tPMid \t" << getPmId() << endl;
cout << "\tTx Rows \t" << getRowTx() << endl;
//if(fOwner.getDebugLvl())
cout << "\tTx Bytes \t" << getBytesTx() << endl;
//if(fOwner.getDebugLvl())
cout << "\tRcv Bytes \t" << getBytesRcv() << endl;
cout << "\tInserted/Read Rows " << fRowsUploadInfo.fRowsInserted << "/"
<< fRowsUploadInfo.fRowsRead << endl;
if (fColOorVec.size() > 0)
cout << "\tCol Id\tColName\t\t\tout-of-range count" << endl;
WEColOorVec::iterator aIt = fColOorVec.begin();
while (aIt != fColOorVec.end())
{
cout << "\t" << (*aIt).fColNum << "\t" << (*aIt).fColName << "\t\t" << (*aIt).fNoOfOORs << endl;
aIt++;
}
if (!fBadDataFile.empty())cout << "\tBad Data Filename " << fBadDataFile << endl;
if (!fErrInfoFile.empty())cout << "\tError Filename " << fErrInfoFile << endl;
cout << "\t(" << getLastInTime() - getStartTime() << "sec)" << endl;
cout << "\t" << endl;
}
}
//------------------------------------------------------------------------------
void WESplClient::onConnect()
{
//TODO
// update all the flags on Connect.
// alert data can be send now
// do not allow to connect back again.
//TODO
// update all the flags on Connect.
// alert data can be send now
// do not allow to connect back again.
// when reconnect happens, reset the variables
setRollbackRslt(0);
setCleanupRslt(0);
setCpiPassed(false);
// when reconnect happens, reset the variables
setRollbackRslt(0);
setCleanupRslt(0);
setCpiPassed(false);
setCpiFailed(false);
setContinue(true);
setConnected(true);
ByteStream bsWrite;
bsWrite << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE;
try
{
this->write(bsWrite); // send the keep init keep alive
}
catch(...)
{
}
setConnected(true);
ByteStream bsWrite;
bsWrite << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE;
// need to send Alarm
fIpAddress = fClnt->addr2String();
try
{
this->write(bsWrite); // send the keep init keep alive
}
catch (...)
{
}
// need to send Alarm
fIpAddress = fClnt->addr2String();
}
//------------------------------------------------------------------------------
void WESplClient::onDisconnect()
{
//TODO
// - set fContinue to false - set the thread free
setContinue(false);
setConnected(false);
setRollbackRslt(-1);
setCleanupRslt(-1);
//TODO
// - set fContinue to false - set the thread free
setContinue(false);
setConnected(false);
setRollbackRslt(-1);
setCleanupRslt(-1);
if((!fCpiPassed)&&(!fCpiFailed)) //a hard disconnection
{
fOwner.onCpimportFail(fPmId);
fOwner.setDisconnectFailure(true);
}
if ((!fCpiPassed) && (!fCpiFailed)) //a hard disconnection
{
fOwner.onCpimportFail(fPmId);
fOwner.setDisconnectFailure(true);
}
// update all the flags of disconnect.
// alert on roll back
// do not allow to connect back again.
// update all the flags of disconnect.
// alert on roll back
// do not allow to connect back again.
try
{
// send alarm
ALARMManager alarmMgr;
//std::string alarmItem = sin_addr2String(fClnt->serv_addr().sin_addr);
std::string alarmItem = fClnt->addr2String();
alarmItem.append(" WriteEngineServer");
alarmMgr.sendAlarmReport(alarmItem.c_str(), oam::CONN_FAILURE, SET);
}
catch(...)
{
// just ignore it for time being.
}
try
{
// send alarm
ALARMManager alarmMgr;
//std::string alarmItem = sin_addr2String(fClnt->serv_addr().sin_addr);
std::string alarmItem = fClnt->addr2String();
alarmItem.append(" WriteEngineServer");
alarmMgr.sendAlarmReport(alarmItem.c_str(), oam::CONN_FAILURE, SET);
}
catch (...)
{
// just ignore it for time being.
}
}
//------------------------------------------------------------------------------
void WESplClient::setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted)
{
fRowsUploadInfo.fRowsRead = RowsRead;
fRowsUploadInfo.fRowsInserted = RowsInserted;
fRowsUploadInfo.fRowsRead = RowsRead;
fRowsUploadInfo.fRowsInserted = RowsInserted;
}
//------------------------------------------------------------------------------
void WESplClient::add2ColOutOfRangeInfo(int ColNum,
CalpontSystemCatalog::ColDataType ColType,
void WESplClient::add2ColOutOfRangeInfo(int ColNum,
CalpontSystemCatalog::ColDataType ColType,
std::string& ColName, int NoOfOors)
{
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = ColNum;
aColOorInfo.fColType = ColType;
aColOorInfo.fColName = ColName;
aColOorInfo.fNoOfOORs = NoOfOors;
fColOorVec.push_back(aColOorInfo);
WEColOORInfo aColOorInfo;
aColOorInfo.fColNum = ColNum;
aColOorInfo.fColType = ColType;
aColOorInfo.fColName = ColName;
aColOorInfo.fNoOfOORs = NoOfOors;
fColOorVec.push_back(aColOorInfo);
}
//------------------------------------------------------------------------------
void WESplClient::setBadDataFile(const std::string& BadDataFile)
{
fBadDataFile = BadDataFile;
fBadDataFile = BadDataFile;
}
//------------------------------------------------------------------------------
@ -454,7 +476,7 @@ void WESplClient::setBadDataFile(const std::string& BadDataFile)
void WESplClient::setErrInfoFile(const std::string& ErrInfoFile)
{
fErrInfoFile = ErrInfoFile;
fErrInfoFile = ErrInfoFile;
}
//------------------------------------------------------------------------------

666
writeengine/splitter/we_splclient.h
View File

@ -14,280 +14,392 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splclient.h
*
* Created on: Oct 20, 2011
* Author: bpaul
*/
#ifndef WE_SPLCLIENT_H_
#define WE_SPLCLIENT_H_
#include "threadsafequeue.h"
#include "resourcemanager.h"
#include "we_messages.h"
#include "calpontsystemcatalog.h"
using namespace execplan;
namespace WriteEngine
{
class WESplClient; //forward decleration
// Structure for holding the Out of Range data from the BRMReport
// This class is also used by we_sdhandler to hold the agregate info.
class WEColOORInfo // Column Out-Of-Range Info
{
public:
WEColOORInfo():fColNum(0),fColType(CalpontSystemCatalog::INT), fNoOfOORs(0){}
~WEColOORInfo(){}
public:
int fColNum;
CalpontSystemCatalog::ColDataType fColType;
std::string fColName;
int fNoOfOORs;
};
typedef std::vector<WEColOORInfo> WEColOorVec;
//------------------------------------------------------------------------------
class WESdHandlerException: public std::exception
{
public:
std::string fWhat;
WESdHandlerException(std::string& What) throw() { fWhat = What; }
virtual ~WESdHandlerException() throw() {}
virtual const char* what() const throw()
{
return fWhat.c_str();
}
};
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
class WESplClientRunner
{
public:
WESplClientRunner(WESplClient& Sc): fOwner(Sc){ /* ctor */ }
virtual ~WESplClientRunner(){/* dtor */}
void operator()();
public:
WESplClient& fOwner;
};
//------------------------------------------------------------------------------
class WESplClient
{
public:
WESplClient(WESDHandler& Sdh, int PmId);
virtual ~WESplClient();
void setup();
void startClientThread();
void sendAndRecv();
void send();
void recv();
void write(const messageqcpp::ByteStream& Msg);
void read(messageqcpp::SBS& Sbs);
void add2SendQueue(const messageqcpp::SBS& Sbs);
void clearSendQueue();
int getSendQSize();
void printStats();
void onConnect();
void onDisconnect();
unsigned int getRowTx() const { return fRowTx; }
uint32_t getBytesRcv() const { return fBytesRcv; }
uint32_t getBytesTx()
{
boost::mutex::scoped_lock aLock(fTxMutex);
return fBytesTx;
}
boost::thread* getFpThread() const { return fpThread; }
time_t getLastInTime()
{
boost::mutex::scoped_lock aLock(fLastInMutex);
return(fLastInTime>0)?fLastInTime:fStartTime; //BUG 4309
}
time_t getStartTime() const { return fStartTime; }
time_t getElapsedTime() { return (getLastInTime() - getStartTime()); }
bool isCpiStarted() const { return fCpiStarted; }
bool isCpiPassed() const { return fCpiPassed; }
bool isCpiFailed() const { return fCpiFailed; }
bool isBrmRptRcvd() const { return fBrmRptRcvd; }
int getRollbackRslt() const { return fRollbackRslt; }
int getCleanupRslt() const { return fCleanupRslt; }
bool getSendFlag() const { return fSend; }
unsigned int getPmId() const { return fPmId; }
unsigned int getDbRootCnt() const { return fDbrCnt; }
unsigned int getDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
return fDbrVar;
}
int getDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
return fDataRqstCnt;
}
long getRdSecTo() const { return fRdSecTo; }
bool isConnected() const { return fConnected; }
bool isContinue() const { return fContinue; }
const std::string& getServer() const { return fServer; }
const std::string& getIpAddress() const { return fIpAddress; }
void setBytesRcv(uint32_t BytesRcv) { fBytesRcv = BytesRcv; }
void setBytesTx(uint32_t BytesTx)
{
boost::mutex::scoped_lock aLock(fTxMutex);
BytesTx = BytesTx;
aLock.unlock();
}
void updateBytesTx(uint32_t fBytes)
{
boost::mutex::scoped_lock aLock(fTxMutex);
fBytesTx += fBytes;
aLock.unlock();
}
void setConnected(bool Connected) { fConnected = Connected; }
void setContinue(bool Continue) { fContinue = Continue; }
void setFpThread(boost::thread* pThread) { fpThread = pThread; }
void setLastInTime(time_t LastInTime) { fLastInTime = LastInTime; }
void setStartTime(time_t StartTime)
{
boost::mutex::scoped_lock aLock(fLastInMutex);
fStartTime = StartTime;
aLock.lock();
}
void setSendFlag(bool Send) { fSend = Send; }
void setCpiStarted(bool Start) { fCpiStarted = Start; }
void setCpiPassed(bool Pass)
{
setLastInTime(time(0));
fCpiPassed = Pass;
}
void setCpiFailed(bool Fail)
{
setLastInTime(time(0));
fCpiFailed = Fail;
fRowsUploadInfo.fRowsRead = 0;
fRowsUploadInfo.fRowsInserted = 0;
}
void setBrmRptRcvd(bool Rcvd) { fBrmRptRcvd = Rcvd; }
void setRollbackRslt(int Rslt) { fRollbackRslt = Rslt; }
void setCleanupRslt(int Rslt) { fCleanupRslt = Rslt; }
void setPmId(unsigned int PmId) { fPmId = PmId; }
void setDbRootCnt(unsigned int DbrCnt) { fDbrCnt = DbrCnt; }
void resetDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
fDbrVar = fDbrCnt;
aLock.unlock();
}
void decDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
if(fDbrVar>0) --fDbrVar;
aLock.unlock();
}
void setRdSecTo(long RdSecTo)
{
fRdSecTo = RdSecTo;
}
void setDataRqstCount(int DataRqstCnt)
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
fDataRqstCnt = DataRqstCnt;
aLock.unlock();
}
void decDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
if(fDataRqstCnt>0) --fDataRqstCnt;
aLock.unlock();
}
void incDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
++fDataRqstCnt;
aLock.unlock();
}
void setServer(const std::string& Server) { fServer = Server; }
void setIpAddress(const std::string& IpAddr) { fIpAddress = IpAddr; }
void updateRowTx(unsigned int aCnt) { fRowTx += aCnt; }
void resetRowTx() { fRowTx = 0; }
private:
bool fContinue;
bool fConnected;
unsigned int fPmId;
unsigned int fDbrCnt;
unsigned int fDbrVar; // Var to keep track next PM to send.
int fDataRqstCnt; // Data request count
long fRdSecTo; // read timeout sec
unsigned int fRowTx; // No. Of Rows Transmitted
uint32_t fBytesTx;
uint32_t fBytesRcv;
time_t fLastInTime;
time_t fStartTime;
bool fSend;
bool fCpiStarted;
bool fCpiPassed;
bool fCpiFailed;
bool fBrmRptRcvd;
int fRollbackRslt;
int fCleanupRslt;
boost::mutex fTxMutex; //mutex for TxBytes
boost::mutex fDataRqstMutex;
boost::mutex fWriteMutex;
boost::mutex fSentQMutex;
boost::mutex fLastInMutex;
typedef std::queue<messageqcpp::SBS> WESendQueue;
WESendQueue fSendQueue;
std::string fServer;
std::string fIpAddress;
boost::shared_ptr<messageqcpp::MessageQueueClient> fClnt;
boost::thread *fpThread;
WESDHandler& fOwner;
class WERowsUploadInfo
{
public:
WERowsUploadInfo():fRowsRead(0),fRowsInserted(0){}
~WERowsUploadInfo(){}
public:
int64_t fRowsRead;
int64_t fRowsInserted;
};
WERowsUploadInfo fRowsUploadInfo;
WEColOorVec fColOorVec;
std::string fBadDataFile;
std::string fErrInfoFile;
void setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted);
void add2ColOutOfRangeInfo(int ColNum,
CalpontSystemCatalog::ColDataType ColType,
std::string& ColName, int NoOfOors);
void setBadDataFile(const std::string& BadDataFile);
void setErrInfoFile(const std::string& ErrInfoFile);
friend class WESDHandler;
};
//------------------------------------------------------------------------------
} /* namespace WriteEngine */
#endif /* WE_SPLCLIENT_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splclient.h
*
* Created on: Oct 20, 2011
* Author: bpaul
*/
#ifndef WE_SPLCLIENT_H_
#define WE_SPLCLIENT_H_
#include "threadsafequeue.h"
#include "resourcemanager.h"
#include "we_messages.h"
#include "calpontsystemcatalog.h"
using namespace execplan;
namespace WriteEngine
{
class WESplClient; //forward decleration
// Structure for holding the Out of Range data from the BRMReport
// This class is also used by we_sdhandler to hold the agregate info.
class WEColOORInfo // Column Out-Of-Range Info
{
public:
WEColOORInfo(): fColNum(0), fColType(CalpontSystemCatalog::INT), fNoOfOORs(0) {}
~WEColOORInfo() {}
public:
int fColNum;
CalpontSystemCatalog::ColDataType fColType;
std::string fColName;
int fNoOfOORs;
};
typedef std::vector<WEColOORInfo> WEColOorVec;
//------------------------------------------------------------------------------
class WESdHandlerException: public std::exception
{
public:
std::string fWhat;
WESdHandlerException(std::string& What) throw()
{
fWhat = What;
}
virtual ~WESdHandlerException() throw() {}
virtual const char* what() const throw()
{
return fWhat.c_str();
}
};
//------------------------------------------------------------------------------
//------------------------------------------------------------------------------
class WESplClientRunner
{
public:
WESplClientRunner(WESplClient& Sc): fOwner(Sc) { /* ctor */ }
virtual ~WESplClientRunner() {/* dtor */}
void operator()();
public:
WESplClient& fOwner;
};
//------------------------------------------------------------------------------
class WESplClient
{
public:
WESplClient(WESDHandler& Sdh, int PmId);
virtual ~WESplClient();
void setup();
void startClientThread();
void sendAndRecv();
void send();
void recv();
void write(const messageqcpp::ByteStream& Msg);
void read(messageqcpp::SBS& Sbs);
void add2SendQueue(const messageqcpp::SBS& Sbs);
void clearSendQueue();
int getSendQSize();
void printStats();
void onConnect();
void onDisconnect();
unsigned int getRowTx() const
{
return fRowTx;
}
uint32_t getBytesRcv() const
{
return fBytesRcv;
}
uint32_t getBytesTx()
{
boost::mutex::scoped_lock aLock(fTxMutex);
return fBytesTx;
}
boost::thread* getFpThread() const
{
return fpThread;
}
time_t getLastInTime()
{
boost::mutex::scoped_lock aLock(fLastInMutex);
return (fLastInTime > 0) ? fLastInTime : fStartTime; //BUG 4309
}
time_t getStartTime() const
{
return fStartTime;
}
time_t getElapsedTime()
{
return (getLastInTime() - getStartTime());
}
bool isCpiStarted() const
{
return fCpiStarted;
}
bool isCpiPassed() const
{
return fCpiPassed;
}
bool isCpiFailed() const
{
return fCpiFailed;
}
bool isBrmRptRcvd() const
{
return fBrmRptRcvd;
}
int getRollbackRslt() const
{
return fRollbackRslt;
}
int getCleanupRslt() const
{
return fCleanupRslt;
}
bool getSendFlag() const
{
return fSend;
}
unsigned int getPmId() const
{
return fPmId;
}
unsigned int getDbRootCnt() const
{
return fDbrCnt;
}
unsigned int getDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
return fDbrVar;
}
int getDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
return fDataRqstCnt;
}
long getRdSecTo() const
{
return fRdSecTo;
}
bool isConnected() const
{
return fConnected;
}
bool isContinue() const
{
return fContinue;
}
const std::string& getServer() const
{
return fServer;
}
const std::string& getIpAddress() const
{
return fIpAddress;
}
void setBytesRcv(uint32_t BytesRcv)
{
fBytesRcv = BytesRcv;
}
void setBytesTx(uint32_t BytesTx)
{
boost::mutex::scoped_lock aLock(fTxMutex);
BytesTx = BytesTx;
aLock.unlock();
}
void updateBytesTx(uint32_t fBytes)
{
boost::mutex::scoped_lock aLock(fTxMutex);
fBytesTx += fBytes;
aLock.unlock();
}
void setConnected(bool Connected)
{
fConnected = Connected;
}
void setContinue(bool Continue)
{
fContinue = Continue;
}
void setFpThread(boost::thread* pThread)
{
fpThread = pThread;
}
void setLastInTime(time_t LastInTime)
{
fLastInTime = LastInTime;
}
void setStartTime(time_t StartTime)
{
boost::mutex::scoped_lock aLock(fLastInMutex);
fStartTime = StartTime;
aLock.lock();
}
void setSendFlag(bool Send)
{
fSend = Send;
}
void setCpiStarted(bool Start)
{
fCpiStarted = Start;
}
void setCpiPassed(bool Pass)
{
setLastInTime(time(0));
fCpiPassed = Pass;
}
void setCpiFailed(bool Fail)
{
setLastInTime(time(0));
fCpiFailed = Fail;
fRowsUploadInfo.fRowsRead = 0;
fRowsUploadInfo.fRowsInserted = 0;
}
void setBrmRptRcvd(bool Rcvd)
{
fBrmRptRcvd = Rcvd;
}
void setRollbackRslt(int Rslt)
{
fRollbackRslt = Rslt;
}
void setCleanupRslt(int Rslt)
{
fCleanupRslt = Rslt;
}
void setPmId(unsigned int PmId)
{
fPmId = PmId;
}
void setDbRootCnt(unsigned int DbrCnt)
{
fDbrCnt = DbrCnt;
}
void resetDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
fDbrVar = fDbrCnt;
aLock.unlock();
}
void decDbRootVar()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
if (fDbrVar > 0) --fDbrVar;
aLock.unlock();
}
void setRdSecTo(long RdSecTo)
{
fRdSecTo = RdSecTo;
}
void setDataRqstCount(int DataRqstCnt)
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
fDataRqstCnt = DataRqstCnt;
aLock.unlock();
}
void decDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
if (fDataRqstCnt > 0) --fDataRqstCnt;
aLock.unlock();
}
void incDataRqstCount()
{
boost::mutex::scoped_lock aLock(fDataRqstMutex);
++fDataRqstCnt;
aLock.unlock();
}
void setServer(const std::string& Server)
{
fServer = Server;
}
void setIpAddress(const std::string& IpAddr)
{
fIpAddress = IpAddr;
}
void updateRowTx(unsigned int aCnt)
{
fRowTx += aCnt;
}
void resetRowTx()
{
fRowTx = 0;
}
private:
bool fContinue;
bool fConnected;
unsigned int fPmId;
unsigned int fDbrCnt;
unsigned int fDbrVar; // Var to keep track next PM to send.
int fDataRqstCnt; // Data request count
long fRdSecTo; // read timeout sec
unsigned int fRowTx; // No. Of Rows Transmitted
uint32_t fBytesTx;
uint32_t fBytesRcv;
time_t fLastInTime;
time_t fStartTime;
bool fSend;
bool fCpiStarted;
bool fCpiPassed;
bool fCpiFailed;
bool fBrmRptRcvd;
int fRollbackRslt;
int fCleanupRslt;
boost::mutex fTxMutex; //mutex for TxBytes
boost::mutex fDataRqstMutex;
boost::mutex fWriteMutex;
boost::mutex fSentQMutex;
boost::mutex fLastInMutex;
typedef std::queue<messageqcpp::SBS> WESendQueue;
WESendQueue fSendQueue;
std::string fServer;
std::string fIpAddress;
boost::shared_ptr<messageqcpp::MessageQueueClient> fClnt;
boost::thread* fpThread;
WESDHandler& fOwner;
class WERowsUploadInfo
{
public:
WERowsUploadInfo(): fRowsRead(0), fRowsInserted(0) {}
~WERowsUploadInfo() {}
public:
int64_t fRowsRead;
int64_t fRowsInserted;
};
WERowsUploadInfo fRowsUploadInfo;
WEColOorVec fColOorVec;
std::string fBadDataFile;
std::string fErrInfoFile;
void setRowsUploadInfo(int64_t RowsRead, int64_t RowsInserted);
void add2ColOutOfRangeInfo(int ColNum,
CalpontSystemCatalog::ColDataType ColType,
std::string& ColName, int NoOfOors);
void setBadDataFile(const std::string& BadDataFile);
void setErrInfoFile(const std::string& ErrInfoFile);
friend class WESDHandler;
};
//------------------------------------------------------------------------------
} /* namespace WriteEngine */
#endif /* WE_SPLCLIENT_H_ */

838
writeengine/splitter/we_splitterapp.cpp Executable file → Normal file
View File

@ -46,7 +46,7 @@ using namespace batchloader;
#include "installdir.h"
static int SPLTR_EXIT_STATUS=0;
static int SPLTR_EXIT_STATUS = 0;
namespace WriteEngine
@ -61,83 +61,89 @@ SimpleSysLog* WESplitterApp::fpSysLog = 0;
//WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) :
// fCmdArgs(CmdArgs), fDh(*this), fpSysLog(0)
WESplitterApp::WESplitterApp(WECmdArgs& CmdArgs) :
fCmdArgs(CmdArgs), fDh(*this)
fCmdArgs(CmdArgs), fDh(*this)
{
fpSysLog = SimpleSysLog::instance();
fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SPLIT));
setupSignalHandlers();
std::string err;
fDh.setDebugLvl(fCmdArgs.getDebugLvl());
fpSysLog = SimpleSysLog::instance();
fpSysLog->setLoggingID(logging::LoggingID(SUBSYSTEM_ID_WE_SPLIT));
setupSignalHandlers();
std::string err;
fDh.setDebugLvl(fCmdArgs.getDebugLvl());
fDh.check4CpiInvokeMode();
fDh.check4CpiInvokeMode();
fCmdArgs.checkForCornerCases();
fCmdArgs.checkForCornerCases();
if (fCmdArgs.isCpimportInvokeMode())
{
try
{
invokeCpimport();
}
catch(std::exception& ex)
{
cout << "Invoking Mode 3" << endl;
cout << ex.what() << endl;
SPLTR_EXIT_STATUS=1;
exit(SPLTR_EXIT_STATUS);
}
exit(SPLTR_EXIT_STATUS);
}
else
{
if(fCmdArgs.isHelpMode()) fCmdArgs.usage();
if (fCmdArgs.isCpimportInvokeMode())
{
try
{
invokeCpimport();
}
catch (std::exception& ex)
{
cout << "Invoking Mode 3" << endl;
cout << ex.what() << endl;
SPLTR_EXIT_STATUS = 1;
exit(SPLTR_EXIT_STATUS);
}
if(fCmdArgs.getMultiTableCount() <= 1)
{
try
{
fDh.setup();
}
catch (std::exception& ex)
{
//err = string("Error in constructing WESplitterApp") + ex.what();
err = ex.what(); //cleaning up for BUG 4298
logging::Message::Args errMsgArgs;
errMsgArgs.add(err);
fpSysLog->logMsg(errMsgArgs,logging::LOG_TYPE_ERROR,logging::M0000);
if (!fCmdArgs.getConsoleOutput())
{
ofstream dmlFile;
ostringstream oss;
oss << "/tmp/" <<fDh.getTableOID() << ".txt";
dmlFile.open(oss.str().c_str());
if (dmlFile.is_open())
{
dmlFile << err;
dmlFile << endl;
dmlFile.close();
}
SPLTR_EXIT_STATUS=2;
}
else
SPLTR_EXIT_STATUS=1;
//cout << err << endl;
fDh.fLog.logMsg( err, MSGLVL_ERROR );
fContinue = false;
//throw runtime_error(err); BUG 4298
}
}
}
exit(SPLTR_EXIT_STATUS);
}
else
{
if (fCmdArgs.isHelpMode()) fCmdArgs.usage();
if (fCmdArgs.getMultiTableCount() <= 1)
{
try
{
fDh.setup();
}
catch (std::exception& ex)
{
//err = string("Error in constructing WESplitterApp") + ex.what();
err = ex.what(); //cleaning up for BUG 4298
logging::Message::Args errMsgArgs;
errMsgArgs.add(err);
fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000);
if (!fCmdArgs.getConsoleOutput())
{
ofstream dmlFile;
ostringstream oss;
oss << "/tmp/" << fDh.getTableOID() << ".txt";
dmlFile.open(oss.str().c_str());
if (dmlFile.is_open())
{
dmlFile << err;
dmlFile << endl;
dmlFile.close();
}
SPLTR_EXIT_STATUS = 2;
}
else
SPLTR_EXIT_STATUS = 1;
//cout << err << endl;
fDh.fLog.logMsg( err, MSGLVL_ERROR );
fContinue = false;
//throw runtime_error(err); BUG 4298
}
}
}
}
WESplitterApp::~WESplitterApp()
{
//fDh.shutdown();
usleep(1000); //1 millisec just checking
//fDh.shutdown();
usleep(1000); //1 millisec just checking
std::string aStr = "Calling WESplitterApp Destructor\n";
if(fDh.getDebugLvl()) cout << aStr << endl;
std::string aStr = "Calling WESplitterApp Destructor\n";
if (fDh.getDebugLvl()) cout << aStr << endl;
}
@ -148,55 +154,60 @@ WESplitterApp::~WESplitterApp()
void WESplitterApp::setupSignalHandlers()
{
#ifdef _MSC_VER
//FIXME
//FIXME
#else
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = WESplitterApp::onSigInterrupt;
sigaction(SIGINT, &sa, 0);
sa.sa_handler = WESplitterApp::onSigTerminate;
sigaction(SIGTERM, &sa, 0);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
sa.sa_handler = WESplitterApp::onSigHup;
sigaction(SIGPIPE, &sa, 0);
sa.sa_handler = WESplitterApp::onSigInterrupt;
sigaction(SIGUSR1, &sa, 0);
/*
signal(SIGPIPE, SIG_IGN);
signal(SIGINT, WESplitterApp::onSigInterrupt);
signal(SIGTERM, WESplitterApp::onSigTerminate);
signal(SIGHUP, WESplitterApp::onSigHup); */
sigaction(SIGINT, &sa, 0);
sa.sa_handler = WESplitterApp::onSigTerminate;
sigaction(SIGTERM, &sa, 0);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, 0);
sa.sa_handler = WESplitterApp::onSigHup;
sigaction(SIGPIPE, &sa, 0);
sa.sa_handler = WESplitterApp::onSigInterrupt;
sigaction(SIGUSR1, &sa, 0);
/*
signal(SIGPIPE, SIG_IGN);
signal(SIGINT, WESplitterApp::onSigInterrupt);
signal(SIGTERM, WESplitterApp::onSigTerminate);
signal(SIGHUP, WESplitterApp::onSigHup); */
#endif
}
//------------------------------------------------------------------------------
// handles on signal Terminate
// handles on signal Terminate
//------------------------------------------------------------------------------
void WESplitterApp::onSigTerminate(int aInt)
{
cout << "onSigTerminate received signal " << aInt << endl;
if(15 == aInt)
{
fSignaled = true;
}
fContinue = false; //force to call destructor
if(aInt == 1) SPLTR_EXIT_STATUS = 1;
cout << "onSigTerminate received signal " << aInt << endl;
if (15 == aInt)
{
fSignaled = true;
}
fContinue = false; //force to call destructor
if (aInt == 1) SPLTR_EXIT_STATUS = 1;
}
//------------------------------------------------------------------------------
// handles on signal Interrupt
// handles on signal Interrupt
//------------------------------------------------------------------------------
void WESplitterApp::onSigInterrupt(int aInt)
{
//cout << "onSigInterrupt received signal " << aInt << endl;
if((2 == aInt) || (10 == aInt))
{
fSignaled = true;
//cout << "ctrl-c received" << endl;
}
fContinue = false; //force to call destructor
if(aInt == 1) SPLTR_EXIT_STATUS = 1;
//cout << "onSigInterrupt received signal " << aInt << endl;
if ((2 == aInt) || (10 == aInt))
{
fSignaled = true;
//cout << "ctrl-c received" << endl;
}
fContinue = false; //force to call destructor
if (aInt == 1) SPLTR_EXIT_STATUS = 1;
}
//------------------------------------------------------------------------------
@ -204,259 +215,286 @@ void WESplitterApp::onSigInterrupt(int aInt)
//------------------------------------------------------------------------------
void WESplitterApp::onSigHup(int aInt)
{
fSigHup = true;
fContinue = false;
std::string aStr = "Interrupt received...Program Exiting...";
cout << aStr << endl;
if(aInt == 1) SPLTR_EXIT_STATUS = 1;
logging::Message::Args errMsgArgs;
errMsgArgs.add(aStr);
fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000);
fSigHup = true;
fContinue = false;
std::string aStr = "Interrupt received...Program Exiting...";
cout << aStr << endl;
exit(SPLTR_EXIT_STATUS); //BUG 4534 - exit w/o writing to log
if (aInt == 1) SPLTR_EXIT_STATUS = 1;
logging::Message::Args errMsgArgs;
errMsgArgs.add(aStr);
fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0000);
exit(SPLTR_EXIT_STATUS); //BUG 4534 - exit w/o writing to log
}
//------------------------------------------------------------------------------
// Process messages on Main thread
// Process messages on Main thread
//------------------------------------------------------------------------------
void WESplitterApp::processMessages()
{
boost::uuids::uuid u=boost::uuids::random_generator()();
boost::uuids::uuid u = boost::uuids::random_generator()();
fCmdArgs.setJobUUID(u);
messageqcpp::ByteStream aBs;
unsigned int aRollCount = 0;
messageqcpp::ByteStream aBs;
unsigned int aRollCount = 0;
if(fDh.getDebugLvl())
cout << "Inside WESplitterApp::processMessages() "
<< "Mode = " << fCmdArgs.getMode() << endl;
if (fDh.getDebugLvl())
cout << "Inside WESplitterApp::processMessages() "
<< "Mode = " << fCmdArgs.getMode() << endl;
//TODO - handle all the messages here
if (fCmdArgs.getMode() == 2)
{
try
{
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
//TODO - handle all the messages here
if (fCmdArgs.getMode() == 2)
{
try
{
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
std::string aJobId = fCmdArgs.getJobId();
if((aJobId.length()>0)&&(!fCmdArgs.isJobLogOnly())) // Export jobFile NOW
{
std::string aJobFileName = fCmdArgs.getJobFileName();
fDh.exportJobFile(aJobId, aJobFileName );
}
std::string aJobId = fCmdArgs.getJobId();
aBs.restart();
std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine();
fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 );
if(fDh.getDebugLvl())
cout << "CPImport cmd line - " << aCpImpCmd << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_CMDLINEARGS;
aBs << aCpImpCmd;
fDh.send2Pm(aBs);
if ((aJobId.length() > 0) && (!fCmdArgs.isJobLogOnly())) // Export jobFile NOW
{
std::string aJobFileName = fCmdArgs.getJobFileName();
fDh.exportJobFile(aJobId, aJobFileName );
}
aBs.restart();
std::string aBrmRpt = fCmdArgs.getBrmRptFileName();
if(fDh.getDebugLvl())
cout << "BrmReport FileName - " << aBrmRpt << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_BRMRPT;
aBs << aBrmRpt;
fDh.send2Pm(aBs);
aBs.restart();
std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine();
fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 );
}
catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS=1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
else if (fCmdArgs.getMode() == 1)
{
try
{
// In this mode we ignore almost all cmd lines args which
// are usually send to cpimport
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
if (fDh.getDebugLvl())
cout << "CPImport cmd line - " << aCpImpCmd << endl;
std::string aJobId = fCmdArgs.getJobId();
if(fDh.getDebugLvl()) cout<<"ProcessMsgs aJobId "<<aJobId<<endl;
if((aJobId.length()>0)&&(!fCmdArgs.isJobLogOnly())) // Export jobFile NOW
{
std::string aJobFileName = fCmdArgs.getJobFileName();
if(fDh.getDebugLvl()) cout<<"ProcessMsgs Calling exportJobFile "<<endl;
fDh.exportJobFile(aJobId, aJobFileName );
if(fDh.getDebugLvl()) cout<<"ProcessMsgs Calling exportJobFile "<<endl;
}
aBs << (ByteStream::byte) WE_CLT_SRV_CMDLINEARGS;
aBs << aCpImpCmd;
fDh.send2Pm(aBs);
aBs.restart();
std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine();
fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 );
if(fDh.getDebugLvl())
cout << "CPImport cmd line - " << aCpImpCmd << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_CMDLINEARGS;
aBs << aCpImpCmd;
fDh.send2Pm(aBs);
aBs.restart();
std::string aBrmRpt = fCmdArgs.getBrmRptFileName();
aBs.restart();
std::string aBrmRpt = fCmdArgs.getBrmRptFileName();
if(fDh.getDebugLvl())
cout << "BrmReport FileName - " << aBrmRpt << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_BRMRPT;
aBs << aBrmRpt;
fDh.send2Pm(aBs);
if (fDh.getDebugLvl())
cout << "BrmReport FileName - " << aBrmRpt << endl;
} catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS=1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
else if (fCmdArgs.getMode() == 0)
{
try
{
// In this mode we ignore almost all cmd lines args which
// are usually send to cpimport
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
aBs << (ByteStream::byte) WE_CLT_SRV_BRMRPT;
aBs << aBrmRpt;
fDh.send2Pm(aBs);
aBs.restart();
std::string aCpImpFileName = fCmdArgs.getPmFile();
if(aCpImpFileName.length()==0)
{
fCmdArgs.setPmFile(fCmdArgs.getLocFile());
aCpImpFileName = fCmdArgs.getPmFile();
if((aCpImpFileName.length()==0)||(aCpImpFileName == "STDIN"))
{
throw (runtime_error("PM Remote filename not specified"));
}
}
if(fDh.getDebugLvl())
cout << "CPImport FileName - " << aCpImpFileName << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_IMPFILENAME;
aBs << aCpImpFileName;
fDh.send2Pm(aBs);
}
catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS=1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
}
catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS = 1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
else if (fCmdArgs.getMode() == 1)
{
try
{
// In this mode we ignore almost all cmd lines args which
// are usually send to cpimport
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
int aNoSec=2;
bool bRollback = false;
bool bForce = false;
int iShutdown;
// TODO - this is for just time being....
// we need to process message of main thread here..
// here we need to cont check the status of different things
while (fContinue)
{
++aRollCount;
usleep(1000000);
// Check to see if someone has ordered a shutdown with rollback or force.
iShutdown = fDh.fDbrm.getSystemShutdownPending(bRollback, bForce);
if (iShutdown >= 0)
{
if (bRollback)
{
if (iShutdown > 0) // Means a shutdown, stop or restart
{
cout << "System stop has been ordered. Rollback" << endl;
}
else
{
cout << "Database writes have been suspended. Rollback" << endl;
}
fSignaled = true;
fContinue = false;
}
else
if (bForce)
{
//BUG 5012 - added to avoid rollback
fContinue = false;
ostringstream oss;
oss << "Table "<<fCmdArgs.getSchemaName()<<".";
oss << fCmdArgs.getTableName() << ": (OID-";
oss << fDh.getTableOID() << ") was NOT successfully loaded.";
cout << oss.str() << endl;
logging::Message::Args errMsgArgs;
//BUG 4152
errMsgArgs.add(fCmdArgs.getSchemaName());
errMsgArgs.add(fCmdArgs.getTableName());
errMsgArgs.add(fDh.getTableOID());
std::string aStr = "Immediate system stop has been ordered, rollback deferred";
cout << aStr << endl;
SPLTR_EXIT_STATUS = 1;
errMsgArgs.add(aStr);
fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0096);
exit(SPLTR_EXIT_STATUS);
//BUG 5012 - commented out to avoid rollback
//cout << "Immediate system stop has been ordered. No rollback" << endl;
//fSignaled = true;
//fContinue = false;
}
}
std::string aJobId = fCmdArgs.getJobId();
// Send out a heartbeat to the WriteEnginServers every 10 seconds
if ((0 == (aRollCount % aNoSec)) && (!fSignaled)) // Debugging - every 10 seconds
{
if(aNoSec<10) aNoSec++; //progressively go up to 10Sec interval
aBs.restart();
aBs << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE;
mutex::scoped_lock aLock(fDh.fSendMutex);
fDh.send2Pm(aBs);
aLock.unlock();
if (fDh.getDebugLvl()) cout << "ProcessMsgs aJobId " << aJobId << endl;
if ((aJobId.length() > 0) && (!fCmdArgs.isJobLogOnly())) // Export jobFile NOW
{
std::string aJobFileName = fCmdArgs.getJobFileName();
if (fDh.getDebugLvl()) cout << "ProcessMsgs Calling exportJobFile " << endl;
fDh.exportJobFile(aJobId, aJobFileName );
if (fDh.getDebugLvl()) cout << "ProcessMsgs Calling exportJobFile " << endl;
}
aBs.restart();
std::string aCpImpCmd = fCmdArgs.getCpImportCmdLine();
fDh.fLog.logMsg( aCpImpCmd, MSGLVL_INFO2 );
if (fDh.getDebugLvl())
cout << "CPImport cmd line - " << aCpImpCmd << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_CMDLINEARGS;
aBs << aCpImpCmd;
fDh.send2Pm(aBs);
aBs.restart();
std::string aBrmRpt = fCmdArgs.getBrmRptFileName();
if (fDh.getDebugLvl())
cout << "BrmReport FileName - " << aBrmRpt << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_BRMRPT;
aBs << aBrmRpt;
fDh.send2Pm(aBs);
}
catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS = 1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
else if (fCmdArgs.getMode() == 0)
{
try
{
// In this mode we ignore almost all cmd lines args which
// are usually send to cpimport
aBs << (ByteStream::byte) WE_CLT_SRV_MODE;
aBs << (ByteStream::quadbyte) fCmdArgs.getMode();
fDh.send2Pm(aBs);
aBs.restart();
std::string aCpImpFileName = fCmdArgs.getPmFile();
if (aCpImpFileName.length() == 0)
{
fCmdArgs.setPmFile(fCmdArgs.getLocFile());
aCpImpFileName = fCmdArgs.getPmFile();
if ((aCpImpFileName.length() == 0) || (aCpImpFileName == "STDIN"))
{
throw (runtime_error("PM Remote filename not specified"));
}
}
if (fDh.getDebugLvl())
cout << "CPImport FileName - " << aCpImpFileName << endl;
aBs << (ByteStream::byte) WE_CLT_SRV_IMPFILENAME;
aBs << aCpImpFileName;
fDh.send2Pm(aBs);
}
catch (std::exception& exp)
{
//cout << exp.what() << endl;
SPLTR_EXIT_STATUS = 1;
//exit(SPLTR_EXIT_STATUS);
throw runtime_error(exp.what());
}
}
int aNoSec = 2;
bool bRollback = false;
bool bForce = false;
int iShutdown;
// TODO - this is for just time being....
// we need to process message of main thread here..
// here we need to cont check the status of different things
while (fContinue)
{
++aRollCount;
usleep(1000000);
// Check to see if someone has ordered a shutdown with rollback or force.
iShutdown = fDh.fDbrm.getSystemShutdownPending(bRollback, bForce);
if (iShutdown >= 0)
{
if (bRollback)
{
if (iShutdown > 0) // Means a shutdown, stop or restart
{
cout << "System stop has been ordered. Rollback" << endl;
}
else
{
cout << "Database writes have been suspended. Rollback" << endl;
}
fSignaled = true;
fContinue = false;
}
else if (bForce)
{
//BUG 5012 - added to avoid rollback
fContinue = false;
ostringstream oss;
oss << "Table " << fCmdArgs.getSchemaName() << ".";
oss << fCmdArgs.getTableName() << ": (OID-";
oss << fDh.getTableOID() << ") was NOT successfully loaded.";
cout << oss.str() << endl;
logging::Message::Args errMsgArgs;
//BUG 4152
errMsgArgs.add(fCmdArgs.getSchemaName());
errMsgArgs.add(fCmdArgs.getTableName());
errMsgArgs.add(fDh.getTableOID());
std::string aStr = "Immediate system stop has been ordered, rollback deferred";
cout << aStr << endl;
SPLTR_EXIT_STATUS = 1;
errMsgArgs.add(aStr);
fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_INFO, logging::M0096);
exit(SPLTR_EXIT_STATUS);
//BUG 5012 - commented out to avoid rollback
//cout << "Immediate system stop has been ordered. No rollback" << endl;
//fSignaled = true;
//fContinue = false;
}
}
// Send out a heartbeat to the WriteEnginServers every 10 seconds
if ((0 == (aRollCount % aNoSec)) && (!fSignaled)) // Debugging - every 10 seconds
{
if (aNoSec < 10) aNoSec++; //progressively go up to 10Sec interval
aBs.restart();
aBs << (ByteStream::byte) WE_CLT_SRV_KEEPALIVE;
mutex::scoped_lock aLock(fDh.fSendMutex);
fDh.send2Pm(aBs);
aLock.unlock();
//fDh.sendHeartbeats();
//fDh.checkForConnections(); - decided to recv SIGHUP from OAM instead of this
}
}
}
}
fDh.shutdown();
fDh.shutdown();
} // processMessages
void WESplitterApp::invokeCpimport()
{
boost::uuids::uuid u=boost::uuids::random_generator()();
boost::uuids::uuid u = boost::uuids::random_generator()();
fCmdArgs.setJobUUID(u);
//BUG 4361 - check cpimport.bin is available or not
std::string aCpiBinFile = getCalpontHome() + "/cpimport.bin"; //BUG 4361
if (access(aCpiBinFile.c_str(), X_OK) != 0)
throw runtime_error("Error: Missing File " + aCpiBinFile);
//BUG 4361 - check cpimport.bin is available or not
std::string aCpiBinFile = getCalpontHome() + "/cpimport.bin"; //BUG 4361
fCmdArgs.setMode(3);
std::string aCmdLineStr = fCmdArgs.getCpImportCmdLine();
if (access(aCpiBinFile.c_str(), X_OK) != 0)
throw runtime_error("Error: Missing File " + aCpiBinFile);
updateCmdLineWithPath(aCmdLineStr);
fCmdArgs.setMode(3);
std::string aCmdLineStr = fCmdArgs.getCpImportCmdLine();
if(fDh.getDebugLvl())
cout << "CPI CmdLineArgs : " << aCmdLineStr << endl;
updateCmdLineWithPath(aCmdLineStr);
std::vector<char*> Cmds;
if (fDh.getDebugLvl())
cout << "CPI CmdLineArgs : " << aCmdLineStr << endl;
std::vector<char*> Cmds;
std::istringstream ss(aCmdLineStr);
std::string arg;
std::vector<std::string> v2(20, "");
std::vector<std::string> v2(20, "");
unsigned int i = 0;
while(ss >> arg)
while (ss >> arg)
{
//we need something that works on Windows as well as linux
//we need something that works on Windows as well as linux
v2[i++] = arg;
}
for (unsigned int j = 0; j < i; ++j)
{
Cmds.push_back(const_cast<char*>(v2[j].c_str()));
@ -464,10 +502,10 @@ void WESplitterApp::invokeCpimport()
Cmds.push_back(0); //null terminate
int aRet = execv(Cmds[0], &Cmds[0]); //NOTE - works with full Path
int aRet = execv(Cmds[0], &Cmds[0]); //NOTE - works with full Path
if(fDh.getDebugLvl())
cout << "Return status of cpimport is " << aRet <<endl;
if (fDh.getDebugLvl())
cout << "Return status of cpimport is " << aRet << endl;
}
@ -481,18 +519,19 @@ void WESplitterApp::invokeCpimport()
**/
std::string WESplitterApp::getCalpontHome()
{
string calpontDir = config::Config::makeConfig()->getConfig(
"SystemConfig", "CalpontHome");
if(0 == calpontDir.length())
{
calpontDir = startup::StartUp::installDir() + "/bin";
}
else
{
calpontDir += "/bin";
}
string calpontDir = config::Config::makeConfig()->getConfig(
"SystemConfig", "CalpontHome");
return calpontDir;
if (0 == calpontDir.length())
{
calpontDir = startup::StartUp::installDir() + "/bin";
}
else
{
calpontDir += "/bin";
}
return calpontDir;
}
//-----------------------------------------------------------------------------
@ -505,10 +544,10 @@ std::string WESplitterApp::getCalpontHome()
**/
std::string WESplitterApp::getPrgmPath(std::string& PrgmName)
{
std::string cpimportPath = getCalpontHome();
cpimportPath += "/";
cpimportPath += PrgmName;
return cpimportPath;
std::string cpimportPath = getCalpontHome();
cpimportPath += "/";
cpimportPath += PrgmName;
return cpimportPath;
}
//-----------------------------------------------------------------------------
@ -522,33 +561,35 @@ std::string WESplitterApp::getPrgmPath(std::string& PrgmName)
void WESplitterApp::updateCmdLineWithPath(string& CmdLine)
{
std::istringstream iSs(CmdLine);
std::ostringstream oSs;
std::string aArg;
int aCount=0;
while(iSs >> aArg)
{
if(0 == aCount)
{
string aPrgmPath = getPrgmPath(aArg);
oSs << aPrgmPath;
}
else
{
oSs << " ";
oSs << aArg;
std::istringstream iSs(CmdLine);
std::ostringstream oSs;
std::string aArg;
int aCount = 0;
}
++aCount;
}
while (iSs >> aArg)
{
if (0 == aCount)
{
string aPrgmPath = getPrgmPath(aArg);
oSs << aPrgmPath;
}
else
{
oSs << " ";
oSs << aArg;
CmdLine = oSs.str();
}
++aCount;
}
CmdLine = oSs.str();
}
//-----------------------------------------------------------------------------
void WESplitterApp::updateWithJobFile(int aIdx)
{
fCmdArgs.updateWithJobFile(aIdx);
fCmdArgs.updateWithJobFile(aIdx);
}
//-----------------------------------------------------------------------------
@ -557,59 +598,64 @@ void WESplitterApp::updateWithJobFile(int aIdx)
} /* namespace WriteEngine */
//------------------------------------------------------------------------------
// main function
// main function
//------------------------------------------------------------------------------
int main(int argc, char** argv)
{
std::string err;
setuid(0); //@BUG 4343 set effective userid to root.
std::cin.sync_with_stdio(false);
try
{
WriteEngine::WECmdArgs aWeCmdArgs(argc, argv);
WriteEngine::WESplitterApp aWESplitterApp(aWeCmdArgs);
int aTblCnt = aWESplitterApp.fCmdArgs.getMultiTableCount();
if(aTblCnt>1)
{
for(int idx=0; idx<aTblCnt; idx++)
{
aWESplitterApp.fDh.reset();
aWESplitterApp.fContinue = true;
aWESplitterApp.updateWithJobFile(idx);
try
{
aWESplitterApp.fDh.setup();
}
catch (std::exception& ex)
{
//err = string("Error in constructing WESplitterApp") + ex.what();
err = ex.what(); //cleaning up for BUG 4298
logging::Message::Args errMsgArgs;
errMsgArgs.add(err);
aWESplitterApp.fpSysLog->logMsg(errMsgArgs,logging::LOG_TYPE_ERROR,logging::M0000);
SPLTR_EXIT_STATUS=1;
aWESplitterApp.fDh.fLog.logMsg( err, MSGLVL_ERROR );
aWESplitterApp.fContinue = false;
//throw runtime_error(err); BUG 4298
}
aWESplitterApp.processMessages();
std::string err;
setuid(0); //@BUG 4343 set effective userid to root.
std::cin.sync_with_stdio(false);
if(SPLTR_EXIT_STATUS == 1) break;
}
}
else
{
aWESplitterApp.processMessages();
}
} catch (std::exception& exp)
{
cerr << exp.what() << endl;
SPLTR_EXIT_STATUS = 1;
exit(SPLTR_EXIT_STATUS); // exit with an error
}
try
{
WriteEngine::WECmdArgs aWeCmdArgs(argc, argv);
WriteEngine::WESplitterApp aWESplitterApp(aWeCmdArgs);
int aTblCnt = aWESplitterApp.fCmdArgs.getMultiTableCount();
return SPLTR_EXIT_STATUS;
if (aTblCnt > 1)
{
for (int idx = 0; idx < aTblCnt; idx++)
{
aWESplitterApp.fDh.reset();
aWESplitterApp.fContinue = true;
aWESplitterApp.updateWithJobFile(idx);
try
{
aWESplitterApp.fDh.setup();
}
catch (std::exception& ex)
{
//err = string("Error in constructing WESplitterApp") + ex.what();
err = ex.what(); //cleaning up for BUG 4298
logging::Message::Args errMsgArgs;
errMsgArgs.add(err);
aWESplitterApp.fpSysLog->logMsg(errMsgArgs, logging::LOG_TYPE_ERROR, logging::M0000);
SPLTR_EXIT_STATUS = 1;
aWESplitterApp.fDh.fLog.logMsg( err, MSGLVL_ERROR );
aWESplitterApp.fContinue = false;
//throw runtime_error(err); BUG 4298
}
aWESplitterApp.processMessages();
if (SPLTR_EXIT_STATUS == 1) break;
}
}
else
{
aWESplitterApp.processMessages();
}
}
catch (std::exception& exp)
{
cerr << exp.what() << endl;
SPLTR_EXIT_STATUS = 1;
exit(SPLTR_EXIT_STATUS); // exit with an error
}
return SPLTR_EXIT_STATUS;
}
// vim:ts=4 sw=4:

170
writeengine/splitter/we_splitterapp.h
View File

@ -14,82 +14,94 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splitterapp.h
*
* Created on: Oct 7, 2011
* Author: bpaul
*/
#ifndef WE_SPLITTERAPP_H_
#define WE_SPLITTERAPP_H_
#include <boost/thread/condition.hpp>
#include <boost/scoped_array.hpp>
#include <boost/thread.hpp>
using namespace boost;
#include "bytestream.h"
using namespace messageqcpp;
#include "we_cmdargs.h"
#include "we_sdhandler.h"
#include "we_simplesyslog.h"
using namespace WriteEngine;
namespace WriteEngine
{
class WESplitterApp
{
public:
WESplitterApp(WECmdArgs& CmdArgs);
virtual ~WESplitterApp();
void processMessages();
int getMode(){ return fCmdArgs.getMode(); }
bool getPmStatus(int Id){ return fCmdArgs.getPmStatus(Id); }
std::string getLocFile() { return fCmdArgs.getLocFile(); }
std::string getPmFile() { return fCmdArgs.getPmFile(); }
void updateWithJobFile(int aIdx);
// setup the signal handlers for the main app
void setupSignalHandlers();
static void onSigTerminate(int aInt);
static void onSigInterrupt(int aInt);
static void onSigHup(int aInt);
void invokeCpimport();
std::string getCalpontHome();
std::string getPrgmPath(std::string& PrgmName);
void updateCmdLineWithPath(string& CmdLine);
private:
public: // for multi table support
WECmdArgs& fCmdArgs;
WESDHandler fDh;
static bool fContinue;
public:
static bool fSignaled;
static bool fSigHup;
public:
static SimpleSysLog* fpSysLog;
public:
friend class WESDHandler;
};
} /* namespace WriteEngine */
#endif /* WE_SPLITTERAPP_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_splitterapp.h
*
* Created on: Oct 7, 2011
* Author: bpaul
*/
#ifndef WE_SPLITTERAPP_H_
#define WE_SPLITTERAPP_H_
#include <boost/thread/condition.hpp>
#include <boost/scoped_array.hpp>
#include <boost/thread.hpp>
using namespace boost;
#include "bytestream.h"
using namespace messageqcpp;
#include "we_cmdargs.h"
#include "we_sdhandler.h"
#include "we_simplesyslog.h"
using namespace WriteEngine;
namespace WriteEngine
{
class WESplitterApp
{
public:
WESplitterApp(WECmdArgs& CmdArgs);
virtual ~WESplitterApp();
void processMessages();
int getMode()
{
return fCmdArgs.getMode();
}
bool getPmStatus(int Id)
{
return fCmdArgs.getPmStatus(Id);
}
std::string getLocFile()
{
return fCmdArgs.getLocFile();
}
std::string getPmFile()
{
return fCmdArgs.getPmFile();
}
void updateWithJobFile(int aIdx);
// setup the signal handlers for the main app
void setupSignalHandlers();
static void onSigTerminate(int aInt);
static void onSigInterrupt(int aInt);
static void onSigHup(int aInt);
void invokeCpimport();
std::string getCalpontHome();
std::string getPrgmPath(std::string& PrgmName);
void updateCmdLineWithPath(string& CmdLine);
private:
public: // for multi table support
WECmdArgs& fCmdArgs;
WESDHandler fDh;
static bool fContinue;
public:
static bool fSignaled;
static bool fSigHup;
public:
static SimpleSysLog* fpSysLog;
public:
friend class WESDHandler;
};
} /* namespace WriteEngine */
#endif /* WE_SPLITTERAPP_H_ */

221
writeengine/splitter/we_tablelockgrabber.cpp
View File

@ -14,111 +14,116 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_tablelockgrabber.cpp
*
* Created on: Dec 19, 2011
* Author: bpaul
*/
#include "brm.h"
#include "brmtypes.h"
using namespace BRM;
#include <iostream>
using namespace std;
#include "we_sdhandler.h"
#include "we_tablelockgrabber.h"
namespace WriteEngine
{
uint64_t WETableLockGrabber::grabTableLock(std::vector<unsigned int> &PmList,
uint32_t tableOID)
{
uint64_t aLockId;
std::string aProcName = "cpimport";
uint32_t aProcId = getpid();
int32_t aSessId = -1;
int32_t aTxnId = -1;
try
{
aLockId = fRef.fDbrm.getTableLock(PmList, tableOID, &aProcName,
&aProcId, &aSessId, &aTxnId, BRM::LOADING);
}
catch (std::exception &e)
{
cout << "ERROR: Failed to get Table Lock " << e.what() << endl;
throw runtime_error(e.what());
}
if (aLockId == 0)
{
ostringstream oss;
oss << " Table currently locked by process-" <<
aProcName << "; pid-" << aProcId <<
"; session-" << aSessId <<
"; txn-" << aTxnId;
throw runtime_error(oss.str());
}
//cout << "lock ID = " << aLockId << endl;
//if (aLockId == 0)
// cout << " existing owner name = " << aProcName << " pid = " <<
// aProcId << " session = " << aSessId << endl;
return aLockId;
}
bool WETableLockGrabber::releaseTableLock(uint64_t LockId)
{
bool aRet;
//cout << "releasing lock " << LockId << endl;
try
{
aRet = fRef.fDbrm.releaseTableLock(LockId);
}
catch (std::exception &e)
{
cout << "caught an exception: " << e.what() << endl;
throw runtime_error(e.what());
}
return aRet;
}
bool WETableLockGrabber::changeTableLockState(uint64_t LockId)
{
bool aRet;
//cout << "changing state of lock " << LockId << endl;
try
{
aRet = fRef.fDbrm.changeState(LockId, BRM::CLEANUP);
}
catch (std::exception &e)
{
cout << "caught an exception: " << e.what() << endl;
throw runtime_error(e.what());
}
return aRet;
}
} /* namespace WriteEngine */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_tablelockgrabber.cpp
*
* Created on: Dec 19, 2011
* Author: bpaul
*/
#include "brm.h"
#include "brmtypes.h"
using namespace BRM;
#include <iostream>
using namespace std;
#include "we_sdhandler.h"
#include "we_tablelockgrabber.h"
namespace WriteEngine
{
uint64_t WETableLockGrabber::grabTableLock(std::vector<unsigned int>& PmList,
uint32_t tableOID)
{
uint64_t aLockId;
std::string aProcName = "cpimport";
uint32_t aProcId = getpid();
int32_t aSessId = -1;
int32_t aTxnId = -1;
try
{
aLockId = fRef.fDbrm.getTableLock(PmList, tableOID, &aProcName,
&aProcId, &aSessId, &aTxnId, BRM::LOADING);
}
catch (std::exception& e)
{
cout << "ERROR: Failed to get Table Lock " << e.what() << endl;
throw runtime_error(e.what());
}
if (aLockId == 0)
{
ostringstream oss;
oss << " Table currently locked by process-" <<
aProcName << "; pid-" << aProcId <<
"; session-" << aSessId <<
"; txn-" << aTxnId;
throw runtime_error(oss.str());
}
//cout << "lock ID = " << aLockId << endl;
//if (aLockId == 0)
// cout << " existing owner name = " << aProcName << " pid = " <<
// aProcId << " session = " << aSessId << endl;
return aLockId;
}
bool WETableLockGrabber::releaseTableLock(uint64_t LockId)
{
bool aRet;
//cout << "releasing lock " << LockId << endl;
try
{
aRet = fRef.fDbrm.releaseTableLock(LockId);
}
catch (std::exception& e)
{
cout << "caught an exception: " << e.what() << endl;
throw runtime_error(e.what());
}
return aRet;
}
bool WETableLockGrabber::changeTableLockState(uint64_t LockId)
{
bool aRet;
//cout << "changing state of lock " << LockId << endl;
try
{
aRet = fRef.fDbrm.changeState(LockId, BRM::CLEANUP);
}
catch (std::exception& e)
{
cout << "caught an exception: " << e.what() << endl;
throw runtime_error(e.what());
}
return aRet;
}
} /* namespace WriteEngine */

94
writeengine/splitter/we_tablelockgrabber.h
View File

@ -14,50 +14,50 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_tableLockgrabber.h
*
* Created on: Dec 19, 2011
* Author: bpaul
*/
#ifndef WE_TABLELOCKGRABBER_H_
#define WE_TABLELOCKGRABBER_H_
namespace WriteEngine
{
class WESDHandler; // forward deceleration
class WETableLockGrabber
{
public:
WETableLockGrabber(WESDHandler& Ref): fRef(Ref) { }
virtual ~WETableLockGrabber() { }
public:
uint64_t grabTableLock(std::vector<unsigned int> &PmList,
uint32_t tableOID);
bool releaseTableLock(uint64_t LockId);
bool changeTableLockState(uint64_t LockId);
private:
WESDHandler& fRef;
};
} /* namespace WriteEngine */
#endif /* WE_TABLELOCKGRABBER_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_tableLockgrabber.h
*
* Created on: Dec 19, 2011
* Author: bpaul
*/
#ifndef WE_TABLELOCKGRABBER_H_
#define WE_TABLELOCKGRABBER_H_
namespace WriteEngine
{
class WESDHandler; // forward deceleration
class WETableLockGrabber
{
public:
WETableLockGrabber(WESDHandler& Ref): fRef(Ref) { }
virtual ~WETableLockGrabber() { }
public:
uint64_t grabTableLock(std::vector<unsigned int>& PmList,
uint32_t tableOID);
bool releaseTableLock(uint64_t LockId);
bool changeTableLockState(uint64_t LockId);
private:
WESDHandler& fRef;
};
} /* namespace WriteEngine */
#endif /* WE_TABLELOCKGRABBER_H_ */

684
writeengine/splitter/we_xmlgetter.cpp
View File

@ -14,330 +14,360 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_xmlgetter.cpp
*
* Created on: Feb 7, 2012
* Author: bpaul
*/
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include <vector>
using namespace std;
#include "we_xmlgetter.h"
using namespace std;
namespace WriteEngine
{
//------------------------------------------------------------------------------
// WEXmlgetter constructor
//------------------------------------------------------------------------------
WEXmlgetter::WEXmlgetter(std::string& ConfigName):
fConfigName(ConfigName),
fDoc( NULL ),
fpRoot( NULL )
{
// xmlNodePtr curPtr;
fDoc = xmlParseFile( ConfigName.c_str() );
if(fDoc == NULL )
throw runtime_error("WEXmlgetter::getConfig(): no XML document!");
fpRoot = xmlDocGetRootElement( fDoc );
if( fpRoot == NULL )
{
xmlFreeDoc( fDoc );
fDoc = NULL;
throw runtime_error("WEXmlgetter::getConfig(): no XML Root Tag!");
}
}
//------------------------------------------------------------------------------
// WEXmlgetter destructor
//------------------------------------------------------------------------------
WEXmlgetter::~WEXmlgetter()
{
xmlFreeDoc( fDoc );
fDoc = NULL;
}
//------------------------------------------------------------------------------
// Get/return the property or attribute value (strVal) for the specified xml tag
// (pNode) and property/attribute (pTag)
//------------------------------------------------------------------------------
bool WEXmlgetter::getNodeAttribute(const xmlNode* pNode,
const char* pTag, std::string& strVal) const
{
xmlChar* pTmp = NULL;
bool bFound = false;
pTmp = xmlGetProp( const_cast<xmlNode*>(pNode), (xmlChar*) pTag );
if( pTmp ) {
bFound = true;
strVal = (char*)pTmp;
xmlFree( pTmp );
}
else {
strVal.clear();
} // end if
return bFound;
}
//------------------------------------------------------------------------------
// Get/return the node content (strVal) for the specified xml tag (pNode)
//------------------------------------------------------------------------------
bool WEXmlgetter::getNodeContent( const xmlNode* pNode,
std::string& strVal) const
{
xmlChar* pTmp = NULL;
bool bFound = false;
if( pNode->children != NULL ) {
pTmp = xmlNodeGetContent( pNode->children );
if( pTmp ) {
bFound = true;
strVal = (char*)pTmp;
xmlFree( pTmp );
}
else {
strVal.clear();
}
}
else {
strVal.clear();
}
return bFound;
}
//------------------------------------------------------------------------------
// Get/returns node content for the "first" child node under each section/name.
// Example:
// <section>
// <name>
// <subname1>
// </subname1>
// </name>
// <name>
// <subname1>
// </subname1>
// </name>
// </section>
//
// Looks like xml2 is currently returning the text node as the first child
// node under a node. So in the example above, this function is currently
// always returning the text node content inside each <name> rather than
// any <subname1> node that might be within each <name> tag.
//------------------------------------------------------------------------------
void WEXmlgetter::getConfig(const string& section,
const string& name, vector<string>& values) const
{
string res;
if (section.length() == 0)
throw invalid_argument("Config::getConfig: section must have a length");
xmlNode* pPtr = fpRoot->xmlChildrenNode;
while (pPtr != NULL)
{
//cout << "pPtr->name: " <<
// (const xmlChar*)pPtr->name << std::endl;
if ((!xmlStrcmp(pPtr->name, (const xmlChar *)section.c_str())))
{
xmlNodePtr pPtr2 = pPtr->xmlChildrenNode;
while (pPtr2 != NULL)
{
//cout << " pPtr2->name: " <<
// (const xmlChar*)pPtr2->name << std::endl;
if ((!xmlStrcmp(pPtr2->name, (const xmlChar*)name.c_str())))
{
xmlNodePtr pPtr3 = pPtr2->xmlChildrenNode;
values.push_back((const char*)pPtr3->content);
//cout << " pPtr3->name: " <<
// (const xmlChar*)pPtr3->name <<
// "; content: " << (const xmlChar*)pPtr3->content <<
// "; len: " << strlen((char*)pPtr3->content) << std::endl;
}
pPtr2 = pPtr2->next;
}
}
pPtr = pPtr->next;
}
}
//------------------------------------------------------------------------------
// Returns node content for the last node in the node tree defined by
// "sections". So if sections[] were:
// sections[0] = "house"
// sections[1] = "room"
// Then this function would return the node content for the first <room>
// tag found under the first <house> tag.
// Function assumes that the desired node has no children nodes other than
// the text content node.
//------------------------------------------------------------------------------
std::string WEXmlgetter::getValue(const vector<string>& sections) const
{
std::string aRet;
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
int aIdx = 0;
//cout << aSize << endl;
while(aIdx < aSize)
{
//cout << aIdx <<" "<< sections[aIdx] << endl;
pPtr = getNode(pPtr, sections[aIdx]);
if((pPtr == NULL) || (aIdx == aSize-1)) break;
else
{
//cout << "getValue Name " << (const char*)pPtr->name << endl;
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
if(pPtr != NULL)
{
//aRet = (const char*)pPtr->content;
std::string aBuff;
if(getNodeContent(pPtr, aBuff)) aRet = aBuff;
}
return aRet;
}
//------------------------------------------------------------------------------
// Iterate through the sibling nodes starting with pParent, looking for
// a node with the specified name (section). The xmlNode (if found) is
// returned.
//------------------------------------------------------------------------------
const xmlNode* WEXmlgetter::getNode(const xmlNode* pParent,
const string& section)const
{
if(pParent == NULL) return NULL;
const xmlNode* pPtr = pParent;
while(pPtr != NULL )
{
//cout << "getNode Name " << (const char*)pPtr->name << endl;
if(!xmlStrcmp(pPtr->name, (const xmlChar *)section.c_str()))
return pPtr;
else
pPtr = pPtr->next;
}
return pPtr;
}
//------------------------------------------------------------------------------
// Iterate down through the node tree represented by the sections vector.
// In the last child of this tree, we look for the specified attribute tag,
// and return its value.
//------------------------------------------------------------------------------
std::string WEXmlgetter::getAttribute(const vector<string>& sections,
const string& Tag) const
{
std::string aRet;
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
if(aSize==0)
throw invalid_argument("WEXmlgetter::getAttribute(): section must be valid");
int aIdx = 0;
//cout << aSize << endl;
while(aIdx < aSize)
{
//cout << aIdx <<" "<< sections[aIdx] << endl;
pPtr = getNode(pPtr, sections[aIdx]);
if((pPtr == NULL) || (aIdx == aSize-1)) break;
else
{
//cout << "getValue Name " << (const char*)pPtr->name << endl;
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
if(pPtr != NULL)
{
std::string aBuff;
//cout << "attrTagNode Name " << (const char*)pPtr->name << endl;
if (getNodeAttribute(pPtr, Tag.c_str(), aBuff))
aRet = aBuff;
//aRet = (const char*)pPtr->content;
//cout << "Attribute("<<Tag<<") = "<< aRet<< endl;
}
return aRet;
}
//------------------------------------------------------------------------------
// Iterate down through the node tree represented by the sections vector.
// At the end of the branch, there may be several sibling nodes matching
// the node search vector.
// For each of the matching children nodes found, we look for the specified
// attribute tag, and return its value. Hence a vector of attribute values
// is returned.
//------------------------------------------------------------------------------
void WEXmlgetter::getAttributeListForAllChildren(
const vector<string>& sections,
const string& attributeTag,
vector<string>& attributeValues)
{
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
if(aSize==0)
{
throw invalid_argument("WEXmlgetter::getAttributeListForAllChildren():"
" No XML nodes specified in section search list");
}
// Step down the branch that has the nodes of interest
int aIdx = 0;
while (aIdx < aSize)
{
pPtr = getNode(pPtr, sections[aIdx]);
if ((pPtr == NULL) || (aIdx == aSize-1))
{
break;
}
else
{
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
// Look for all the "matching" nodes at the end of the branch, and
// get the requested attribute value for each matching node.
if (pPtr != NULL)
{
while(pPtr != NULL )
{
std::string attrib;
if (getNodeAttribute(pPtr, attributeTag.c_str(), attrib))
{
attributeValues.push_back(attrib);
}
pPtr = pPtr->next;
}
}
}
} /* namespace WriteEngine */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_xmlgetter.cpp
*
* Created on: Feb 7, 2012
* Author: bpaul
*/
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include <vector>
using namespace std;
#include "we_xmlgetter.h"
using namespace std;
namespace WriteEngine
{
//------------------------------------------------------------------------------
// WEXmlgetter constructor
//------------------------------------------------------------------------------
WEXmlgetter::WEXmlgetter(std::string& ConfigName):
fConfigName(ConfigName),
fDoc( NULL ),
fpRoot( NULL )
{
// xmlNodePtr curPtr;
fDoc = xmlParseFile( ConfigName.c_str() );
if (fDoc == NULL )
throw runtime_error("WEXmlgetter::getConfig(): no XML document!");
fpRoot = xmlDocGetRootElement( fDoc );
if ( fpRoot == NULL )
{
xmlFreeDoc( fDoc );
fDoc = NULL;
throw runtime_error("WEXmlgetter::getConfig(): no XML Root Tag!");
}
}
//------------------------------------------------------------------------------
// WEXmlgetter destructor
//------------------------------------------------------------------------------
WEXmlgetter::~WEXmlgetter()
{
xmlFreeDoc( fDoc );
fDoc = NULL;
}
//------------------------------------------------------------------------------
// Get/return the property or attribute value (strVal) for the specified xml tag
// (pNode) and property/attribute (pTag)
//------------------------------------------------------------------------------
bool WEXmlgetter::getNodeAttribute(const xmlNode* pNode,
const char* pTag, std::string& strVal) const
{
xmlChar* pTmp = NULL;
bool bFound = false;
pTmp = xmlGetProp( const_cast<xmlNode*>(pNode), (xmlChar*) pTag );
if ( pTmp )
{
bFound = true;
strVal = (char*)pTmp;
xmlFree( pTmp );
}
else
{
strVal.clear();
} // end if
return bFound;
}
//------------------------------------------------------------------------------
// Get/return the node content (strVal) for the specified xml tag (pNode)
//------------------------------------------------------------------------------
bool WEXmlgetter::getNodeContent( const xmlNode* pNode,
std::string& strVal) const
{
xmlChar* pTmp = NULL;
bool bFound = false;
if ( pNode->children != NULL )
{
pTmp = xmlNodeGetContent( pNode->children );
if ( pTmp )
{
bFound = true;
strVal = (char*)pTmp;
xmlFree( pTmp );
}
else
{
strVal.clear();
}
}
else
{
strVal.clear();
}
return bFound;
}
//------------------------------------------------------------------------------
// Get/returns node content for the "first" child node under each section/name.
// Example:
// <section>
// <name>
// <subname1>
// </subname1>
// </name>
// <name>
// <subname1>
// </subname1>
// </name>
// </section>
//
// Looks like xml2 is currently returning the text node as the first child
// node under a node. So in the example above, this function is currently
// always returning the text node content inside each <name> rather than
// any <subname1> node that might be within each <name> tag.
//------------------------------------------------------------------------------
void WEXmlgetter::getConfig(const string& section,
const string& name, vector<string>& values) const
{
string res;
if (section.length() == 0)
throw invalid_argument("Config::getConfig: section must have a length");
xmlNode* pPtr = fpRoot->xmlChildrenNode;
while (pPtr != NULL)
{
//cout << "pPtr->name: " <<
// (const xmlChar*)pPtr->name << std::endl;
if ((!xmlStrcmp(pPtr->name, (const xmlChar*)section.c_str())))
{
xmlNodePtr pPtr2 = pPtr->xmlChildrenNode;
while (pPtr2 != NULL)
{
//cout << " pPtr2->name: " <<
// (const xmlChar*)pPtr2->name << std::endl;
if ((!xmlStrcmp(pPtr2->name, (const xmlChar*)name.c_str())))
{
xmlNodePtr pPtr3 = pPtr2->xmlChildrenNode;
values.push_back((const char*)pPtr3->content);
//cout << " pPtr3->name: " <<
// (const xmlChar*)pPtr3->name <<
// "; content: " << (const xmlChar*)pPtr3->content <<
// "; len: " << strlen((char*)pPtr3->content) << std::endl;
}
pPtr2 = pPtr2->next;
}
}
pPtr = pPtr->next;
}
}
//------------------------------------------------------------------------------
// Returns node content for the last node in the node tree defined by
// "sections". So if sections[] were:
// sections[0] = "house"
// sections[1] = "room"
// Then this function would return the node content for the first <room>
// tag found under the first <house> tag.
// Function assumes that the desired node has no children nodes other than
// the text content node.
//------------------------------------------------------------------------------
std::string WEXmlgetter::getValue(const vector<string>& sections) const
{
std::string aRet;
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
int aIdx = 0;
//cout << aSize << endl;
while (aIdx < aSize)
{
//cout << aIdx <<" "<< sections[aIdx] << endl;
pPtr = getNode(pPtr, sections[aIdx]);
if ((pPtr == NULL) || (aIdx == aSize - 1)) break;
else
{
//cout << "getValue Name " << (const char*)pPtr->name << endl;
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
if (pPtr != NULL)
{
//aRet = (const char*)pPtr->content;
std::string aBuff;
if (getNodeContent(pPtr, aBuff)) aRet = aBuff;
}
return aRet;
}
//------------------------------------------------------------------------------
// Iterate through the sibling nodes starting with pParent, looking for
// a node with the specified name (section). The xmlNode (if found) is
// returned.
//------------------------------------------------------------------------------
const xmlNode* WEXmlgetter::getNode(const xmlNode* pParent,
const string& section)const
{
if (pParent == NULL) return NULL;
const xmlNode* pPtr = pParent;
while (pPtr != NULL )
{
//cout << "getNode Name " << (const char*)pPtr->name << endl;
if (!xmlStrcmp(pPtr->name, (const xmlChar*)section.c_str()))
return pPtr;
else
pPtr = pPtr->next;
}
return pPtr;
}
//------------------------------------------------------------------------------
// Iterate down through the node tree represented by the sections vector.
// In the last child of this tree, we look for the specified attribute tag,
// and return its value.
//------------------------------------------------------------------------------
std::string WEXmlgetter::getAttribute(const vector<string>& sections,
const string& Tag) const
{
std::string aRet;
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
if (aSize == 0)
throw invalid_argument("WEXmlgetter::getAttribute(): section must be valid");
int aIdx = 0;
//cout << aSize << endl;
while (aIdx < aSize)
{
//cout << aIdx <<" "<< sections[aIdx] << endl;
pPtr = getNode(pPtr, sections[aIdx]);
if ((pPtr == NULL) || (aIdx == aSize - 1)) break;
else
{
//cout << "getValue Name " << (const char*)pPtr->name << endl;
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
if (pPtr != NULL)
{
std::string aBuff;
//cout << "attrTagNode Name " << (const char*)pPtr->name << endl;
if (getNodeAttribute(pPtr, Tag.c_str(), aBuff))
aRet = aBuff;
//aRet = (const char*)pPtr->content;
//cout << "Attribute("<<Tag<<") = "<< aRet<< endl;
}
return aRet;
}
//------------------------------------------------------------------------------
// Iterate down through the node tree represented by the sections vector.
// At the end of the branch, there may be several sibling nodes matching
// the node search vector.
// For each of the matching children nodes found, we look for the specified
// attribute tag, and return its value. Hence a vector of attribute values
// is returned.
//------------------------------------------------------------------------------
void WEXmlgetter::getAttributeListForAllChildren(
const vector<string>& sections,
const string& attributeTag,
vector<string>& attributeValues)
{
const xmlNode* pPtr = fpRoot;
int aSize = sections.size();
if (aSize == 0)
{
throw invalid_argument("WEXmlgetter::getAttributeListForAllChildren():"
" No XML nodes specified in section search list");
}
// Step down the branch that has the nodes of interest
int aIdx = 0;
while (aIdx < aSize)
{
pPtr = getNode(pPtr, sections[aIdx]);
if ((pPtr == NULL) || (aIdx == aSize - 1))
{
break;
}
else
{
pPtr = pPtr->xmlChildrenNode;
aIdx++;
}
}
// Look for all the "matching" nodes at the end of the branch, and
// get the requested attribute value for each matching node.
if (pPtr != NULL)
{
while (pPtr != NULL )
{
std::string attrib;
if (getNodeAttribute(pPtr, attributeTag.c_str(), attrib))
{
attributeValues.push_back(attrib);
}
pPtr = pPtr->next;
}
}
}
} /* namespace WriteEngine */

110
writeengine/splitter/we_xmlgetter.h
View File

@ -14,58 +14,58 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_xmlgetter.h
*
* Created on: Feb 7, 2012
* Author: bpaul
*/
#ifndef WE_XMLGETTER_H_
#define WE_XMLGETTER_H_
#include <libxml/parser.h>
namespace WriteEngine
{
class WEXmlgetter
{
public:
WEXmlgetter(std::string& ConfigName);
virtual ~WEXmlgetter();
public:
//..Public methods
std::string getValue(const vector<string>& section) const;
std::string getAttribute(const std::vector<string>& sections,
const std::string& Tag) const;
void getConfig(const std::string& section,
const std::string& name, std::vector<std::string>& values ) const;
void getAttributeListForAllChildren(
const vector<string>& sections,
const string& attributeTag,
vector<string>& attributeValues);
private:
//..Private methods
const xmlNode* getNode(const xmlNode* pParent,
const std::string& section)const;
bool getNodeAttribute(const xmlNode* pNode,
const char* pTag, std::string& strVal ) const;
bool getNodeContent( const xmlNode* pNode, std::string& strVal) const;
//..Private data members
std::string fConfigName; // xml filename
xmlDocPtr fDoc; // xml document pointer
xmlNode* fpRoot; // root element
};
} /* namespace WriteEngine */
#endif /* WE_XMLGETTER_H_ */
/*******************************************************************************
* $Id$
*
*******************************************************************************/
/*
* we_xmlgetter.h
*
* Created on: Feb 7, 2012
* Author: bpaul
*/
#ifndef WE_XMLGETTER_H_
#define WE_XMLGETTER_H_
#include <libxml/parser.h>
namespace WriteEngine
{
class WEXmlgetter
{
public:
WEXmlgetter(std::string& ConfigName);
virtual ~WEXmlgetter();
public:
//..Public methods
std::string getValue(const vector<string>& section) const;
std::string getAttribute(const std::vector<string>& sections,
const std::string& Tag) const;
void getConfig(const std::string& section,
const std::string& name, std::vector<std::string>& values ) const;
void getAttributeListForAllChildren(
const vector<string>& sections,
const string& attributeTag,
vector<string>& attributeValues);
private:
//..Private methods
const xmlNode* getNode(const xmlNode* pParent,
const std::string& section)const;
bool getNodeAttribute(const xmlNode* pNode,
const char* pTag, std::string& strVal ) const;
bool getNodeContent( const xmlNode* pNode, std::string& strVal) const;
//..Private data members
std::string fConfigName; // xml filename
xmlDocPtr fDoc; // xml document pointer
xmlNode* fpRoot; // root element
};
} /* namespace WriteEngine */
#endif /* WE_XMLGETTER_H_ */