database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3893
primitives/primproc/batchprimitiveprocessor.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

510
primitives/primproc/batchprimitiveprocessor.h Executable file → Normal file
View File

@ -65,300 +65,340 @@ namespace primitiveprocessor
typedef boost::shared_ptr<BatchPrimitiveProcessor> SBPP;
class scalar_exception : public std::exception {
const char * what() const throw() { return "Not a scalar subquery."; }
class scalar_exception : public std::exception
{
const char* what() const throw()
{
return "Not a scalar subquery.";
}
};
class NeedToRestartJob : public std::runtime_error
{
public:
NeedToRestartJob() : std::runtime_error("NeedToRestartJob") { }
NeedToRestartJob(const std::string &s) :
std::runtime_error(s) { }
public:
NeedToRestartJob() : std::runtime_error("NeedToRestartJob") { }
NeedToRestartJob(const std::string& s) :
std::runtime_error(s) { }
};
class BatchPrimitiveProcessor
{
public:
BatchPrimitiveProcessor(messageqcpp::ByteStream &, double prefetchThresh,
boost::shared_ptr<BPPSendThread>);
public:
BatchPrimitiveProcessor(messageqcpp::ByteStream&, double prefetchThresh,
boost::shared_ptr<BPPSendThread>);
~BatchPrimitiveProcessor();
~BatchPrimitiveProcessor();
/* Interface used by primproc */
void initBPP(messageqcpp::ByteStream &);
void resetBPP(messageqcpp::ByteStream &, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& outputSock);
void addToJoiner(messageqcpp::ByteStream &);
int endOfJoiner();
int operator()();
void setLBIDForScan(uint64_t rid);
/* Interface used by primproc */
void initBPP(messageqcpp::ByteStream&);
void resetBPP(messageqcpp::ByteStream&, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& outputSock);
void addToJoiner(messageqcpp::ByteStream&);
int endOfJoiner();
int operator()();
void setLBIDForScan(uint64_t rid);
/* Duplicate() returns a deep copy of this object as it was init'd by initBPP.
It's thread-safe wrt resetBPP. */
SBPP duplicate();
/* Duplicate() returns a deep copy of this object as it was init'd by initBPP.
It's thread-safe wrt resetBPP. */
SBPP duplicate();
/* These need to be updated */
//bool operator==(const BatchPrimitiveProcessor&) const;
//inline bool operator!=(const BatchPrimitiveProcessor& bpp) const
//{
// return !(*this == bpp);
//}
/* These need to be updated */
//bool operator==(const BatchPrimitiveProcessor&) const;
//inline bool operator!=(const BatchPrimitiveProcessor& bpp) const
//{
// return !(*this == bpp);
//}
inline uint32_t getSessionID() { return sessionID; }
inline uint32_t getStepID() { return stepID; }
inline uint32_t getUniqueID() { return uniqueID; }
inline bool busy() { return fBusy; }
inline void busy(bool b) { fBusy = b; }
inline uint32_t getSessionID()
{
return sessionID;
}
inline uint32_t getStepID()
{
return stepID;
}
inline uint32_t getUniqueID()
{
return uniqueID;
}
inline bool busy()
{
return fBusy;
}
inline void busy(bool b)
{
fBusy = b;
}
uint16_t FilterCount() const {return filterCount;}
uint16_t ProjectCount() const {return projectCount;}
uint32_t PhysIOCount() const { return physIO;}
uint32_t CachedIOCount() const { return cachedIO;}
uint32_t BlocksTouchedCount() const { return touchedBlocks;}
uint16_t FilterCount() const
{
return filterCount;
}
uint16_t ProjectCount() const
{
return projectCount;
}
uint32_t PhysIOCount() const
{
return physIO;
}
uint32_t CachedIOCount() const
{
return cachedIO;
}
uint32_t BlocksTouchedCount() const
{
return touchedBlocks;
}
void setError(const std::string& error, logging::ErrorCodeValues errorCode) {}
void setError(const std::string& error, logging::ErrorCodeValues errorCode) {}
// these two functions are used by BPPV to create BPP instances
// on demand. TRY not to use unlock() for anything else.
void unlock() { pthread_mutex_unlock(&objLock); }
bool hasJoin() { return doJoin; }
private:
BatchPrimitiveProcessor();
BatchPrimitiveProcessor(const BatchPrimitiveProcessor &);
BatchPrimitiveProcessor& operator=(const BatchPrimitiveProcessor &);
// these two functions are used by BPPV to create BPP instances
// on demand. TRY not to use unlock() for anything else.
void unlock()
{
pthread_mutex_unlock(&objLock);
}
bool hasJoin()
{
return doJoin;
}
private:
BatchPrimitiveProcessor();
BatchPrimitiveProcessor(const BatchPrimitiveProcessor&);
BatchPrimitiveProcessor& operator=(const BatchPrimitiveProcessor&);
void initProcessor();
void initProcessor();
#ifdef PRIMPROC_STOPWATCH
void execute(logging::StopWatch *stopwatch);
void execute(logging::StopWatch* stopwatch);
#else
void execute();
void execute();
#endif
void writeProjectionPreamble();
void makeResponse();
void sendResponse();
void writeProjectionPreamble();
void makeResponse();
void sendResponse();
/* Used by scan operations to increment the LBIDs in successive steps */
void nextLBID();
/* Used by scan operations to increment the LBIDs in successive steps */
void nextLBID();
/* these send relative rids, should this be abs rids? */
void serializeElementTypes();
void serializeStrings();
/* these send relative rids, should this be abs rids? */
void serializeElementTypes();
void serializeStrings();
void asyncLoadProjectColumns();
void writeErrorMsg(const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true);
void asyncLoadProjectColumns();
void writeErrorMsg(const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true);
BPSOutputType ot;
BPSOutputType ot;
BRM::QueryContext versionInfo;
uint32_t txnID;
uint32_t sessionID;
uint32_t stepID;
uint32_t uniqueID;
BRM::QueryContext versionInfo;
uint32_t txnID;
uint32_t sessionID;
uint32_t stepID;
uint32_t uniqueID;
// # of times to loop over the command arrays
// ... This is 1, except when the first command is a scan, in which case
// this single BPP object produces count responses.
uint16_t count;
uint64_t baseRid; // first rid of the logical block
// # of times to loop over the command arrays
// ... This is 1, except when the first command is a scan, in which case
// this single BPP object produces count responses.
uint16_t count;
uint64_t baseRid; // first rid of the logical block
uint16_t relRids[LOGICAL_BLOCK_RIDS];
int64_t values[LOGICAL_BLOCK_RIDS];
boost::scoped_array<uint64_t> absRids;
boost::scoped_array<std::string> strValues;
uint16_t ridCount;
bool needStrValues;
uint16_t relRids[LOGICAL_BLOCK_RIDS];
int64_t values[LOGICAL_BLOCK_RIDS];
boost::scoped_array<uint64_t> absRids;
boost::scoped_array<std::string> strValues;
uint16_t ridCount;
bool needStrValues;
/* Common space for primitive data */
static const uint32_t BUFFER_SIZE = 65536;
uint8_t blockData[BLOCK_SIZE * 8];
boost::scoped_array<uint8_t> outputMsg;
uint32_t outMsgSize;
/* Common space for primitive data */
static const uint32_t BUFFER_SIZE = 65536;
uint8_t blockData[BLOCK_SIZE * 8];
boost::scoped_array<uint8_t> outputMsg;
uint32_t outMsgSize;
std::vector<SCommand> filterSteps;
std::vector<SCommand> projectSteps;
//@bug 1136
uint16_t filterCount;
uint16_t projectCount;
bool sendRidsAtDelivery;
uint8_t ridMap;
bool gotAbsRids;
bool gotValues;
std::vector<SCommand> filterSteps;
std::vector<SCommand> projectSteps;
//@bug 1136
uint16_t filterCount;
uint16_t projectCount;
bool sendRidsAtDelivery;
uint8_t ridMap;
bool gotAbsRids;
bool gotValues;
bool hasScan;
bool validCPData;
int64_t minVal, maxVal; // CP data from a scanned column
uint64_t lbidForCP;
bool hasScan;
bool validCPData;
int64_t minVal, maxVal; // CP data from a scanned column
uint64_t lbidForCP;
// IO counters
boost::mutex counterLock;
uint32_t busyLoaderCount;
// IO counters
boost::mutex counterLock;
uint32_t busyLoaderCount;
uint32_t physIO, cachedIO, touchedBlocks;
uint32_t physIO, cachedIO, touchedBlocks;
SP_UM_IOSOCK sock;
messageqcpp::SBS serialized;
SP_UM_MUTEX writelock;
SP_UM_IOSOCK sock;
messageqcpp::SBS serialized;
SP_UM_MUTEX writelock;
// MCOL-744 using pthread mutex instead of Boost mutex because
// in it is possible that this lock could be unlocked when it is
// already unlocked. In Ubuntu 16.04's Boost this triggers a
// crash. Whilst it is very hard to hit this it is still bad.
// Longer term TODO: fix/remove objLock and/or refactor BPP
pthread_mutex_t objLock;
bool LBIDTrace;
bool fBusy;
// MCOL-744 using pthread mutex instead of Boost mutex because
// in it is possible that this lock could be unlocked when it is
// already unlocked. In Ubuntu 16.04's Boost this triggers a
// crash. Whilst it is very hard to hit this it is still bad.
// Longer term TODO: fix/remove objLock and/or refactor BPP
pthread_mutex_t objLock;
bool LBIDTrace;
bool fBusy;
/* Join support TODO: Make join ops a seperate Command class. */
boost::shared_ptr<joiner::Joiner> joiner;
std::vector<joblist::ElementType> smallSideMatches;
bool doJoin;
uint32_t joinerSize;
uint16_t preJoinRidCount;
boost::mutex addToJoinerLock;
void executeJoin();
/* Join support TODO: Make join ops a seperate Command class. */
boost::shared_ptr<joiner::Joiner> joiner;
std::vector<joblist::ElementType> smallSideMatches;
bool doJoin;
uint32_t joinerSize;
uint16_t preJoinRidCount;
boost::mutex addToJoinerLock;
void executeJoin();
// uint32_t ridsIn, ridsOut;
//@bug 1051 FilterStep on PM
bool hasFilterStep;
bool filtOnString;
boost::scoped_array<uint16_t> fFiltCmdRids[2];
boost::scoped_array<int64_t> fFiltCmdValues[2];
boost::scoped_array<std::string> fFiltStrValues[2];
uint64_t fFiltRidCount[2];
//@bug 1051 FilterStep on PM
bool hasFilterStep;
bool filtOnString;
boost::scoped_array<uint16_t> fFiltCmdRids[2];
boost::scoped_array<int64_t> fFiltCmdValues[2];
boost::scoped_array<std::string> fFiltStrValues[2];
uint64_t fFiltRidCount[2];
// query density threshold for prefetch & async loading
double prefetchThreshold;
// query density threshold for prefetch & async loading
double prefetchThreshold;
/* RowGroup support */
rowgroup::RowGroup outputRG;
boost::scoped_ptr<rowgroup::RGData> outRowGroupData;
boost::shared_array<int> rgMap; // maps input cols to output cols
boost::shared_array<int> projectionMap; // maps the projection steps to the output RG
bool hasRowGroup;
/* RowGroup support */
rowgroup::RowGroup outputRG;
boost::scoped_ptr<rowgroup::RGData> outRowGroupData;
boost::shared_array<int> rgMap; // maps input cols to output cols
boost::shared_array<int> projectionMap; // maps the projection steps to the output RG
bool hasRowGroup;
/* Rowgroups + join */
typedef std::tr1::unordered_multimap<uint64_t, uint32_t,
joiner::TupleJoiner::hasher, std::equal_to<uint64_t>,
utils::SimpleAllocator<std::pair<const uint64_t, uint32_t> > > TJoiner;
/* Rowgroups + join */
typedef std::tr1::unordered_multimap<uint64_t, uint32_t,
joiner::TupleJoiner::hasher, std::equal_to<uint64_t>,
utils::SimpleAllocator<std::pair<const uint64_t, uint32_t> > > TJoiner;
typedef std::tr1::unordered_multimap<joiner::TypelessData,
uint32_t, joiner::TupleJoiner::hasher> TLJoiner;
typedef std::tr1::unordered_multimap<joiner::TypelessData,
uint32_t, joiner::TupleJoiner::hasher> TLJoiner;
bool generateJoinedRowGroup(rowgroup::Row &baseRow, const uint32_t depth = 0);
/* generateJoinedRowGroup helper fcns & vars */
void initGJRG(); // called once after joining
void resetGJRG(); // called after every rowgroup returned by generateJRG
boost::scoped_array<uint32_t> gjrgPlaceHolders;
uint32_t gjrgRowNumber;
bool gjrgFull;
rowgroup::Row largeRow, joinedRow, baseJRow;
boost::scoped_array<uint8_t> baseJRowMem;
boost::scoped_ptr<rowgroup::RGData> joinedRGMem;
boost::scoped_array<rowgroup::Row> smallRows;
boost::shared_array<boost::shared_array<int> > gjrgMappings;
bool generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth = 0);
/* generateJoinedRowGroup helper fcns & vars */
void initGJRG(); // called once after joining
void resetGJRG(); // called after every rowgroup returned by generateJRG
boost::scoped_array<uint32_t> gjrgPlaceHolders;
uint32_t gjrgRowNumber;
bool gjrgFull;
rowgroup::Row largeRow, joinedRow, baseJRow;
boost::scoped_array<uint8_t> baseJRowMem;
boost::scoped_ptr<rowgroup::RGData> joinedRGMem;
boost::scoped_array<rowgroup::Row> smallRows;
boost::shared_array<boost::shared_array<int> > gjrgMappings;
boost::shared_array<boost::shared_ptr<TJoiner> > tJoiners;
typedef std::vector<uint32_t> MatchedData[LOGICAL_BLOCK_RIDS];
boost::shared_array<MatchedData> tSmallSideMatches;
void executeTupleJoin();
bool getTupleJoinRowGroupData;
std::vector<rowgroup::RowGroup> smallSideRGs;
rowgroup::RowGroup largeSideRG;
boost::shared_array<rowgroup::RGData> smallSideRowData;
boost::shared_array<rowgroup::RGData> smallNullRowData;
boost::shared_array<rowgroup::Row::Pointer> smallNullPointers;
boost::shared_array<uint64_t> ssrdPos; // this keeps track of position when building smallSideRowData
boost::shared_array<uint32_t> smallSideRowLengths;
boost::shared_array<joblist::JoinType> joinTypes;
uint32_t joinerCount;
boost::shared_array<uint32_t> tJoinerSizes;
// LSKC[i] = the column in outputRG joiner i uses as its key column
boost::shared_array<uint32_t> largeSideKeyColumns;
// KCPP[i] = true means a joiner uses projection step i as a key column
boost::shared_array<bool> keyColumnProj;
rowgroup::Row oldRow, newRow; // used by executeTupleJoin()
boost::shared_array<uint64_t> joinNullValues;
boost::shared_array<bool> doMatchNulls;
boost::scoped_array<boost::scoped_ptr<funcexp::FuncExpWrapper> > joinFEFilters;
bool hasJoinFEFilters;
bool hasSmallOuterJoin;
boost::shared_array<boost::shared_ptr<TJoiner> > tJoiners;
typedef std::vector<uint32_t> MatchedData[LOGICAL_BLOCK_RIDS];
boost::shared_array<MatchedData> tSmallSideMatches;
void executeTupleJoin();
bool getTupleJoinRowGroupData;
std::vector<rowgroup::RowGroup> smallSideRGs;
rowgroup::RowGroup largeSideRG;
boost::shared_array<rowgroup::RGData> smallSideRowData;
boost::shared_array<rowgroup::RGData> smallNullRowData;
boost::shared_array<rowgroup::Row::Pointer> smallNullPointers;
boost::shared_array<uint64_t> ssrdPos; // this keeps track of position when building smallSideRowData
boost::shared_array<uint32_t> smallSideRowLengths;
boost::shared_array<joblist::JoinType> joinTypes;
uint32_t joinerCount;
boost::shared_array<uint32_t> tJoinerSizes;
// LSKC[i] = the column in outputRG joiner i uses as its key column
boost::shared_array<uint32_t> largeSideKeyColumns;
// KCPP[i] = true means a joiner uses projection step i as a key column
boost::shared_array<bool> keyColumnProj;
rowgroup::Row oldRow, newRow; // used by executeTupleJoin()
boost::shared_array<uint64_t> joinNullValues;
boost::shared_array<bool> doMatchNulls;
boost::scoped_array<boost::scoped_ptr<funcexp::FuncExpWrapper> > joinFEFilters;
bool hasJoinFEFilters;
bool hasSmallOuterJoin;
/* extra typeless join vars & fcns*/
boost::shared_array<bool> typelessJoin;
boost::shared_array<std::vector<uint32_t> > tlLargeSideKeyColumns;
boost::shared_array<boost::shared_ptr<TLJoiner> > tlJoiners;
boost::shared_array<uint32_t> tlKeyLengths;
inline void getJoinResults(const rowgroup::Row &r, uint32_t jIndex, std::vector<uint32_t>& v);
// these allocators hold the memory for the keys stored in tlJoiners
boost::shared_array<utils::PoolAllocator> storedKeyAllocators;
// these allocators hold the memory for the large side keys which are short-lived
boost::scoped_array<utils::FixedAllocator> tmpKeyAllocators;
/* extra typeless join vars & fcns*/
boost::shared_array<bool> typelessJoin;
boost::shared_array<std::vector<uint32_t> > tlLargeSideKeyColumns;
boost::shared_array<boost::shared_ptr<TLJoiner> > tlJoiners;
boost::shared_array<uint32_t> tlKeyLengths;
inline void getJoinResults(const rowgroup::Row& r, uint32_t jIndex, std::vector<uint32_t>& v);
// these allocators hold the memory for the keys stored in tlJoiners
boost::shared_array<utils::PoolAllocator> storedKeyAllocators;
// these allocators hold the memory for the large side keys which are short-lived
boost::scoped_array<utils::FixedAllocator> tmpKeyAllocators;
/* PM Aggregation */
rowgroup::RowGroup joinedRG; // if there's a join, the rows are formatted with this
rowgroup::SP_ROWAGG_PM_t fAggregator;
rowgroup::RowGroup fAggregateRG;
rowgroup::RGData fAggRowGroupData;
//boost::scoped_array<uint8_t> fAggRowGroupData;
boost::shared_array<boost::shared_ptr<utils::SimplePool> > _pools;
/* PM Aggregation */
rowgroup::RowGroup joinedRG; // if there's a join, the rows are formatted with this
rowgroup::SP_ROWAGG_PM_t fAggregator;
rowgroup::RowGroup fAggregateRG;
rowgroup::RGData fAggRowGroupData;
//boost::scoped_array<uint8_t> fAggRowGroupData;
boost::shared_array<boost::shared_ptr<utils::SimplePool> > _pools;
/* OR hacks */
uint8_t bop; // BOP_AND or BOP_OR
bool hasPassThru;
uint8_t forHJ;
/* OR hacks */
uint8_t bop; // BOP_AND or BOP_OR
bool hasPassThru;
uint8_t forHJ;
boost::scoped_ptr<funcexp::FuncExpWrapper> fe1, fe2;
rowgroup::RowGroup fe1Input, fe2Output, *fe2Input;
// note, joinFERG is only for metadata, and is shared between BPPs
boost::shared_ptr<rowgroup::RowGroup> joinFERG;
boost::scoped_array<uint8_t> joinFERowData;
boost::scoped_ptr<rowgroup::RGData> fe1Data, fe2Data; // can probably make these RGDatas not pointers to RGDatas
boost::shared_array<int> projectForFE1;
boost::shared_array<int> fe1ToProjection, fe2Mapping; // RG mappings
boost::scoped_array<boost::shared_array<int> > joinFEMappings;
rowgroup::Row fe1In, fe1Out, fe2In, fe2Out, joinFERow;
boost::scoped_ptr<funcexp::FuncExpWrapper> fe1, fe2;
rowgroup::RowGroup fe1Input, fe2Output, *fe2Input;
// note, joinFERG is only for metadata, and is shared between BPPs
boost::shared_ptr<rowgroup::RowGroup> joinFERG;
boost::scoped_array<uint8_t> joinFERowData;
boost::scoped_ptr<rowgroup::RGData> fe1Data, fe2Data; // can probably make these RGDatas not pointers to RGDatas
boost::shared_array<int> projectForFE1;
boost::shared_array<int> fe1ToProjection, fe2Mapping; // RG mappings
boost::scoped_array<boost::shared_array<int> > joinFEMappings;
rowgroup::Row fe1In, fe1Out, fe2In, fe2Out, joinFERow;
bool hasDictStep;
bool hasDictStep;
primitives::PrimitiveProcessor pp;
primitives::PrimitiveProcessor pp;
/* VSS cache members */
VSSCache vssCache;
void buildVSSCache(uint32_t loopCount);
/* VSS cache members */
VSSCache vssCache;
void buildVSSCache(uint32_t loopCount);
/* To support limited DEC queues on the PM */
boost::shared_ptr<BPPSendThread> sendThread;
bool newConnection; // to support the load balancing code in sendThread
/* To support limited DEC queues on the PM */
boost::shared_ptr<BPPSendThread> sendThread;
bool newConnection; // to support the load balancing code in sendThread
/* To support reentrancy */
uint32_t currentBlockOffset;
boost::scoped_array<uint64_t> relLBID;
boost::scoped_array<bool> asyncLoaded;
/* To support reentrancy */
uint32_t currentBlockOffset;
boost::scoped_array<uint64_t> relLBID;
boost::scoped_array<bool> asyncLoaded;
/* To support a smaller memory footprint when idle */
static const uint64_t maxIdleBufferSize = 16*1024*1024; // arbitrary
void allocLargeBuffers();
void freeLargeBuffers();
/* To support a smaller memory footprint when idle */
static const uint64_t maxIdleBufferSize = 16 * 1024 * 1024; // arbitrary
void allocLargeBuffers();
void freeLargeBuffers();
/* To ensure all packets of an LBID go out the same socket */
int sockIndex;
/* To ensure all packets of an LBID go out the same socket */
int sockIndex;
/* Shared nothing vars */
uint32_t dbRoot;
/* Shared nothing vars */
uint32_t dbRoot;
bool endOfJoinerRan;
bool endOfJoinerRan;
friend class Command;
friend class ColumnCommand;
friend class DictStep;
friend class PassThruCommand;
friend class RTSCommand;
friend class FilterCommand;
friend class ScaledFilterCmd;
friend class StrFilterCmd;
friend class PseudoCC;
friend class Command;
friend class ColumnCommand;
friend class DictStep;
friend class PassThruCommand;
friend class RTSCommand;
friend class FilterCommand;
friend class ScaledFilterCmd;
friend class StrFilterCmd;
friend class PseudoCC;
};
}

510
primitives/primproc/bppseeder.cpp
View File

@ -54,13 +54,20 @@ using namespace std;
namespace primitiveprocessor
{
struct PTLogs{
PTLogs() {};
PTLogs(const int t, const char * fname):thdId(t) {logFD.open(fname, ios_base::app | ios_base::ate);}
~PTLogs() {logFD.close();}
struct PTLogs
{
PTLogs() {};
PTLogs(const int t, const char* fname): thdId(t)
{
logFD.open(fname, ios_base::app | ios_base::ate);
}
~PTLogs()
{
logFD.close();
}
int thdId;
ofstream logFD;
int thdId;
ofstream logFD;
};
typedef PTLogs PTLogs_t;
@ -70,45 +77,48 @@ typedef std::tr1::unordered_map<pthread_t, SPPTLogs_t> PTLogsMap_t;
PTLogsMap_t gFDList;
SPPTLogs_t gLogFD;
boost::mutex gFDMutex; //pthread_mutex_t gFDMutex=PTHREAD_MUTEX_INITIALIZER;
int gThdCnt=0;
int gThdCnt = 0;
extern dbbc::BlockRequestProcessor **BRPp;
extern dbbc::BlockRequestProcessor** BRPp;
extern int fCacheCount;
void timespec_sub(const struct timespec &tv1,
const struct timespec &tv2,
double &tm)
void timespec_sub(const struct timespec& tv1,
const struct timespec& tv2,
double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9*(tv2.tv_nsec - tv1.tv_nsec);
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
BPPSeeder::BPPSeeder(const SBS &b,
const SP_UM_MUTEX& w,
const SP_UM_IOSOCK& s,
const int pmThreads,
const bool trace) :
bs(b), writelock(w), sock(s), fPMThreads(pmThreads), fTrace(trace),
failCount(0),
firstRun(true)
BPPSeeder::BPPSeeder(const SBS& b,
const SP_UM_MUTEX& w,
const SP_UM_IOSOCK& s,
const int pmThreads,
const bool trace) :
bs(b), writelock(w), sock(s), fPMThreads(pmThreads), fTrace(trace),
failCount(0),
firstRun(true)
{
uint8_t *buf = b->buf();
uint32_t pos = sizeof(ISMPacketHeader);
uint8_t* buf = b->buf();
uint32_t pos = sizeof(ISMPacketHeader);
sessionID = *((uint32_t *) &buf[pos]); pos += 4;
stepID = *((uint32_t *) &buf[pos]); pos += 4;
uniqueID = *((uint32_t *) &buf[pos]); pos +=4;
_priority = *((uint32_t *) &buf[pos]);
sessionID = *((uint32_t*) &buf[pos]);
pos += 4;
stepID = *((uint32_t*) &buf[pos]);
pos += 4;
uniqueID = *((uint32_t*) &buf[pos]);
pos += 4;
_priority = *((uint32_t*) &buf[pos]);
dieTime = boost::posix_time::second_clock::universal_time() +
boost::posix_time::seconds(100);
dieTime = boost::posix_time::second_clock::universal_time() +
boost::posix_time::seconds(100);
}
BPPSeeder::BPPSeeder(const BPPSeeder &b)
: bs(b.bs), writelock(b.writelock), sock(b.sock),
fPMThreads(b.fPMThreads), fTrace(b.fTrace), uniqueID(b.uniqueID),
sessionID(b.sessionID), stepID(b.stepID), failCount(b.failCount), bpp(b.bpp),
firstRun(b.firstRun), _priority(b._priority)
BPPSeeder::BPPSeeder(const BPPSeeder& b)
: bs(b.bs), writelock(b.writelock), sock(b.sock),
fPMThreads(b.fPMThreads), fTrace(b.fTrace), uniqueID(b.uniqueID),
sessionID(b.sessionID), stepID(b.stepID), failCount(b.failCount), bpp(b.bpp),
firstRun(b.firstRun), _priority(b._priority)
{
}
@ -118,250 +128,293 @@ BPPSeeder::~BPPSeeder()
int BPPSeeder::operator()()
{
uint32_t pos;
const uint8_t *buf = bs->buf();
BPPMap::iterator it;
ostringstream logData;
struct timespec tm;
struct timespec tm2;
double tm3=0;
bool ptLock=false;
bool gotBPP = false;
PTLogs_t* logFD=NULL;
int ret = 0;
pthread_t tid=0;
boost::mutex::scoped_lock scoped(bppLock, boost::defer_lock_t());
uint32_t pos;
const uint8_t* buf = bs->buf();
BPPMap::iterator it;
ostringstream logData;
struct timespec tm;
struct timespec tm2;
double tm3 = 0;
bool ptLock = false;
bool gotBPP = false;
PTLogs_t* logFD = NULL;
int ret = 0;
pthread_t tid = 0;
boost::mutex::scoped_lock scoped(bppLock, boost::defer_lock_t());
try {
if (firstRun) {
pos = sizeof(ISMPacketHeader) - 2;
uint16_t status = *((uint16_t *) &buf[pos]); pos += 2;
try
{
if (firstRun)
{
pos = sizeof(ISMPacketHeader) - 2;
uint16_t status = *((uint16_t*) &buf[pos]);
pos += 2;
sessionID = *((uint32_t *) &buf[pos]); pos += 4;
stepID = *((uint32_t *) &buf[pos]); pos += 4;
uniqueID = *((uint32_t *) &buf[pos]); pos += 4;
_priority = *((uint32_t *) &buf[pos]);
sessionID = *((uint32_t*) &buf[pos]);
pos += 4;
stepID = *((uint32_t*) &buf[pos]);
pos += 4;
uniqueID = *((uint32_t*) &buf[pos]);
pos += 4;
_priority = *((uint32_t*) &buf[pos]);
if (0 < status)
{
sendErrorMsg(uniqueID, status, stepID);
return ret;
}
if (0 < status)
{
sendErrorMsg(uniqueID, status, stepID);
return ret;
}
//if (!(sessionID & 0x80000000))
//cout << "got request for <" << sessionID <<", " << stepID << ">\n";
scoped.lock();
if (!bppv) {
it = bppMap.find(uniqueID);
if (it == bppMap.end()) {
/* mitigate a small race between creation and use */
scoped.unlock();
if (boost::posix_time::second_clock::universal_time() > dieTime) {
//if (!(sessionID & 0x80000000))
//cout << "got request for <" << sessionID <<", " << stepID << ">\n";
scoped.lock();
if (!bppv)
{
it = bppMap.find(uniqueID);
if (it == bppMap.end())
{
/* mitigate a small race between creation and use */
scoped.unlock();
if (boost::posix_time::second_clock::universal_time() > dieTime)
{
#if 0 // for debugging
#ifndef _MSC_VER
boost::posix_time::ptime pt = boost::posix_time::microsec_clock::local_time();
if (sessionID & 0x80000000)
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< (int) (sessionID ^ 0x80000000) << " stepID=" << stepID << " (syscat)" << pt << endl;
else
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< sessionID << " stepID=" << stepID << pt << endl;
boost::posix_time::ptime pt = boost::posix_time::microsec_clock::local_time();
if (sessionID & 0x80000000)
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< (int) (sessionID ^ 0x80000000) << " stepID=" << stepID << " (syscat)" << pt << endl;
else
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< sessionID << " stepID=" << stepID << pt << endl;
#endif
throw logic_error("BPPSeeder couldn't find the sessionID/stepID pair");
throw logic_error("BPPSeeder couldn't find the sessionID/stepID pair");
#endif
return 0;
}
return 0;
}
// if (!isSysCat())
return -1;
return -1;
// else { // syscat queries aren't run by a threadpool, can't reschedule those jobs
// usleep(1000);
// goto retry;
// }
}
bppv = it->second;
}
if (bppv->aborted())
return 0;
bpp = bppv->next();
scoped.unlock();
if (!bpp) {
}
bppv = it->second;
}
if (bppv->aborted())
return 0;
bpp = bppv->next();
scoped.unlock();
if (!bpp)
{
// if (isSysCat()) {
// usleep(1000);
// goto retry;
// }
return -1; // all BPP instances are busy, make threadpool reschedule
}
gotBPP = true;
bpp->resetBPP(*bs, writelock, sock);
firstRun = false;
} // firstRun
return -1; // all BPP instances are busy, make threadpool reschedule
}
gotBPP = true;
bpp->resetBPP(*bs, writelock, sock);
firstRun = false;
} // firstRun
if (fTrace)
{
PTLogsMap_t::iterator it;
if (fTrace)
{
PTLogsMap_t::iterator it;
#ifdef _MSC_VER
tid = GetCurrentThreadId();
tid = GetCurrentThreadId();
#else
tid = pthread_self();
tid = pthread_self();
#endif
// only lock map while inserted objects
// once there is an object for each thread
// there is not need to lock
if (gFDList.size()<(uint32_t)fPMThreads) {
gFDMutex.lock();
ptLock=true;
}
// only lock map while inserted objects
// once there is an object for each thread
// there is not need to lock
if (gFDList.size() < (uint32_t)fPMThreads)
{
gFDMutex.lock();
ptLock = true;
}
it = gFDList.find(tid);
if (it==gFDList.end())
{
ostringstream LogFileName;
SPPTLogs_t spof;
it = gFDList.find(tid);
if (it == gFDList.end())
{
ostringstream LogFileName;
SPPTLogs_t spof;
#ifdef _MSC_VER
LogFileName << "C:/Calpont/log/trace/pt." << tid;
LogFileName << "C:/Calpont/log/trace/pt." << tid;
#else
LogFileName << "/var/log/mariadb/columnstore/trace/pt." << tid;
LogFileName << "/var/log/mariadb/columnstore/trace/pt." << tid;
#endif
spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str()));
gThdCnt++;
// TODO: add error checking
if (spof->logFD.is_open()) {
gFDList[tid] = spof;
logFD = spof.get();
}
} else
logFD =(*it).second.get();
spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str()));
gThdCnt++;
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
clock_gettime(CLOCK_MONOTONIC, &tm);
} // if (fTrace)
// TODO: add error checking
if (spof->logFD.is_open())
{
gFDList[tid] = spof;
logFD = spof.get();
}
}
else
logFD = (*it).second.get();
uint32_t retries = 0;
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
clock_gettime(CLOCK_MONOTONIC, &tm);
} // if (fTrace)
uint32_t retries = 0;
restart:
try {
ret = (*bpp)();
}
catch (NeedToRestartJob &e) {
ostringstream os;
// experimentally the race can exist longer than 10s. "No way" should
// it take 10 minutes. If it does, the user will have to resubmit their
// query.
// 9/27/12 - changed the timeout to 2 mins b/c people report the system
// is hung if it does nothing for 10 mins. 2 mins should still be more
// than enough
if (++retries == 120) {
os << e.what() << ": Restarted a syscat job " << retries << " times, bailing\n";
throw NeedToRestartJob(os.str());
}
flushSyscatOIDs();
bs->rewind();
bpp->resetBPP(*bs, writelock, sock);
sleep(1);
goto restart;
}
try
{
ret = (*bpp)();
}
catch (NeedToRestartJob& e)
{
ostringstream os;
// experimentally the race can exist longer than 10s. "No way" should
// it take 10 minutes. If it does, the user will have to resubmit their
// query.
if (ret)
return ret;
// 9/27/12 - changed the timeout to 2 mins b/c people report the system
// is hung if it does nothing for 10 mins. 2 mins should still be more
// than enough
if (++retries == 120)
{
os << e.what() << ": Restarted a syscat job " << retries << " times, bailing\n";
throw NeedToRestartJob(os.str());
}
if (fTrace)
if (logFD && logFD->logFD.is_open()) {
clock_gettime(CLOCK_MONOTONIC, &tm2);
timespec_sub(tm, tm2, tm3);
logFD->logFD
<< left << setw(3) << logFD->thdId
<< right << fixed << ((double)(tm.tv_sec+(1.e-9*tm.tv_nsec))) << " "
<< right << fixed << tm3 << " "
<< right << setw(6) << bpp->getSessionID() << " "
<< right << setw(4) << bpp->getStepID() << " "
<< right << setw(2) << bpp->FilterCount() << " "
<< right << setw(2) << bpp->ProjectCount() << " "
<< right << setw(9) << bpp->PhysIOCount() << " "
<< right << setw(9) << bpp->CachedIOCount() << " "
<< right << setw(9) << bpp->BlocksTouchedCount()
<< endl;
} // if (logFD...
flushSyscatOIDs();
bs->rewind();
bpp->resetBPP(*bs, writelock, sock);
sleep(1);
goto restart;
}
}
catch (scalar_exception &se)
{
if (gotBPP)
bpp->busy(false);
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
}
catch(exception& ex)
{
if (gotBPP)
bpp->busy(false);
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
catchHandler(ex.what(), uniqueID, stepID);
cout << "BPPSeeder step " << stepID << " caught an exception: " << ex.what() << endl;
}
catch(...)
{
if (gotBPP)
bpp->busy(false);
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
string msg("BPPSeeder caught an unknown exception");
catchHandler(msg, uniqueID, stepID);
cout << msg << endl;
}
return ret;
if (ret)
return ret;
if (fTrace)
if (logFD && logFD->logFD.is_open())
{
clock_gettime(CLOCK_MONOTONIC, &tm2);
timespec_sub(tm, tm2, tm3);
logFD->logFD
<< left << setw(3) << logFD->thdId
<< right << fixed << ((double)(tm.tv_sec + (1.e-9 * tm.tv_nsec))) << " "
<< right << fixed << tm3 << " "
<< right << setw(6) << bpp->getSessionID() << " "
<< right << setw(4) << bpp->getStepID() << " "
<< right << setw(2) << bpp->FilterCount() << " "
<< right << setw(2) << bpp->ProjectCount() << " "
<< right << setw(9) << bpp->PhysIOCount() << " "
<< right << setw(9) << bpp->CachedIOCount() << " "
<< right << setw(9) << bpp->BlocksTouchedCount()
<< endl;
} // if (logFD...
}
catch (scalar_exception& se)
{
if (gotBPP)
bpp->busy(false);
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
}
catch (exception& ex)
{
if (gotBPP)
bpp->busy(false);
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
catchHandler(ex.what(), uniqueID, stepID);
cout << "BPPSeeder step " << stepID << " caught an exception: " << ex.what() << endl;
}
catch (...)
{
if (gotBPP)
bpp->busy(false);
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
string msg("BPPSeeder caught an unknown exception");
catchHandler(msg, uniqueID, stepID);
cout << msg << endl;
}
return ret;
}
void BPPSeeder::catchHandler(const string& ex, uint32_t id, uint32_t step)
{
Logger log;
log.logMessage(ex);
sendErrorMsg(id, logging::bppSeederErr, step);
Logger log;
log.logMessage(ex);
sendErrorMsg(id, logging::bppSeederErr, step);
}
void BPPSeeder::sendErrorMsg(uint32_t id, uint16_t status, uint32_t step)
{
ISMPacketHeader ism;
PrimitiveHeader ph = {0};
ISMPacketHeader ism;
PrimitiveHeader ph = {0};
ism.Status = status;
ph.UniqueID = id;
ph.StepID = step;
ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader));
msg.append((uint8_t *) &ism, sizeof(ism));
msg.append((uint8_t *) &ph, sizeof(ph));
ism.Status = status;
ph.UniqueID = id;
ph.StepID = step;
ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader));
msg.append((uint8_t*) &ism, sizeof(ism));
msg.append((uint8_t*) &ph, sizeof(ph));
boost::mutex::scoped_lock lk(*writelock);
sock->write(msg);
boost::mutex::scoped_lock lk(*writelock);
sock->write(msg);
}
bool BPPSeeder::isSysCat()
{
const uint8_t *buf;
uint32_t sessionIDOffset = sizeof(ISMPacketHeader);
uint32_t sessionID;
const uint8_t* buf;
uint32_t sessionIDOffset = sizeof(ISMPacketHeader);
uint32_t sessionID;
buf = bs->buf();
sessionID = *((uint32_t *) &buf[sessionIDOffset]);
return (sessionID & 0x80000000);
buf = bs->buf();
sessionID = *((uint32_t*) &buf[sessionIDOffset]);
return (sessionID & 0x80000000);
}
uint32_t BPPSeeder::getID()
{
return uniqueID;
return uniqueID;
}
/* This is part of the syscat-retry hack. We should get rid of it once we
@ -369,14 +422,15 @@ uint32_t BPPSeeder::getID()
*/
void BPPSeeder::flushSyscatOIDs()
{
vector<BRM::OID_t> syscatOIDs;
vector<BRM::OID_t> syscatOIDs;
syscatOIDs = execplan::getAllSysCatOIDs();
syscatOIDs = execplan::getAllSysCatOIDs();
for (int i = 0; i < fCacheCount; i++) {
dbbc::blockCacheClient bc(*BRPp[i]);
bc.flushOIDs((const uint32_t *) &syscatOIDs[0], syscatOIDs.size());
}
for (int i = 0; i < fCacheCount; i++)
{
dbbc::blockCacheClient bc(*BRPp[i]);
bc.flushOIDs((const uint32_t*) &syscatOIDs[0], syscatOIDs.size());
}
}
};

68
primitives/primproc/bppseeder.h
View File

@ -50,46 +50,52 @@ namespace primitiveprocessor
{
class BPPSeeder : public threadpool::PriorityThreadPool::Functor
{
public:
BPPSeeder(const messageqcpp::SBS &,
const SP_UM_MUTEX& wLock,
const SP_UM_IOSOCK& ios,
const int pmThreads,
const bool trace=false);
BPPSeeder(const BPPSeeder &b);
public:
BPPSeeder(const messageqcpp::SBS&,
const SP_UM_MUTEX& wLock,
const SP_UM_IOSOCK& ios,
const int pmThreads,
const bool trace = false);
BPPSeeder(const BPPSeeder& b);
virtual ~BPPSeeder();
virtual ~BPPSeeder();
int operator()();
int operator()();
bool isSysCat();
boost::shared_ptr<std::ofstream> spof;
bool isSysCat();
boost::shared_ptr<std::ofstream> spof;
uint32_t getID();
uint32_t getID();
void priority(uint32_t p) { _priority = p; }
uint32_t priority() { return _priority; }
void priority(uint32_t p)
{
_priority = p;
}
uint32_t priority()
{
return _priority;
}
private:
BPPSeeder();
void catchHandler(const std::string& s, uint32_t uniqueID, uint32_t step);
void sendErrorMsg(uint32_t id, uint16_t status, uint32_t step);
void flushSyscatOIDs();
private:
BPPSeeder();
void catchHandler(const std::string& s, uint32_t uniqueID, uint32_t step);
void sendErrorMsg(uint32_t id, uint16_t status, uint32_t step);
void flushSyscatOIDs();
messageqcpp::SBS bs;
SP_UM_MUTEX writelock;
SP_UM_IOSOCK sock;
int fPMThreads;
bool fTrace;
messageqcpp::SBS bs;
SP_UM_MUTEX writelock;
SP_UM_IOSOCK sock;
int fPMThreads;
bool fTrace;
/* To support reentrancy */
uint32_t uniqueID, sessionID, stepID, failCount;
boost::shared_ptr<BatchPrimitiveProcessor> bpp;
SBPPV bppv;
bool firstRun;
boost::posix_time::ptime dieTime;
/* To support reentrancy */
uint32_t uniqueID, sessionID, stepID, failCount;
boost::shared_ptr<BatchPrimitiveProcessor> bpp;
SBPPV bppv;
bool firstRun;
boost::posix_time::ptime dieTime;
uint32_t _priority;
uint32_t _priority;
};
};

341
primitives/primproc/bppsendthread.cpp
View File

@ -32,196 +32,241 @@ using namespace boost;
namespace primitiveprocessor
{
extern uint32_t connectionsPerUM;
BPPSendThread::BPPSendThread() : die(false), gotException(false), mainThreadWaiting(false),
sizeThreshold(100), msgsLeft(-1), waiting(false), sawAllConnections(false),
fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
sizeThreshold(100), msgsLeft(-1), waiting(false), sawAllConnections(false),
fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
{
runner = boost::thread(Runner_t(this));
runner = boost::thread(Runner_t(this));
}
BPPSendThread::BPPSendThread(uint32_t initMsgsLeft) : die(false), gotException(false),
mainThreadWaiting(false), sizeThreshold(100), msgsLeft(initMsgsLeft), waiting(false),
sawAllConnections(false), fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
mainThreadWaiting(false), sizeThreshold(100), msgsLeft(initMsgsLeft), waiting(false),
sawAllConnections(false), fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
{
runner = boost::thread(Runner_t(this));
runner = boost::thread(Runner_t(this));
}
BPPSendThread::~BPPSendThread()
{
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
runner.join();
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
runner.join();
}
bool BPPSendThread::okToProceed()
{
// keep the queue size below the 100 msg threshold & below the 25MB mark,
// but at least 2 msgs so there is always 1 ready to be sent.
return ((msgQueue.size() < sizeThreshold && currentByteSize < maxByteSize)
|| msgQueue.size() < 3) && !die;
// keep the queue size below the 100 msg threshold & below the 25MB mark,
// but at least 2 msgs so there is always 1 ready to be sent.
return ((msgQueue.size() < sizeThreshold && currentByteSize < maxByteSize)
|| msgQueue.size() < 3) && !die;
}
void BPPSendThread::sendResult(const Msg_t &msg, bool newConnection)
void BPPSendThread::sendResult(const Msg_t& msg, bool newConnection)
{
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msg.msg->lengthWithHdrOverhead());
msgQueue.push(msg);
if (!sawAllConnections && newConnection) {
Connection_t ins(msg.sockLock, msg.sock);
bool inserted = connections_s.insert(ins).second;
if (inserted) {
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM) {
connections_s.clear();
sawAllConnections = true;
}
}
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msg.msg->lengthWithHdrOverhead());
msgQueue.push(msg);
if (!sawAllConnections && newConnection)
{
Connection_t ins(msg.sockLock, msg.sock);
bool inserted = connections_s.insert(ins).second;
if (inserted)
{
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM)
{
connections_s.clear();
sawAllConnections = true;
}
}
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
}
void BPPSendThread::sendResults(const vector<Msg_t> &msgs, bool newConnection)
void BPPSendThread::sendResults(const vector<Msg_t>& msgs, bool newConnection)
{
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
if (!sawAllConnections && newConnection) {
idbassert(msgs.size() > 0);
Connection_t ins(msgs[0].sockLock, msgs[0].sock);
bool inserted = connections_s.insert(ins).second;
if (inserted) {
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM) {
connections_s.clear();
sawAllConnections = true;
}
}
}
for (uint32_t i = 0; i < msgs.size(); i++) {
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msgs[i].msg->lengthWithHdrOverhead());
msgQueue.push(msgs[i]);
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
if (!sawAllConnections && newConnection)
{
idbassert(msgs.size() > 0);
Connection_t ins(msgs[0].sockLock, msgs[0].sock);
bool inserted = connections_s.insert(ins).second;
if (inserted)
{
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM)
{
connections_s.clear();
sawAllConnections = true;
}
}
}
for (uint32_t i = 0; i < msgs.size(); i++)
{
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msgs[i].msg->lengthWithHdrOverhead());
msgQueue.push(msgs[i]);
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
}
void BPPSendThread::sendMore(int num)
{
mutex::scoped_lock sl(ackLock);
mutex::scoped_lock sl(ackLock);
// cout << "got an ACK for " << num << " msgsLeft=" << msgsLeft << endl;
if (num == -1)
fcEnabled = false;
else if (num == 0) {
fcEnabled = true;
msgsLeft = 0;
}
else
(void)atomicops::atomicAdd(&msgsLeft, num);
if (waiting)
okToSend.notify_one();
if (num == -1)
fcEnabled = false;
else if (num == 0)
{
fcEnabled = true;
msgsLeft = 0;
}
else
(void)atomicops::atomicAdd(&msgsLeft, num);
if (waiting)
okToSend.notify_one();
}
bool BPPSendThread::flowControlEnabled()
{
return fcEnabled;
return fcEnabled;
}
void BPPSendThread::mainLoop()
{
const uint32_t msgCap = 20;
boost::scoped_array<Msg_t> msg;
uint32_t msgCount = 0, i, msgsSent;
SP_UM_MUTEX lock;
SP_UM_IOSOCK sock;
bool doLoadBalancing = false;
const uint32_t msgCap = 20;
boost::scoped_array<Msg_t> msg;
uint32_t msgCount = 0, i, msgsSent;
SP_UM_MUTEX lock;
SP_UM_IOSOCK sock;
bool doLoadBalancing = false;
msg.reset(new Msg_t[msgCap]);
msg.reset(new Msg_t[msgCap]);
while (!die) {
mutex::scoped_lock sl(msgQueueLock);
if (msgQueue.empty() && !die) {
mainThreadWaiting = true;
queueNotEmpty.wait(sl);
mainThreadWaiting = false;
continue;
}
while (!die)
{
mutex::scoped_lock sl(msgQueueLock);
msgCount = (msgQueue.size() > msgCap ? msgCap : msgQueue.size());
for (i = 0; i < msgCount; i++) {
msg[i] = msgQueue.front();
msgQueue.pop();
}
doLoadBalancing = sawAllConnections;
sl.unlock();
if (msgQueue.empty() && !die)
{
mainThreadWaiting = true;
queueNotEmpty.wait(sl);
mainThreadWaiting = false;
continue;
}
/* In the send loop below, msgsSent tracks progress on sending the msg array,
* i how many msgs are sent by 1 run of the loop, limited by msgCount or msgsLeft. */
msgsSent = 0;
while (msgsSent < msgCount && !die) {
uint64_t bsSize;
if (msgsLeft <= 0 && fcEnabled && !die) {
mutex::scoped_lock sl2(ackLock);
while (msgsLeft <= 0 && fcEnabled && !die) {
waiting = true;
okToSend.wait(sl2);
waiting = false;
}
}
for (i = 0; msgsSent < msgCount && ((fcEnabled && msgsLeft > 0) || !fcEnabled) && !die;
msgsSent++, i++) {
if (doLoadBalancing) {
// Bug 4475 move control of sockIndex to batchPrimitiveProcessor
lock = connections_v[msg[msgsSent].sockIndex].sockLock;
sock = connections_v[msg[msgsSent].sockIndex].sock;
}
else {
lock = msg[msgsSent].sockLock;
sock = msg[msgsSent].sock;
}
bsSize = msg[msgsSent].msg->lengthWithHdrOverhead();
try {
mutex::scoped_lock sl2(*lock);
sock->write(*msg[msgsSent].msg);
//cout << "sent 1 msg\n";
}
catch (std::exception &e) {
sl.lock();
exceptionString = e.what();
gotException = true;
return;
}
(void)atomicops::atomicDec(&msgsLeft);
(void)atomicops::atomicSub(&currentByteSize, bsSize);
msg[msgsSent].msg.reset();
}
}
}
msgCount = (msgQueue.size() > msgCap ? msgCap : msgQueue.size());
for (i = 0; i < msgCount; i++)
{
msg[i] = msgQueue.front();
msgQueue.pop();
}
doLoadBalancing = sawAllConnections;
sl.unlock();
/* In the send loop below, msgsSent tracks progress on sending the msg array,
* i how many msgs are sent by 1 run of the loop, limited by msgCount or msgsLeft. */
msgsSent = 0;
while (msgsSent < msgCount && !die)
{
uint64_t bsSize;
if (msgsLeft <= 0 && fcEnabled && !die)
{
mutex::scoped_lock sl2(ackLock);
while (msgsLeft <= 0 && fcEnabled && !die)
{
waiting = true;
okToSend.wait(sl2);
waiting = false;
}
}
for (i = 0; msgsSent < msgCount && ((fcEnabled && msgsLeft > 0) || !fcEnabled) && !die;
msgsSent++, i++)
{
if (doLoadBalancing)
{
// Bug 4475 move control of sockIndex to batchPrimitiveProcessor
lock = connections_v[msg[msgsSent].sockIndex].sockLock;
sock = connections_v[msg[msgsSent].sockIndex].sock;
}
else
{
lock = msg[msgsSent].sockLock;
sock = msg[msgsSent].sock;
}
bsSize = msg[msgsSent].msg->lengthWithHdrOverhead();
try
{
mutex::scoped_lock sl2(*lock);
sock->write(*msg[msgsSent].msg);
//cout << "sent 1 msg\n";
}
catch (std::exception& e)
{
sl.lock();
exceptionString = e.what();
gotException = true;
return;
}
(void)atomicops::atomicDec(&msgsLeft);
(void)atomicops::atomicSub(&currentByteSize, bsSize);
msg[msgsSent].msg.reset();
}
}
}
}
void BPPSendThread::abort()
{
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
}
}

155
primitives/primproc/bppsendthread.h
View File

@ -35,79 +35,96 @@
namespace primitiveprocessor
{
class BPPSendThread {
class BPPSendThread
{
public:
BPPSendThread(); // starts unthrottled
BPPSendThread(uint32_t initMsgsLeft); // starts throttled
virtual ~BPPSendThread();
public:
BPPSendThread(); // starts unthrottled
BPPSendThread(uint32_t initMsgsLeft); // starts throttled
virtual ~BPPSendThread();
struct Msg_t {
messageqcpp::SBS msg;
SP_UM_IOSOCK sock;
SP_UM_MUTEX sockLock;
int sockIndex; // Socket index for round robin control. Bug 4475
Msg_t() : sockIndex(0) { }
Msg_t(const Msg_t &m) : msg(m.msg), sock(m.sock), sockLock(m.sockLock), sockIndex(m.sockIndex) { }
Msg_t & operator=(const Msg_t &m)
{ msg = m.msg; sock = m.sock; sockLock = m.sockLock; sockIndex = m.sockIndex; return *this; }
Msg_t(const messageqcpp::SBS &m, const SP_UM_IOSOCK &so, const SP_UM_MUTEX &sl, int si) :
msg(m), sock(so), sockLock(sl), sockIndex(si) { }
};
struct Msg_t
{
messageqcpp::SBS msg;
SP_UM_IOSOCK sock;
SP_UM_MUTEX sockLock;
int sockIndex; // Socket index for round robin control. Bug 4475
Msg_t() : sockIndex(0) { }
Msg_t(const Msg_t& m) : msg(m.msg), sock(m.sock), sockLock(m.sockLock), sockIndex(m.sockIndex) { }
Msg_t& operator=(const Msg_t& m)
{
msg = m.msg;
sock = m.sock;
sockLock = m.sockLock;
sockIndex = m.sockIndex;
return *this;
}
Msg_t(const messageqcpp::SBS& m, const SP_UM_IOSOCK& so, const SP_UM_MUTEX& sl, int si) :
msg(m), sock(so), sockLock(sl), sockIndex(si) { }
};
bool okToProceed();
void sendMore(int num);
void sendResults(const std::vector<Msg_t> &msgs, bool newConnection);
void sendResult(const Msg_t &msg, bool newConnection);
void mainLoop();
bool flowControlEnabled();
void abort();
inline bool aborted() const
{
return die;
}
private:
BPPSendThread(const BPPSendThread&);
BPPSendThread& operator=(const BPPSendThread&);
bool okToProceed();
void sendMore(int num);
void sendResults(const std::vector<Msg_t>& msgs, bool newConnection);
void sendResult(const Msg_t& msg, bool newConnection);
void mainLoop();
bool flowControlEnabled();
void abort();
inline bool aborted() const
{
return die;
}
struct Runner_t {
BPPSendThread *bppst;
Runner_t(BPPSendThread *b) : bppst(b) { }
void operator()() { bppst->mainLoop(); }
};
boost::thread runner;
std::queue<Msg_t> msgQueue;
boost::mutex msgQueueLock;
boost::condition queueNotEmpty;
volatile bool die, gotException, mainThreadWaiting;
std::string exceptionString;
uint32_t sizeThreshold;
volatile int32_t msgsLeft;
bool waiting;
boost::mutex ackLock;
boost::condition okToSend;
/* Load balancing structures */
struct Connection_t {
Connection_t(const SP_UM_MUTEX &lock, const SP_UM_IOSOCK &so) :
sockLock(lock), sock(so) { }
SP_UM_MUTEX sockLock;
SP_UM_IOSOCK sock;
bool operator<(const Connection_t &t) const
{ return sockLock.get() < t.sockLock.get(); }
bool operator==(const Connection_t &t) const
{ return sockLock.get() == t.sockLock.get(); }
};
std::set<Connection_t> connections_s;
std::vector<Connection_t> connections_v;
bool sawAllConnections;
volatile bool fcEnabled;
/* secondary queue size restriction based on byte size */
volatile uint64_t currentByteSize;
uint64_t maxByteSize;
private:
BPPSendThread(const BPPSendThread&);
BPPSendThread& operator=(const BPPSendThread&);
struct Runner_t
{
BPPSendThread* bppst;
Runner_t(BPPSendThread* b) : bppst(b) { }
void operator()()
{
bppst->mainLoop();
}
};
boost::thread runner;
std::queue<Msg_t> msgQueue;
boost::mutex msgQueueLock;
boost::condition queueNotEmpty;
volatile bool die, gotException, mainThreadWaiting;
std::string exceptionString;
uint32_t sizeThreshold;
volatile int32_t msgsLeft;
bool waiting;
boost::mutex ackLock;
boost::condition okToSend;
/* Load balancing structures */
struct Connection_t
{
Connection_t(const SP_UM_MUTEX& lock, const SP_UM_IOSOCK& so) :
sockLock(lock), sock(so) { }
SP_UM_MUTEX sockLock;
SP_UM_IOSOCK sock;
bool operator<(const Connection_t& t) const
{
return sockLock.get() < t.sockLock.get();
}
bool operator==(const Connection_t& t) const
{
return sockLock.get() == t.sockLock.get();
}
};
std::set<Connection_t> connections_s;
std::vector<Connection_t> connections_v;
bool sawAllConnections;
volatile bool fcEnabled;
/* secondary queue size restriction based on byte size */
volatile uint64_t currentByteSize;
uint64_t maxByteSize;
};
}

1362
primitives/primproc/columncommand.cpp
View File

File diff suppressed because it is too large Load Diff

179
primitives/primproc/columncommand.h
View File

@ -41,105 +41,132 @@ namespace primitiveprocessor
class ColumnCommand : public Command
{
public:
ColumnCommand();
virtual ~ColumnCommand();
ColumnCommand();
virtual ~ColumnCommand();
inline uint64_t getLBID() { return lbid; }
inline uint8_t getWidth() { return colType.colWidth; }
inline uint8_t getScale() { return colType.scale; }
uint16_t getFilterCount() { return filterCount; }
const execplan::CalpontSystemCatalog::ColType& getColType() { return colType; }
inline uint64_t getLBID()
{
return lbid;
}
inline uint8_t getWidth()
{
return colType.colWidth;
}
inline uint8_t getScale()
{
return colType.scale;
}
uint16_t getFilterCount()
{
return filterCount;
}
const execplan::CalpontSystemCatalog::ColType& getColType()
{
return colType;
}
void execute();
void execute(int64_t *vals); //used by RTSCommand to redirect values
void prep(int8_t outputType, bool makeAbsRids);
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t pos);
void nextLBID();
bool isScan() { return _isScan; }
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
void setMakeAbsRids(bool m) { makeAbsRids = m; }
bool willPrefetch();
const uint64_t getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const;
const int64_t getLastLbid();
void getLBIDList(uint32_t loopCount, std::vector<int64_t> *lbids);
void execute();
void execute(int64_t* vals); //used by RTSCommand to redirect values
void prep(int8_t outputType, bool makeAbsRids);
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t pos);
void nextLBID();
bool isScan()
{
return _isScan;
}
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
void setMakeAbsRids(bool m)
{
makeAbsRids = m;
}
bool willPrefetch();
const uint64_t getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const;
const int64_t getLastLbid();
void getLBIDList(uint32_t loopCount, std::vector<int64_t>* lbids);
virtual SCommand duplicate();
bool operator==(const ColumnCommand &) const;
bool operator!=(const ColumnCommand &) const;
virtual SCommand duplicate();
bool operator==(const ColumnCommand&) const;
bool operator!=(const ColumnCommand&) const;
/* OR hacks */
void setScan(bool b) { _isScan = b; }
void disableFilters();
void enableFilters();
/* OR hacks */
void setScan(bool b)
{
_isScan = b;
}
void disableFilters();
void enableFilters();
int getCompType() const { return colType.compressionType; }
int getCompType() const
{
return colType.compressionType;
}
protected:
virtual void loadData();
void duplicate(ColumnCommand *);
virtual void loadData();
void duplicate(ColumnCommand*);
// we only care about the width and type fields.
//On the PM the rest is uninitialized
execplan::CalpontSystemCatalog::ColType colType;
// we only care about the width and type fields.
//On the PM the rest is uninitialized
execplan::CalpontSystemCatalog::ColType colType;
private:
ColumnCommand(const ColumnCommand &);
ColumnCommand& operator=(const ColumnCommand &);
ColumnCommand(const ColumnCommand&);
ColumnCommand& operator=(const ColumnCommand&);
void _execute();
void issuePrimitive();
void processResult();
void process_OT_BOTH();
void process_OT_RID();
void process_OT_DATAVALUE();
void process_OT_ROWGROUP();
void projectResult();
void projectResultRG(rowgroup::RowGroup &rg, uint32_t pos);
void removeRowsFromRowGroup(rowgroup::RowGroup &);
void makeScanMsg();
void makeStepMsg();
void setLBID(uint64_t rid);
void _execute();
void issuePrimitive();
void processResult();
void process_OT_BOTH();
void process_OT_RID();
void process_OT_DATAVALUE();
void process_OT_ROWGROUP();
void projectResult();
void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos);
void removeRowsFromRowGroup(rowgroup::RowGroup&);
void makeScanMsg();
void makeStepMsg();
void setLBID(uint64_t rid);
bool _isScan;
bool _isScan;
boost::scoped_array<uint8_t> inputMsg;
NewColRequestHeader *primMsg;
NewColResultHeader *outMsg;
boost::scoped_array<uint8_t> inputMsg;
NewColRequestHeader* primMsg;
NewColResultHeader* outMsg;
// the length of base prim msg, which is everything up to the
// rid array for the pCol message
uint32_t baseMsgLength;
// the length of base prim msg, which is everything up to the
// rid array for the pCol message
uint32_t baseMsgLength;
uint64_t lbid;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
messageqcpp::ByteStream filterString;
uint16_t filterCount;
bool makeAbsRids;
int64_t *values; // this is usually bpp->values; RTSCommand needs to use a different container
uint64_t lbid;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
messageqcpp::ByteStream filterString;
uint16_t filterCount;
bool makeAbsRids;
int64_t* values; // this is usually bpp->values; RTSCommand needs to use a different container
uint8_t mask, shift; // vars for the selective block loader
uint8_t mask, shift; // vars for the selective block loader
// counters to decide whether to prefetch or not
uint32_t blockCount, loadCount;
// counters to decide whether to prefetch or not
uint32_t blockCount, loadCount;
boost::shared_ptr<primitives::ParsedColumnFilter> parsedColumnFilter;
boost::shared_ptr<primitives::ParsedColumnFilter> parsedColumnFilter;
/* OR hacks */
boost::shared_ptr<primitives::ParsedColumnFilter> emptyFilter;
bool suppressFilter;
/* OR hacks */
boost::shared_ptr<primitives::ParsedColumnFilter> emptyFilter;
bool suppressFilter;
std::vector<uint64_t> lastLbid;
std::vector<uint64_t> lastLbid;
/* speculative optimizations for projectintorowgroup() */
rowgroup::Row r;
uint32_t rowSize;
/* speculative optimizations for projectintorowgroup() */
rowgroup::Row r;
uint32_t rowSize;
bool wasVersioned;
bool wasVersioned;
friend class RTSCommand;
friend class RTSCommand;
};
}

204
primitives/primproc/command.cpp
View File

@ -30,124 +30,146 @@ Command::Command(CommandType c) : cmdType(c), fFilterFeeder(NOT_FEEDER) { }
Command::~Command() { };
void Command::createCommand(ByteStream &bs)
void Command::createCommand(ByteStream& bs)
{
bs >> OID;
bs >> tupleKey;
bs >> queryUuid;
bs >> stepUuid;
bs >> OID;
bs >> tupleKey;
bs >> queryUuid;
bs >> stepUuid;
}
void Command::resetCommand(ByteStream &bs) { };
void Command::resetCommand(ByteStream& bs) { };
void Command::setMakeAbsRids(bool) { }
Command* Command::makeCommand(ByteStream &bs, CommandType *type, vector<SCommand>& cmds)
Command* Command::makeCommand(ByteStream& bs, CommandType* type, vector<SCommand>& cmds)
{
Command* ret;
uint8_t tmp8;
Command* ret;
uint8_t tmp8;
bs.peek(tmp8);
*type = (CommandType) tmp8;
switch (*type)
{
case COLUMN_COMMAND:
ret = new ColumnCommand();
break;
case DICT_STEP:
ret = new DictStep();
break;
case PASS_THRU:
ret = new PassThruCommand();
break;
case RID_TO_STRING:
ret = new RTSCommand();
break;
case FILTER_COMMAND:
ret = FilterCommand::makeFilterCommand(bs, cmds);
break;
bs.peek(tmp8);
*type = (CommandType) tmp8;
switch (*type) {
case COLUMN_COMMAND:
ret = new ColumnCommand();
break;
case DICT_STEP:
ret = new DictStep();
break;
case PASS_THRU:
ret = new PassThruCommand();
break;
case RID_TO_STRING:
ret = new RTSCommand();
break;
case FILTER_COMMAND:
ret = FilterCommand::makeFilterCommand(bs, cmds);
break;
case PSEUDOCOLUMN:
ret = new PseudoCC();
break;
default:
throw logic_error("Command::makeCommand(): can't deserialize this bytestream");
};
ret->createCommand(bs);
return ret;
default:
throw logic_error("Command::makeCommand(): can't deserialize this bytestream");
};
ret->createCommand(bs);
return ret;
}
void Command::setBatchPrimitiveProcessor(BatchPrimitiveProcessor *b)
void Command::setBatchPrimitiveProcessor(BatchPrimitiveProcessor* b)
{
bpp = b;
bpp = b;
}
void Command::copyRidsForFilterCmd()
{
if (fFilterFeeder == LEFT_FEEDER)
{
bpp->fFiltRidCount[0] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[0][i] = bpp->relRids[i];
}
else // if (fFilterFeeder == RIGHT_FEEDER)
{
bpp->fFiltRidCount[1] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[1][i] = bpp->relRids[i];
}
if (fFilterFeeder == LEFT_FEEDER)
{
bpp->fFiltRidCount[0] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[0][i] = bpp->relRids[i];
}
else // if (fFilterFeeder == RIGHT_FEEDER)
{
bpp->fFiltRidCount[1] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[1][i] = bpp->relRids[i];
}
}
bool Command::operator==(const Command &c) const
bool Command::operator==(const Command& c) const
{
const type_info &cType = typeid(c);
const type_info& cType = typeid(c);
if (cType != typeid(*this))
return false;
if (cType != typeid(*this))
return false;
if (cType == typeid(ColumnCommand)) {
const ColumnCommand *cc = dynamic_cast<const ColumnCommand *>(&c);
const ColumnCommand *t = dynamic_cast<const ColumnCommand *>(this);
if (*cc != *t)
return false;
}
else if (cType == typeid(DictStep)) {
const DictStep *ds = dynamic_cast<const DictStep *>(&c);
const DictStep *t = dynamic_cast<const DictStep *>(this);
if (*ds != *t)
return false;
}
else if (cType == typeid(PassThruCommand)) {
const PassThruCommand *pt = dynamic_cast<const PassThruCommand *>(&c);
const PassThruCommand *t = dynamic_cast<const PassThruCommand *>(this);
if (*pt != *t)
return false;
}
else if (cType == typeid(RTSCommand)) {
const RTSCommand *rts = dynamic_cast<const RTSCommand *>(&c);
const RTSCommand *t = dynamic_cast<const RTSCommand *>(this);
if (*rts != *t)
return false;
}
else if (cType == typeid(FilterCommand)) {
const FilterCommand *fc = dynamic_cast<const FilterCommand *>(&c);
const FilterCommand *t = dynamic_cast<const FilterCommand *>(this);
if (*fc != *t)
return false;
}
else
cerr << "unknown Command type\n";
if (cType == typeid(ColumnCommand))
{
const ColumnCommand* cc = dynamic_cast<const ColumnCommand*>(&c);
const ColumnCommand* t = dynamic_cast<const ColumnCommand*>(this);
return true;
if (*cc != *t)
return false;
}
else if (cType == typeid(DictStep))
{
const DictStep* ds = dynamic_cast<const DictStep*>(&c);
const DictStep* t = dynamic_cast<const DictStep*>(this);
if (*ds != *t)
return false;
}
else if (cType == typeid(PassThruCommand))
{
const PassThruCommand* pt = dynamic_cast<const PassThruCommand*>(&c);
const PassThruCommand* t = dynamic_cast<const PassThruCommand*>(this);
if (*pt != *t)
return false;
}
else if (cType == typeid(RTSCommand))
{
const RTSCommand* rts = dynamic_cast<const RTSCommand*>(&c);
const RTSCommand* t = dynamic_cast<const RTSCommand*>(this);
if (*rts != *t)
return false;
}
else if (cType == typeid(FilterCommand))
{
const FilterCommand* fc = dynamic_cast<const FilterCommand*>(&c);
const FilterCommand* t = dynamic_cast<const FilterCommand*>(this);
if (*fc != *t)
return false;
}
else
cerr << "unknown Command type\n";
return true;
}
void Command::duplicate(Command *c)
void Command::duplicate(Command* c)
{
bpp = c->bpp;
cmdType = c->cmdType;
fFilterFeeder = c->fFilterFeeder;
OID = c->OID;
tupleKey = c->tupleKey;
queryUuid = c->queryUuid;
stepUuid = c->stepUuid;
bpp = c->bpp;
cmdType = c->cmdType;
fFilterFeeder = c->fFilterFeeder;
OID = c->OID;
tupleKey = c->tupleKey;
queryUuid = c->queryUuid;
stepUuid = c->stepUuid;
}
}

123
primitives/primproc/command.h
View File

@ -35,74 +35,93 @@ typedef boost::shared_ptr<Command> SCommand;
class Command
{
public:
enum CommandType {
NONE,
COLUMN_COMMAND,
DICT_STEP,
DICT_SCAN,
PASS_THRU,
RID_TO_STRING,
FILTER_COMMAND,
enum CommandType
{
NONE,
COLUMN_COMMAND,
DICT_STEP,
DICT_SCAN,
PASS_THRU,
RID_TO_STRING,
FILTER_COMMAND,
PSEUDOCOLUMN
};
};
static const uint8_t NOT_FEEDER = 0;
static const uint8_t FILT_FEEDER = 1;
static const uint8_t LEFT_FEEDER = 3;
static const uint8_t RIGHT_FEEDER = 5;
static const uint8_t NOT_FEEDER = 0;
static const uint8_t FILT_FEEDER = 1;
static const uint8_t LEFT_FEEDER = 3;
static const uint8_t RIGHT_FEEDER = 5;
Command(CommandType c);
virtual ~Command();
Command(CommandType c);
virtual ~Command();
virtual void execute() = 0;
virtual void project() = 0;
virtual void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t columnPosition) = 0;
virtual uint64_t getLBID() = 0;
virtual void getLBIDList(uint32_t loopCount, std::vector<int64_t> *out) {} // the default fcn returns 0 lbids
virtual void nextLBID() = 0;
virtual void createCommand(messageqcpp::ByteStream &);
virtual void resetCommand(messageqcpp::ByteStream &);
virtual void execute() = 0;
virtual void project() = 0;
virtual void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t columnPosition) = 0;
virtual uint64_t getLBID() = 0;
virtual void getLBIDList(uint32_t loopCount, std::vector<int64_t>* out) {} // the default fcn returns 0 lbids
virtual void nextLBID() = 0;
virtual void createCommand(messageqcpp::ByteStream&);
virtual void resetCommand(messageqcpp::ByteStream&);
/* Duplicate() makes a copy of this object as constructed by createCommand().
It's thread-safe */
virtual SCommand duplicate() = 0;
bool operator==(const Command &) const;
bool operator!=(const Command& c) const { return !(*this == c); }
/* Duplicate() makes a copy of this object as constructed by createCommand().
It's thread-safe */
virtual SCommand duplicate() = 0;
bool operator==(const Command&) const;
bool operator!=(const Command& c) const
{
return !(*this == c);
}
/* Put bootstrap code here (ie, build the template primitive msg) */
virtual void prep(int8_t outputType, bool makeAbsRids) = 0;
virtual void setBatchPrimitiveProcessor(BatchPrimitiveProcessor *);
virtual void setMakeAbsRids(bool);
/* Put bootstrap code here (ie, build the template primitive msg) */
virtual void prep(int8_t outputType, bool makeAbsRids) = 0;
virtual void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*);
virtual void setMakeAbsRids(bool);
CommandType getCommandType() const { return cmdType; }
CommandType getCommandType() const
{
return cmdType;
}
// if feeding a filtercommand
// note: a filtercommand itself can feed another filtercommand
uint8_t filterFeeder() { return fFilterFeeder; }
void filterFeeder(uint8_t f) { fFilterFeeder = f; }
virtual void copyRidsForFilterCmd();
// if feeding a filtercommand
// note: a filtercommand itself can feed another filtercommand
uint8_t filterFeeder()
{
return fFilterFeeder;
}
void filterFeeder(uint8_t f)
{
fFilterFeeder = f;
}
virtual void copyRidsForFilterCmd();
static Command* makeCommand(messageqcpp::ByteStream&, CommandType*, std::vector<SCommand>&);
static Command* makeCommand(messageqcpp::ByteStream&, CommandType*, std::vector<SCommand>&);
uint32_t getOID() const { return OID; }
uint32_t getTupleKey() const { return tupleKey; }
uint32_t getOID() const
{
return OID;
}
uint32_t getTupleKey() const
{
return tupleKey;
}
virtual int getCompType() const=0;
virtual int getCompType() const = 0;
protected:
BatchPrimitiveProcessor *bpp;
CommandType cmdType;
uint8_t fFilterFeeder;
uint32_t OID;
uint32_t tupleKey;
boost::uuids::uuid queryUuid;
boost::uuids::uuid stepUuid;
BatchPrimitiveProcessor* bpp;
CommandType cmdType;
uint8_t fFilterFeeder;
uint32_t OID;
uint32_t tupleKey;
boost::uuids::uuid queryUuid;
boost::uuids::uuid stepUuid;
void duplicate(Command *);
void duplicate(Command*);
private:
Command();
Command(const Command &);
Command();
Command(const Command&);
};

910
primitives/primproc/dictstep.cpp
View File

File diff suppressed because it is too large Load Diff

179
primitives/primproc/dictstep.h
View File

@ -19,7 +19,7 @@
// $Id: dictstep.h 2110 2013-06-19 15:51:38Z bwilkinson $
// C++ Interface: dictstep
//
// Description:
// Description:
//
//
// Author: Patrick LeBlanc <pleblanc@calpont.com>, (C) 2008
@ -34,106 +34,121 @@
#include "command.h"
#include "primitivemsg.h"
namespace primitiveprocessor {
namespace primitiveprocessor
{
class DictStep : public Command
{
public:
DictStep();
virtual ~DictStep();
public:
DictStep();
virtual ~DictStep();
void execute();
void project();
void project(int64_t *vals); //used by RTSCommand to redirect input
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t row);
void projectIntoRowGroup(rowgroup::RowGroup &rg, int64_t *vals, uint32_t col);
uint64_t getLBID();
void execute();
void project();
void project(int64_t* vals); //used by RTSCommand to redirect input
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t row);
void projectIntoRowGroup(rowgroup::RowGroup& rg, int64_t* vals, uint32_t col);
uint64_t getLBID();
/* This doesn't do anything for this class... make it column-specific or not? */
void nextLBID();
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
/* This doesn't do anything for this class... make it column-specific or not? */
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
SCommand duplicate();
bool operator==(const DictStep &) const;
bool operator!=(const DictStep &) const;
SCommand duplicate();
bool operator==(const DictStep&) const;
bool operator!=(const DictStep&) const;
int getCompType() const { return compressionType; }
void setCompType(int ct) { compressionType = ct; }
int getCompType() const
{
return compressionType;
}
void setCompType(int ct)
{
compressionType = ct;
}
private:
DictStep(const DictStep &);
DictStep& operator=(const DictStep &);
private:
DictStep(const DictStep&);
DictStep& operator=(const DictStep&);
struct StringPtr {
const uint8_t* ptr;
unsigned len;
struct StringPtr
{
const uint8_t* ptr;
unsigned len;
StringPtr() : ptr(NULL), len(0) {;}
StringPtr(const uint8_t* p, unsigned l) : ptr(p), len(l) {;}
};
StringPtr() : ptr(NULL), len(0) {;}
StringPtr(const uint8_t* p, unsigned l) : ptr(p), len(l) {;}
};
void _execute();
void issuePrimitive(bool isProjection);
void processResult();
void projectResult(std::string* tmpStrings);
void projectResult(StringPtr *tmpStrings);
void _project();
void _projectToRG(rowgroup::RowGroup &rg, uint32_t col);
void _execute();
void issuePrimitive(bool isProjection);
void processResult();
void projectResult(std::string* tmpStrings);
void projectResult(StringPtr* tmpStrings);
void _project();
void _projectToRG(rowgroup::RowGroup& rg, uint32_t col);
// struct used for scratch space
struct OrderedToken {
uint64_t rid;
uint64_t token;
uint16_t pos;
std::string str;
bool inResult;
OrderedToken() : inResult(false) { }
~OrderedToken() { }
};
struct TokenSorter {
inline bool operator()(const OrderedToken &c1, const OrderedToken &c2) const
{ return (c1.token < c2.token); }
};
struct PosSorter {
inline bool operator()(const OrderedToken &c1, const OrderedToken &c2) const
{ return (c1.pos < c2.pos); }
};
// struct used for scratch space
struct OrderedToken
{
uint64_t rid;
uint64_t token;
uint16_t pos;
std::string str;
bool inResult;
OrderedToken() : inResult(false) { }
~OrderedToken() { }
};
struct TokenSorter
{
inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const
{
return (c1.token < c2.token);
}
};
struct PosSorter
{
inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const
{
return (c1.pos < c2.pos);
}
};
//bug 3679. FilterCommand depends on the result being in the same relative
//order as the input. These fcns help restore the original order.
void copyResultToTmpSpace(OrderedToken *ot);
void copyResultToFinalPosition(OrderedToken *ot);
//bug 3679. FilterCommand depends on the result being in the same relative
//order as the input. These fcns help restore the original order.
void copyResultToTmpSpace(OrderedToken* ot);
void copyResultToFinalPosition(OrderedToken* ot);
// Worst case, 8192 tokens in the msg. Each is 10 bytes. */
boost::scoped_array<uint8_t> inputMsg;
uint32_t tmpResultCounter;
uint32_t totalResultLength;
DictInput *primMsg;
std::vector<uint8_t> result;
// Worst case, 8192 tokens in the msg. Each is 10 bytes. */
boost::scoped_array<uint8_t> inputMsg;
uint32_t tmpResultCounter;
uint32_t totalResultLength;
DictInput* primMsg;
std::vector<uint8_t> result;
uint64_t lbid;
uint32_t fbo;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
int64_t *values;
boost::scoped_array<std::string>* strValues;
int compressionType;
ByteStream filterString;
uint32_t filterCount;
uint32_t bufferSize;
uint16_t inputRidCount;
uint64_t lbid;
uint32_t fbo;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
int64_t* values;
boost::scoped_array<std::string>* strValues;
int compressionType;
ByteStream filterString;
uint32_t filterCount;
uint32_t bufferSize;
uint16_t inputRidCount;
bool hasEqFilter;
boost::shared_ptr<primitives::DictEqualityFilter> eqFilter;
boost::shared_array<primitives::idb_regex_t> likeFilter;
uint8_t eqOp; // COMPARE_EQ or COMPARE_NE
bool hasEqFilter;
boost::shared_ptr<primitives::DictEqualityFilter> eqFilter;
boost::shared_array<primitives::idb_regex_t> likeFilter;
uint8_t eqOp; // COMPARE_EQ or COMPARE_NE
friend class RTSCommand;
friend class RTSCommand;
};
} // namespace

770
primitives/primproc/filtercommand.cpp
View File

@ -49,121 +49,128 @@ namespace primitiveprocessor
Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector<SCommand>& cmds)
{
bs.advance(1);
// find out the # of commands in the cmds vector,
// vector::size() will not work, because cmds is resize() to filterCount
uint64_t nc = 0;
while (cmds[nc].get() != NULL) nc++;
// find out the # of commands in the cmds vector,
// vector::size() will not work, because cmds is resize() to filterCount
uint64_t nc = 0;
// figure out the feeding commands
// must have 2 columncommands, may have 1 or 2 dictsteps.
uint64_t cols = 0; // # of columncommands
uint32_t columns = 0;
uint64_t i = nc;
while (i > 0 && cols < 2)
{
Command::CommandType cmdType = cmds[i-1]->getCommandType();
if (cmdType != Command::COLUMN_COMMAND && cmdType != Command::DICT_STEP)
{
stringstream msg;
msg << "FilterCommand: feeded by " << cmdType << " is not supported.";
throw logic_error(msg.str());
}
while (cmds[nc].get() != NULL) nc++;
columns = (columns<<8) + cmdType;
if (cmdType == Command::COLUMN_COMMAND)
cols++;
i--;
}
// figure out the feeding commands
// must have 2 columncommands, may have 1 or 2 dictsteps.
uint64_t cols = 0; // # of columncommands
uint32_t columns = 0;
uint64_t i = nc;
// should not happen
if (cols < 2) throw logic_error("FilterCommand: not enough feeding ColumnCommands.");
while (i > 0 && cols < 2)
{
Command::CommandType cmdType = cmds[i - 1]->getCommandType();
if (cmdType != Command::COLUMN_COMMAND && cmdType != Command::DICT_STEP)
{
stringstream msg;
msg << "FilterCommand: feeded by " << cmdType << " is not supported.";
throw logic_error(msg.str());
}
columns = (columns << 8) + cmdType;
if (cmdType == Command::COLUMN_COMMAND)
cols++;
i--;
}
// should not happen
if (cols < 2) throw logic_error("FilterCommand: not enough feeding ColumnCommands.");
FilterCommand* fc = NULL;
// the order setting left/right feeder is important, left is the smaller index.
// because the doFilter relies on the rids of right-feeder is a subset of left.
if (columns == CC)
{
cmds[nc-2]->filterFeeder(LEFT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-2].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc-1].get());
int scale0 = cmd0->getScale();
int scale1 = cmd1->getScale();
// char[] is stored as int, but cannot directly compare if length is different
// due to endian issue
if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::TEXT)
{
StrFilterCmd* sc = new StrFilterCmd();
sc->setCompareFunc(CC);
fc = sc;
}
else if (scale0 == scale1)
{
fc = new FilterCommand();
}
else
{
ScaledFilterCmd* sc = new ScaledFilterCmd();
sc->setFactor(pow(10.0, scale1) / pow(10.0, scale0));
fc = sc;
}
// the order setting left/right feeder is important, left is the smaller index.
// because the doFilter relies on the rids of right-feeder is a subset of left.
if (columns == CC)
{
cmds[nc - 2]->filterFeeder(LEFT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == DCDC) // both string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc-4]->filterFeeder(FILT_FEEDER);
cmds[nc-3]->filterFeeder(LEFT_FEEDER);
cmds[nc-2]->filterFeeder(FILT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
sc->setCompareFunc(DCDC);
fc = sc;
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 2].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc - 1].get());
int scale0 = cmd0->getScale();
int scale1 = cmd1->getScale();
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-4].get());
ColumnCommand* cmd2 = dynamic_cast<ColumnCommand*>(cmds[nc-2].get());
fc->setColTypes(cmd0->getColType(), cmd2->getColType());
}
else if (columns == DCC) // lhs: char[]; rhs: string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc-3]->filterFeeder(LEFT_FEEDER);
cmds[nc-2]->filterFeeder(FILT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc-2].get());
size_t cl = cmd0->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(DCC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == CDC) // lhs: string; rhs: char[]
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc-3]->filterFeeder(FILT_FEEDER);
cmds[nc-2]->filterFeeder(LEFT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc-1].get());
size_t cl = cmd1->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(CDC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else
{
stringstream msg;
msg << "FilterCommand does not handle this column code: " << hex << columns << dec;
throw logic_error(msg.str());
}
// char[] is stored as int, but cannot directly compare if length is different
// due to endian issue
if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::TEXT)
{
StrFilterCmd* sc = new StrFilterCmd();
sc->setCompareFunc(CC);
fc = sc;
}
else if (scale0 == scale1)
{
fc = new FilterCommand();
}
else
{
ScaledFilterCmd* sc = new ScaledFilterCmd();
sc->setFactor(pow(10.0, scale1) / pow(10.0, scale0));
fc = sc;
}
return fc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == DCDC) // both string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc - 4]->filterFeeder(FILT_FEEDER);
cmds[nc - 3]->filterFeeder(LEFT_FEEDER);
cmds[nc - 2]->filterFeeder(FILT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
sc->setCompareFunc(DCDC);
fc = sc;
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 4].get());
ColumnCommand* cmd2 = dynamic_cast<ColumnCommand*>(cmds[nc - 2].get());
fc->setColTypes(cmd0->getColType(), cmd2->getColType());
}
else if (columns == DCC) // lhs: char[]; rhs: string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc - 3]->filterFeeder(LEFT_FEEDER);
cmds[nc - 2]->filterFeeder(FILT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc - 2].get());
size_t cl = cmd0->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(DCC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == CDC) // lhs: string; rhs: char[]
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc - 3]->filterFeeder(FILT_FEEDER);
cmds[nc - 2]->filterFeeder(LEFT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc - 1].get());
size_t cl = cmd1->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(CDC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else
{
stringstream msg;
msg << "FilterCommand does not handle this column code: " << hex << columns << dec;
throw logic_error(msg.str());
}
return fc;
}
@ -179,18 +186,18 @@ FilterCommand::~FilterCommand()
void FilterCommand::execute()
{
doFilter();
doFilter();
}
void FilterCommand::createCommand(ByteStream& bs)
{
bs >> fBOP;
Command::createCommand(bs);
bs >> fBOP;
Command::createCommand(bs);
}
void FilterCommand::resetCommand(ByteStream &bs)
void FilterCommand::resetCommand(ByteStream& bs)
{
}
@ -204,14 +211,14 @@ void FilterCommand::project()
{
}
void FilterCommand::projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col)
void FilterCommand::projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col)
{
}
uint64_t FilterCommand::getLBID()
{
return 0;
return 0;
}
@ -222,102 +229,108 @@ void FilterCommand::nextLBID()
SCommand FilterCommand::duplicate()
{
SCommand ret;
FilterCommand* filterCmd;
SCommand ret;
FilterCommand* filterCmd;
ret.reset(new FilterCommand());
filterCmd = (FilterCommand *) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->leftColType = leftColType;
filterCmd->rightColType = rightColType;
filterCmd->Command::duplicate(this);
return ret;
ret.reset(new FilterCommand());
filterCmd = (FilterCommand*) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->leftColType = leftColType;
filterCmd->rightColType = rightColType;
filterCmd->Command::duplicate(this);
return ret;
}
void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& left,
const execplan::CalpontSystemCatalog::ColType& right)
const execplan::CalpontSystemCatalog::ColType& right)
{
leftColType = left;
rightColType = right;
leftColType = left;
rightColType = right;
}
void FilterCommand::doFilter()
{
bpp->ridMap = 0;
bpp->ridCount = 0;
bpp->ridMap = 0;
bpp->ridCount = 0;
// rids in [0] is used for scan [1], so [1] is a subset of [0], and same order.
// -- see makeFilterCommand() above.
for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1]; )
{
if (bpp->fFiltCmdRids[0][i] != bpp->fFiltCmdRids[1][j])
{
i++;
}
else
{
if (compare(i,j) == true)
{
bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i];
bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i];
bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 10);
bpp->ridCount++;
}
// rids in [0] is used for scan [1], so [1] is a subset of [0], and same order.
// -- see makeFilterCommand() above.
for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1]; )
{
if (bpp->fFiltCmdRids[0][i] != bpp->fFiltCmdRids[1][j])
{
i++;
}
else
{
if (compare(i, j) == true)
{
bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i];
bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i];
bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 10);
bpp->ridCount++;
}
i++;
j++;
}
}
i++;
j++;
}
}
// bug 1247 -- reset the rid count
bpp->fFiltRidCount[0] = bpp->fFiltRidCount[1] = 0;
// bug 1247 -- reset the rid count
bpp->fFiltRidCount[0] = bpp->fFiltRidCount[1] = 0;
}
bool FilterCommand::compare(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
switch(fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i] > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i] < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i] == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i] >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i] <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i] != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i] > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i] < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i] == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i] >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i] <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i] != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
}
bool FilterCommand::operator==(const FilterCommand& c) const
{
return (fBOP == c.fBOP);
return (fBOP == c.fBOP);
}
bool FilterCommand::operator!=(const FilterCommand& c) const
{
return !(*this == c);
return !(*this == c);
}
@ -334,72 +347,78 @@ ScaledFilterCmd::~ScaledFilterCmd()
SCommand ScaledFilterCmd::duplicate()
{
SCommand ret;
ScaledFilterCmd* filterCmd;
SCommand ret;
ScaledFilterCmd* filterCmd;
ret.reset(new ScaledFilterCmd());
filterCmd = (ScaledFilterCmd *) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fFactor = fFactor;
ret.reset(new ScaledFilterCmd());
filterCmd = (ScaledFilterCmd*) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fFactor = fFactor;
return ret;
return ret;
}
bool ScaledFilterCmd::compare(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
switch(fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i]*fFactor > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i]*fFactor < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i]*fFactor == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i]*fFactor >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i]*fFactor <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i]*fFactor != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i] * fFactor > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i] * fFactor < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i] * fFactor == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i] * fFactor >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i] * fFactor <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i] * fFactor != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
}
void ScaledFilterCmd::setFactor(double f)
{
fFactor = f;
fFactor = f;
}
double ScaledFilterCmd::factor()
{
return fFactor;
return fFactor;
}
bool ScaledFilterCmd::operator==(const ScaledFilterCmd& c) const
{
return ((fBOP == c.fBOP) && (fFactor == c.fFactor));
return ((fBOP == c.fBOP) && (fFactor == c.fFactor));
}
bool ScaledFilterCmd::operator!=(const ScaledFilterCmd& c) const
{
return !(*this == c);
return !(*this == c);
}
@ -416,216 +435,241 @@ StrFilterCmd::~StrFilterCmd()
SCommand StrFilterCmd::duplicate()
{
SCommand ret;
StrFilterCmd* filterCmd;
SCommand ret;
StrFilterCmd* filterCmd;
ret.reset(new StrFilterCmd());
filterCmd = (StrFilterCmd *) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fCompare = fCompare;
filterCmd->fCharLength = fCharLength;
ret.reset(new StrFilterCmd());
filterCmd = (StrFilterCmd*) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fCompare = fCompare;
filterCmd->fCharLength = fCharLength;
return ret;
return ret;
}
void StrFilterCmd::execute()
{
doFilter();
doFilter();
}
bool StrFilterCmd::compare(uint64_t i, uint64_t j)
{
return (this->*fCompare)(i, j);
return (this->*fCompare)(i, j);
}
void StrFilterCmd::setCompareFunc(uint32_t columns)
{
if (columns == CC) // char[] : char
{
fCompare = &StrFilterCmd::compare_cc;
}
else if (columns == DCDC) // string : string
{
fCompare = &StrFilterCmd::compare_ss;
}
else if (columns == DCC) // char[] : string
{
fCompare = &StrFilterCmd::compare_cs;
}
else if (columns == CDC) // string : char[]
{
fCompare = &StrFilterCmd::compare_sc;
}
else
{
stringstream msg;
msg << "StrFilterCmd: unhandled column combination " << hex << columns << dec;
throw logic_error(msg.str());
}
if (columns == CC) // char[] : char
{
fCompare = &StrFilterCmd::compare_cc;
}
else if (columns == DCDC) // string : string
{
fCompare = &StrFilterCmd::compare_ss;
}
else if (columns == DCC) // char[] : string
{
fCompare = &StrFilterCmd::compare_cs;
}
else if (columns == CDC) // string : char[]
{
fCompare = &StrFilterCmd::compare_sc;
}
else
{
stringstream msg;
msg << "StrFilterCmd: unhandled column combination " << hex << columns << dec;
throw logic_error(msg.str());
}
}
bool StrFilterCmd::compare_cc(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
switch(fBOP)
{
case COMPARE_GT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) > uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) < uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_EQ:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) == uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_GE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) >= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) <= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_NE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) != uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) > uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) < uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_EQ:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) == uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_GE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) >= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) <= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_NE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) != uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
default:
return false;
break;
}
}
bool StrFilterCmd::compare_ss(uint64_t i, uint64_t j)
{
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[1][j] == "" ||
bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[1][j] == "" ||
bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
switch(fBOP)
{
case COMPARE_GT:
return bpp->fFiltStrValues[0][i] > bpp->fFiltStrValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltStrValues[0][i] < bpp->fFiltStrValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltStrValues[0][i] == bpp->fFiltStrValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltStrValues[0][i] >= bpp->fFiltStrValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltStrValues[0][i] <= bpp->fFiltStrValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltStrValues[0][i] != bpp->fFiltStrValues[1][j];
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return bpp->fFiltStrValues[0][i] > bpp->fFiltStrValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltStrValues[0][i] < bpp->fFiltStrValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltStrValues[0][i] == bpp->fFiltStrValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltStrValues[0][i] >= bpp->fFiltStrValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltStrValues[0][i] <= bpp->fFiltStrValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltStrValues[0][i] != bpp->fFiltStrValues[1][j];
break;
default:
return false;
break;
}
}
bool StrFilterCmd::compare_cs(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
bpp->fFiltStrValues[1][j] == "" || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
bpp->fFiltStrValues[1][j] == "" || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
int cmp = strncmp(reinterpret_cast<const char*>(&bpp->fFiltCmdValues[0][i]),
bpp->fFiltStrValues[1][j].c_str(), fCharLength);
switch(fBOP)
{
case COMPARE_GT:
return (cmp > 0);
break;
case COMPARE_LT:
return (cmp < 0 || (cmp == 0 && fCharLength < bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_EQ:
return (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length());
break;
case COMPARE_GE:
return (cmp > 0 || (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_LE:
return (cmp <= 0);
break;
case COMPARE_NE:
return (cmp != 0 || fCharLength < bpp->fFiltStrValues[1][j].length());
break;
default:
return false;
break;
}
int cmp = strncmp(reinterpret_cast<const char*>(&bpp->fFiltCmdValues[0][i]),
bpp->fFiltStrValues[1][j].c_str(), fCharLength);
switch (fBOP)
{
case COMPARE_GT:
return (cmp > 0);
break;
case COMPARE_LT:
return (cmp < 0 || (cmp == 0 && fCharLength < bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_EQ:
return (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length());
break;
case COMPARE_GE:
return (cmp > 0 || (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_LE:
return (cmp <= 0);
break;
case COMPARE_NE:
return (cmp != 0 || fCharLength < bpp->fFiltStrValues[1][j].length());
break;
default:
return false;
break;
}
}
bool StrFilterCmd::compare_sc(uint64_t i, uint64_t j)
{
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
int cmp = strncmp(bpp->fFiltStrValues[0][i].c_str(),
reinterpret_cast<const char*>(&bpp->fFiltCmdValues[1][j]), fCharLength);
int cmp = strncmp(bpp->fFiltStrValues[0][i].c_str(),
reinterpret_cast<const char*>(&bpp->fFiltCmdValues[1][j]), fCharLength);
switch(fBOP)
{
case COMPARE_GT:
return (cmp > 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() > fCharLength));
break;
case COMPARE_LT:
return (cmp < 0);
break;
case COMPARE_EQ:
return (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength);
break;
case COMPARE_GE:
return (cmp >= 0);
break;
case COMPARE_LE:
return (cmp < 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength));
break;
case COMPARE_NE:
return (cmp != 0 || bpp->fFiltStrValues[0][i].length() > fCharLength);
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return (cmp > 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() > fCharLength));
break;
case COMPARE_LT:
return (cmp < 0);
break;
case COMPARE_EQ:
return (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength);
break;
case COMPARE_GE:
return (cmp >= 0);
break;
case COMPARE_LE:
return (cmp < 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength));
break;
case COMPARE_NE:
return (cmp != 0 || bpp->fFiltStrValues[0][i].length() > fCharLength);
break;
default:
return false;
break;
}
}
void StrFilterCmd::setCharLength(size_t l)
{
fCharLength = l;
fCharLength = l;
}
size_t StrFilterCmd::charLength()
{
return fCharLength;
return fCharLength;
}
bool StrFilterCmd::operator==(const StrFilterCmd& c) const
{
return ((fBOP == c.fBOP) && fCharLength == c.fCharLength);
return ((fBOP == c.fBOP) && fCharLength == c.fCharLength);
}
bool StrFilterCmd::operator!=(const StrFilterCmd& c) const
{
return !(*this == c);
return !(*this == c);
}

165
primitives/primproc/filtercommand.h
View File

@ -39,119 +39,122 @@ namespace primitiveprocessor
class FilterCommand : public Command
{
public:
FilterCommand();
virtual ~FilterCommand();
public:
FilterCommand();
virtual ~FilterCommand();
// returns a FilterCommand based on column types
static Command* makeFilterCommand(messageqcpp::ByteStream&, std::vector<SCommand>& cmds);
// returns a FilterCommand based on column types
static Command* makeFilterCommand(messageqcpp::ByteStream&, std::vector<SCommand>& cmds);
// virtuals from base class -- Command
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
void prep(int8_t outputType, bool makeAbsRids);
// virtuals from base class -- Command
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
void prep(int8_t outputType, bool makeAbsRids);
void setColTypes(const execplan::CalpontSystemCatalog::ColType& left,
const execplan::CalpontSystemCatalog::ColType& right);
void setColTypes(const execplan::CalpontSystemCatalog::ColType& left,
const execplan::CalpontSystemCatalog::ColType& right);
// operator override
bool operator==(const FilterCommand&) const;
bool operator!=(const FilterCommand&) const;
// operator override
bool operator==(const FilterCommand&) const;
bool operator!=(const FilterCommand&) const;
int getCompType() const { return 0; }
int getCompType() const
{
return 0;
}
protected:
// filter operation
virtual void doFilter();
protected:
// filter operation
virtual void doFilter();
// compare method, take the indices to the values array
virtual bool compare(uint64_t, uint64_t);
// compare method, take the indices to the values array
virtual bool compare(uint64_t, uint64_t);
// binary operator
uint8_t fBOP;
// binary operator
uint8_t fBOP;
// column type for null check
execplan::CalpontSystemCatalog::ColType leftColType;
execplan::CalpontSystemCatalog::ColType rightColType;
// column type for null check
execplan::CalpontSystemCatalog::ColType leftColType;
execplan::CalpontSystemCatalog::ColType rightColType;
private:
// disabled copy constructor and operator
FilterCommand(const FilterCommand&);
FilterCommand& operator=(const FilterCommand&);
private:
// disabled copy constructor and operator
FilterCommand(const FilterCommand&);
FilterCommand& operator=(const FilterCommand&);
};
class ScaledFilterCmd : public FilterCommand
{
public:
ScaledFilterCmd();
virtual ~ScaledFilterCmd();
SCommand duplicate();
public:
ScaledFilterCmd();
virtual ~ScaledFilterCmd();
SCommand duplicate();
void setFactor(double);
double factor();
void setFactor(double);
double factor();
// operator override
bool operator==(const ScaledFilterCmd&) const;
bool operator!=(const ScaledFilterCmd&) const;
// operator override
bool operator==(const ScaledFilterCmd&) const;
bool operator!=(const ScaledFilterCmd&) const;
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
// value used in comparison;
double fFactor;
// value used in comparison;
double fFactor;
private:
// disabled copy constructor and operator
ScaledFilterCmd(const ScaledFilterCmd &);
ScaledFilterCmd& operator=(const ScaledFilterCmd &);
private:
// disabled copy constructor and operator
ScaledFilterCmd(const ScaledFilterCmd&);
ScaledFilterCmd& operator=(const ScaledFilterCmd&);
};
class StrFilterCmd : public FilterCommand
{
public:
StrFilterCmd();
virtual ~StrFilterCmd();
public:
StrFilterCmd();
virtual ~StrFilterCmd();
// override FilterCommand methods
void execute();
SCommand duplicate();
// override FilterCommand methods
void execute();
SCommand duplicate();
void setCompareFunc(uint32_t);
void setCharLength(size_t);
size_t charLength();
void setCompareFunc(uint32_t);
void setCharLength(size_t);
size_t charLength();
// operator override
bool operator==(const StrFilterCmd&) const;
bool operator!=(const StrFilterCmd&) const;
// operator override
bool operator==(const StrFilterCmd&) const;
bool operator!=(const StrFilterCmd&) const;
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
// compare method for differernt column combination, c--char[], s--string
// compare char[]-char[] is not the same as int-int due to endian issue.
bool compare_cc(uint64_t, uint64_t);
bool compare_ss(uint64_t, uint64_t);
bool compare_cs(uint64_t, uint64_t);
bool compare_sc(uint64_t, uint64_t);
bool (StrFilterCmd::*fCompare)(uint64_t, uint64_t);
// compare method for differernt column combination, c--char[], s--string
// compare char[]-char[] is not the same as int-int due to endian issue.
bool compare_cc(uint64_t, uint64_t);
bool compare_ss(uint64_t, uint64_t);
bool compare_cs(uint64_t, uint64_t);
bool compare_sc(uint64_t, uint64_t);
bool (StrFilterCmd::*fCompare)(uint64_t, uint64_t);
// colWidth of columns the don't need a dictionary
size_t fCharLength;
// colWidth of columns the don't need a dictionary
size_t fCharLength;
private:
// disabled copy constructor and operator
StrFilterCmd(const StrFilterCmd &);
StrFilterCmd& operator=(const StrFilterCmd &);
private:
// disabled copy constructor and operator
StrFilterCmd(const StrFilterCmd&);
StrFilterCmd& operator=(const StrFilterCmd&);
};

66
primitives/primproc/logger.cpp
View File

@ -33,53 +33,55 @@ namespace primitiveprocessor
{
Logger::Logger() :
fMl1(LoggingID(28))
fMl1(LoggingID(28))
{
fMsgMap[logging::M0000] = Message(logging::M0000);
fMsgMap[logging::M0016] = Message(logging::M0016);
fMsgMap[logging::M0045] = Message(logging::M0045);
fMsgMap[logging::M0053] = Message(logging::M0053);
fMsgMap[logging::M0058] = Message(logging::M0058);
fMsgMap[logging::M0061] = Message(logging::M0061);
fMsgMap[logging::M0069] = Message(logging::M0069);
fMsgMap[logging::M0070] = Message(logging::M0070);
fMsgMap[logging::M0077] = Message(logging::M0077);
fMsgMap[logging::M0000] = Message(logging::M0000);
fMsgMap[logging::M0016] = Message(logging::M0016);
fMsgMap[logging::M0045] = Message(logging::M0045);
fMsgMap[logging::M0053] = Message(logging::M0053);
fMsgMap[logging::M0058] = Message(logging::M0058);
fMsgMap[logging::M0061] = Message(logging::M0061);
fMsgMap[logging::M0069] = Message(logging::M0069);
fMsgMap[logging::M0070] = Message(logging::M0070);
fMsgMap[logging::M0077] = Message(logging::M0077);
}
void Logger::logMessage(const Message::MessageID mid,
const Message::Args& args,
bool critical)
{
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
msgIter->second.reset();
msgIter->second.format(args);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
if (critical)
{
fMl1.logCriticalMessage(msgIter->second);
}
else
{
fMl1.logWarningMessage(msgIter->second);
}
msgIter->second.reset();
msgIter->second.format(args);
if (critical)
{
fMl1.logCriticalMessage(msgIter->second);
}
else
{
fMl1.logWarningMessage(msgIter->second);
}
}
void Logger::logInfoMessage(const Message::MessageID mid,
const Message::Args& args)
const Message::Args& args)
{
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
msgIter->second.reset();
msgIter->second.format(args);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
fMl1.logInfoMessage(msgIter->second);
msgIter->second.reset();
msgIter->second.format(args);
fMl1.logInfoMessage(msgIter->second);
}
}

183
primitives/primproc/passthrucommand.cpp
View File

@ -49,10 +49,11 @@ PassThruCommand::~PassThruCommand()
void PassThruCommand::prep(int8_t outputType, bool makeAbsRids)
{
if (bpp->ot == ROW_GROUP) {
bpp->outputRG.initRow(&r);
rowSize = r.getSize();
}
if (bpp->ot == ROW_GROUP)
{
bpp->outputRG.initRow(&r);
rowSize = r.getSize();
}
}
void PassThruCommand::execute()
@ -62,118 +63,142 @@ void PassThruCommand::execute()
void PassThruCommand::project()
{
uint32_t i;
uint32_t i;
*bpp->serialized << (uint32_t) (bpp->ridCount * colWidth);
*bpp->serialized << (uint32_t) (bpp->ridCount * colWidth);
#if 0
cout << "pass thru serializing " << (uint32_t) (bpp->ridCount * colWidth) << " bytes:\n";
cout << "at relative position " << bpp->serialized->length() - sizeof(ISMPacketHeader) - sizeof(PrimitiveHeader) - 4 << endl;
for (i = 0; i < bpp->ridCount; i++)
cout << " " << i << ": " << bpp->values[i] << endl;
cout << "pass thru serializing " << (uint32_t) (bpp->ridCount * colWidth) << " bytes:\n";
cout << "at relative position " << bpp->serialized->length() - sizeof(ISMPacketHeader) - sizeof(PrimitiveHeader) - 4 << endl;
for (i = 0; i < bpp->ridCount; i++)
cout << " " << i << ": " << bpp->values[i] << endl;
#endif
switch (colWidth) {
case 8:
bpp->serialized->append((uint8_t *) bpp->values, bpp->ridCount << 3);
break;
case 4:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint32_t) bpp->values[i];
break;
case 2:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint16_t) bpp->values[i];
break;
case 1:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint8_t) bpp->values[i];
break;
default:
throw logic_error("PassThruCommand has a bad column width");
}
switch (colWidth)
{
case 8:
bpp->serialized->append((uint8_t*) bpp->values, bpp->ridCount << 3);
break;
case 4:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint32_t) bpp->values[i];
break;
case 2:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint16_t) bpp->values[i];
break;
case 1:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint8_t) bpp->values[i];
break;
default:
throw logic_error("PassThruCommand has a bad column width");
}
}
void PassThruCommand::projectIntoRowGroup(RowGroup &rg, uint32_t col)
void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col)
{
uint32_t i;
uint32_t i;
rg.initRow(&r);
rg.getRow(0, &r);
uint32_t offset = r.getOffset(col);
rowSize = r.getSize();
rg.initRow(&r);
rg.getRow(0, &r);
uint32_t offset = r.getOffset(col);
rowSize = r.getSize();
switch (colWidth) {
case 1:
for (i = 0; i < bpp->ridCount; i++) {
switch (colWidth)
{
case 1:
for (i = 0; i < bpp->ridCount; i++)
{
// cout << "PTC: " << bpp->values[i] << endl;
r.setUintField_offset<1>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 2:
for (i = 0; i < bpp->ridCount; i++) {
r.setUintField_offset<1>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 2:
for (i = 0; i < bpp->ridCount; i++)
{
// cout << "PTC: " << bpp->values[i] << endl;
r.setUintField_offset<2>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 4:
for (i = 0; i < bpp->ridCount; i++) {
r.setUintField_offset<4>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 8:
for (i = 0; i < bpp->ridCount; i++) {
r.setUintField_offset<2>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 4:
for (i = 0; i < bpp->ridCount; i++)
{
r.setUintField_offset<4>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 8:
for (i = 0; i < bpp->ridCount; i++)
{
// cout << "PTC: " << bpp->values[i] << endl;
r.setUintField_offset<8>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
}
r.setUintField_offset<8>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
}
}
uint64_t PassThruCommand::getLBID()
{
return 0;
return 0;
}
void PassThruCommand::nextLBID()
{
}
void PassThruCommand::createCommand(ByteStream &bs)
void PassThruCommand::createCommand(ByteStream& bs)
{
bs.advance(1);
bs >> colWidth;
Command::createCommand(bs);
bs >> colWidth;
Command::createCommand(bs);
}
void PassThruCommand::resetCommand(ByteStream &bs)
void PassThruCommand::resetCommand(ByteStream& bs)
{
}
SCommand PassThruCommand::duplicate()
{
SCommand ret;
PassThruCommand *p;
SCommand ret;
PassThruCommand* p;
ret.reset(new PassThruCommand());
p = (PassThruCommand *) ret.get();
p->colWidth = colWidth;
p->Command::duplicate(this);
return ret;
ret.reset(new PassThruCommand());
p = (PassThruCommand*) ret.get();
p->colWidth = colWidth;
p->Command::duplicate(this);
return ret;
}
bool PassThruCommand::operator==(const PassThruCommand &p) const
bool PassThruCommand::operator==(const PassThruCommand& p) const
{
if (colWidth != p.colWidth)
return false;
return true;
if (colWidth != p.colWidth)
return false;
return true;
}
bool PassThruCommand::operator!=(const PassThruCommand &p) const
bool PassThruCommand::operator!=(const PassThruCommand& p) const
{
return !(*this == p);
return !(*this == p);
}
};

47
primitives/primproc/passthrucommand.h
View File

@ -19,7 +19,7 @@
// $Id: passthrucommand.h 2035 2013-01-21 14:12:19Z rdempsey $
// C++ Interface: passthrucommand
//
// Description:
// Description:
//
//
// Author: Patrick <pleblanc@localhost.localdomain>, (C) 2008
@ -38,31 +38,34 @@ namespace primitiveprocessor
class PassThruCommand : public Command
{
public:
PassThruCommand();
virtual ~PassThruCommand();
public:
PassThruCommand();
virtual ~PassThruCommand();
void prep(int8_t outputType, bool makeAbsRids);
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
SCommand duplicate();
bool operator==(const PassThruCommand &) const;
bool operator!=(const PassThruCommand &) const;
void prep(int8_t outputType, bool makeAbsRids);
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
bool operator==(const PassThruCommand&) const;
bool operator!=(const PassThruCommand&) const;
int getCompType() const { return 0; }
private:
PassThruCommand(const PassThruCommand &);
int getCompType() const
{
return 0;
}
private:
PassThruCommand(const PassThruCommand&);
uint8_t colWidth;
uint8_t colWidth;
/* Minor optimization for projectIntoRowGroup() */
rowgroup::Row r;
uint32_t rowSize;
/* Minor optimization for projectIntoRowGroup() */
rowgroup::Row r;
uint32_t rowSize;
};
}

38
primitives/primproc/pp_logger.h
View File

@ -37,32 +37,32 @@ namespace primitiveprocessor
class Logger
{
public:
Logger();
Logger();
// critical=true logs CRITICAL; critical=false logs WARNING
void logMessage(const logging::Message::MessageID mid,
// critical=true logs CRITICAL; critical=false logs WARNING
void logMessage(const logging::Message::MessageID mid,
const logging::Message::Args& args,
bool critical=false);
void logInfoMessage(const logging::Message::MessageID mid,
const logging::Message::Args& args);
bool critical = false);
void logInfoMessage(const logging::Message::MessageID mid,
const logging::Message::Args& args);
void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 )
{
logging::Message::Args args;
args.add(msg);
logMessage(mid, args, critical);
}
void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 )
{
logging::Message::Args args;
args.add(msg);
logMessage(mid, args, critical);
}
private:
// defaults okay
//Logger(const Logger& rhs);
//Logger& operator=(const Logger& rhs);
// defaults okay
//Logger(const Logger& rhs);
//Logger& operator=(const Logger& rhs);
typedef std::map<logging::Message::MessageID, logging::Message> MsgMap;
typedef std::map<logging::Message::MessageID, logging::Message> MsgMap;
MsgMap fMsgMap;
boost::mutex fLogLock;
logging::MessageLog fMl1;
MsgMap fMsgMap;
boost::mutex fLogLock;
logging::MessageLog fMl1;
};

3681
primitives/primproc/primitiveserver.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

230
primitives/primproc/primitiveserver.h
View File

@ -47,131 +47,153 @@
#endif
#include "oamcache.h"
extern oam::OamCache *oamCache;
extern oam::OamCache* oamCache;
namespace primitiveprocessor
{
extern boost::shared_ptr<threadpool::PriorityThreadPool> OOBPool;
extern dbbc::BlockRequestProcessor **BRPp;
extern BRM::DBRM *brm;
extern boost::mutex bppLock;
extern uint32_t highPriorityThreads, medPriorityThreads, lowPriorityThreads;
extern boost::shared_ptr<threadpool::PriorityThreadPool> OOBPool;
extern dbbc::BlockRequestProcessor** BRPp;
extern BRM::DBRM* brm;
extern boost::mutex bppLock;
extern uint32_t highPriorityThreads, medPriorityThreads, lowPriorityThreads;
#ifdef PRIMPROC_STOPWATCH
extern map<pthread_t, logging::StopWatch*> stopwatchMap;
extern pthread_mutex_t stopwatchMapMutex;
extern bool stopwatchThreadCreated;
extern map<pthread_t, logging::StopWatch*> stopwatchMap;
extern pthread_mutex_t stopwatchMapMutex;
extern bool stopwatchThreadCreated;
extern void pause_(int seconds);
extern void *autoFinishStopwatchThread(void *arg);
extern void pause_(int seconds);
extern void* autoFinishStopwatchThread(void* arg);
#endif
class BPPV {
public:
BPPV();
~BPPV();
boost::shared_ptr<BatchPrimitiveProcessor> next();
void add(boost::shared_ptr<BatchPrimitiveProcessor> a);
const std::vector<boost::shared_ptr<BatchPrimitiveProcessor> > & get();
inline boost::shared_ptr<BPPSendThread> getSendThread() { return sendThread; }
void abort();
bool aborted();
volatile bool joinDataReceived;
private:
std::vector<boost::shared_ptr<BatchPrimitiveProcessor> > v;
boost::shared_ptr<BPPSendThread> sendThread;
class BPPV
{
public:
BPPV();
~BPPV();
boost::shared_ptr<BatchPrimitiveProcessor> next();
void add(boost::shared_ptr<BatchPrimitiveProcessor> a);
const std::vector<boost::shared_ptr<BatchPrimitiveProcessor> >& get();
inline boost::shared_ptr<BPPSendThread> getSendThread()
{
return sendThread;
}
void abort();
bool aborted();
volatile bool joinDataReceived;
private:
std::vector<boost::shared_ptr<BatchPrimitiveProcessor> > v;
boost::shared_ptr<BPPSendThread> sendThread;
// the instance other instances are created from
boost::shared_ptr<BatchPrimitiveProcessor> unusedInstance;
// the instance other instances are created from
boost::shared_ptr<BatchPrimitiveProcessor> unusedInstance;
// pos keeps the position of the last BPP returned by next(),
// next() will start searching at this pos on the next call.
uint32_t pos;
};
// pos keeps the position of the last BPP returned by next(),
// next() will start searching at this pos on the next call.
uint32_t pos;
};
typedef boost::shared_ptr<BPPV> SBPPV;
typedef std::map<uint32_t, SBPPV> BPPMap;
extern BPPMap bppMap;
typedef boost::shared_ptr<BPPV> SBPPV;
typedef std::map<uint32_t, SBPPV> BPPMap;
extern BPPMap bppMap;
void prefetchBlocks(uint64_t lbid, const int compType, uint32_t* rCount);
void prefetchExtent(uint64_t lbid, uint32_t ver, uint32_t txn, uint32_t* rCount);
void loadBlock(uint64_t lbid, BRM::QueryContext q, uint32_t txn, int compType, void* bufferPtr,
bool* pWasBlockInCache, uint32_t* rCount=NULL, bool LBIDTrace = false,
uint32_t sessionID = 0, bool doPrefetch=true, VSSCache *vssCache = NULL);
void loadBlockAsync(uint64_t lbid, const BRM::QueryContext &q, uint32_t txn, int CompType,
uint32_t *cCount, uint32_t *rCount, bool LBIDTrace, uint32_t sessionID,
boost::mutex *m, uint32_t *busyLoaders, boost::shared_ptr<BPPSendThread> sendThread, VSSCache* vssCache=0);
uint32_t loadBlocks(BRM::LBID_t *lbids, BRM::QueryContext q, BRM::VER_t txn, int compType,
uint8_t **bufferPtrs, uint32_t *rCount, bool LBIDTrace, uint32_t sessionID,
uint32_t blockCount, bool *wasVersioned, bool doPrefetch = true, VSSCache *vssCache = NULL);
uint32_t cacheNum(uint64_t lbid);
void buildFileName(BRM::OID_t oid, char* fileName);
void prefetchBlocks(uint64_t lbid, const int compType, uint32_t* rCount);
void prefetchExtent(uint64_t lbid, uint32_t ver, uint32_t txn, uint32_t* rCount);
void loadBlock(uint64_t lbid, BRM::QueryContext q, uint32_t txn, int compType, void* bufferPtr,
bool* pWasBlockInCache, uint32_t* rCount = NULL, bool LBIDTrace = false,
uint32_t sessionID = 0, bool doPrefetch = true, VSSCache* vssCache = NULL);
void loadBlockAsync(uint64_t lbid, const BRM::QueryContext& q, uint32_t txn, int CompType,
uint32_t* cCount, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID,
boost::mutex* m, uint32_t* busyLoaders, boost::shared_ptr<BPPSendThread> sendThread, VSSCache* vssCache = 0);
uint32_t loadBlocks(BRM::LBID_t* lbids, BRM::QueryContext q, BRM::VER_t txn, int compType,
uint8_t** bufferPtrs, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID,
uint32_t blockCount, bool* wasVersioned, bool doPrefetch = true, VSSCache* vssCache = NULL);
uint32_t cacheNum(uint64_t lbid);
void buildFileName(BRM::OID_t oid, char* fileName);
/** @brief process primitives as they arrive
*/
class PrimitiveServer
{
public:
/** @brief ctor
*/
PrimitiveServer(int serverThreads,
int serverQueueSize,
int processorWeight,
int processorQueueSize,
bool rotatingDestination,
uint32_t BRPBlocks=(1024 * 1024 * 2),
int BRPThreads=64,
int cacheCount = 8,
int maxBlocksPerRead = 128,
int readAheadBlocks=256,
uint32_t deleteBlocks = 0,
bool ptTrace=false,
double prefetchThreshold = 0,
uint64_t pmSmallSide = 0);
/** @brief process primitives as they arrive
*/
class PrimitiveServer
{
public:
/** @brief ctor
*/
PrimitiveServer(int serverThreads,
int serverQueueSize,
int processorWeight,
int processorQueueSize,
bool rotatingDestination,
uint32_t BRPBlocks = (1024 * 1024 * 2),
int BRPThreads = 64,
int cacheCount = 8,
int maxBlocksPerRead = 128,
int readAheadBlocks = 256,
uint32_t deleteBlocks = 0,
bool ptTrace = false,
double prefetchThreshold = 0,
uint64_t pmSmallSide = 0);
/** @brief dtor
*/
~PrimitiveServer();
/** @brief dtor
*/
~PrimitiveServer();
/** @brief start the primitive server
*
*/
void start();
/** @brief start the primitive server
*
*/
void start();
/** @brief get a pointer the shared processor thread pool
*/
inline boost::shared_ptr<threadpool::PriorityThreadPool> getProcessorThreadPool() const { return fProcessorPool; }
/** @brief get a pointer the shared processor thread pool
*/
inline boost::shared_ptr<threadpool::PriorityThreadPool> getProcessorThreadPool() const
{
return fProcessorPool;
}
// int fCacheCount;
const int ReadAheadBlocks() const {return fReadAheadBlocks;}
bool rotatingDestination() const {return fRotatingDestination;}
bool PTTrace() const {return fPTTrace;}
double prefetchThreshold() const { return fPrefetchThreshold; }
uint32_t ProcessorThreads() const { return highPriorityThreads + medPriorityThreads + lowPriorityThreads; }
protected:
const int ReadAheadBlocks() const
{
return fReadAheadBlocks;
}
bool rotatingDestination() const
{
return fRotatingDestination;
}
bool PTTrace() const
{
return fPTTrace;
}
double prefetchThreshold() const
{
return fPrefetchThreshold;
}
uint32_t ProcessorThreads() const
{
return highPriorityThreads + medPriorityThreads + lowPriorityThreads;
}
protected:
private:
/** @brief the thread pool used to listen for
* incoming primitive commands
*/
threadpool::ThreadPool fServerpool;
private:
/** @brief the thread pool used to listen for
* incoming primitive commands
*/
threadpool::ThreadPool fServerpool;
/** @brief the thread pool used to process
* primitive commands
*/
boost::shared_ptr<threadpool::PriorityThreadPool> fProcessorPool;
/** @brief the thread pool used to process
* primitive commands
*/
boost::shared_ptr<threadpool::PriorityThreadPool> fProcessorPool;
int fServerThreads;
int fServerQueueSize;
int fProcessorWeight;
int fProcessorQueueSize;
int fMaxBlocksPerRead;
int fReadAheadBlocks;
bool fRotatingDestination;
bool fPTTrace;
double fPrefetchThreshold;
uint64_t fPMSmallSide;
};
int fServerThreads;
int fServerQueueSize;
int fProcessorWeight;
int fProcessorQueueSize;
int fMaxBlocksPerRead;
int fReadAheadBlocks;
bool fRotatingDestination;
bool fPTTrace;
double fPrefetchThreshold;
uint64_t fPMSmallSide;
};
} // namespace primitiveprocessor

843
primitives/primproc/primproc.cpp
View File

File diff suppressed because it is too large Load Diff

32
primitives/primproc/primproc.h
View File

@ -94,25 +94,25 @@ namespace primitiveprocessor
std::cout << message1 << message2 << message3 << std::endl; \
}
enum DebugLevel /** @brief Debug level type enumeration */
{
NONE = 0, /** @brief No debug info */
STATS = 1, /** @brief stats info */
SUMMARY = 2, /** @brief Summary level debug info */
DETAIL = 3, /** @brief A little detail debug info */
VERBOSE = 4, /** @brief Detailed debug info */
};
enum DebugLevel /** @brief Debug level type enumeration */
{
NONE = 0, /** @brief No debug info */
STATS = 1, /** @brief stats info */
SUMMARY = 2, /** @brief Summary level debug info */
DETAIL = 3, /** @brief A little detail debug info */
VERBOSE = 4, /** @brief Detailed debug info */
};
bool isDebug( const DebugLevel level );
bool isDebug( const DebugLevel level );
const int MAX_BUFFER_SIZE = 32768 * 2;
const int MAX_BUFFER_SIZE = 32768 * 2;
// message log globals
//const logging::LoggingID lid1(28);
//extern logging::Message msg16;
//extern logging::MessageLog ml1;
//extern boost::mutex logLock;
extern Logger* mlp;
// message log globals
//const logging::LoggingID lid1(28);
//extern logging::Message msg16;
//extern logging::MessageLog ml1;
//extern boost::mutex logLock;
extern Logger* mlp;
}
#endif //PRIMPROC_H

368
primitives/primproc/pseudocc.cpp
View File

@ -22,7 +22,8 @@
using namespace std;
using namespace execplan;
namespace primitiveprocessor {
namespace primitiveprocessor
{
PseudoCC::PseudoCC() : ColumnCommand()
{
@ -34,29 +35,30 @@ PseudoCC::~PseudoCC()
SCommand PseudoCC::duplicate()
{
SCommand ret;
PseudoCC *pseudo;
SCommand ret;
PseudoCC* pseudo;
pseudo = new PseudoCC();
ret.reset(pseudo);
pseudo->function = function;
pseudo->valueFromUM = valueFromUM;
ColumnCommand::duplicate(pseudo);
return ret;
pseudo = new PseudoCC();
ret.reset(pseudo);
pseudo->function = function;
pseudo->valueFromUM = valueFromUM;
ColumnCommand::duplicate(pseudo);
return ret;
}
void PseudoCC::createCommand(messageqcpp::ByteStream &bs)
void PseudoCC::createCommand(messageqcpp::ByteStream& bs)
{
bs.advance(1);
bs >> function;
ColumnCommand::createCommand(bs);
bs.advance(1);
bs >> function;
ColumnCommand::createCommand(bs);
}
void PseudoCC::resetCommand(messageqcpp::ByteStream &bs)
void PseudoCC::resetCommand(messageqcpp::ByteStream& bs)
{
if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN || function == PSEUDO_EXTENTID)
bs >> valueFromUM;
ColumnCommand::resetCommand(bs);
if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN || function == PSEUDO_EXTENTID)
bs >> valueFromUM;
ColumnCommand::resetCommand(bs);
}
#ifdef __GNUC__
@ -69,147 +71,197 @@ __attribute__((optimize("no-tree-vectorize")))
void PseudoCC::loadData()
{
switch (function) {
case PSEUDO_PM:
switch(colType.colWidth) {
case 1:
loadPMNumber<uint8_t>();
break;
case 2:
loadPMNumber<uint16_t>();
break;
case 4:
loadPMNumber<uint32_t>();
break;
case 8:
loadPMNumber<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_EXTENTRELATIVERID:
switch(colType.colWidth) {
case 1:
loadRIDs<uint8_t>();
break;
case 2:
loadRIDs<uint16_t>();
break;
case 4:
loadRIDs<uint32_t>();
break;
case 8:
loadRIDs<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENT:
switch(colType.colWidth) {
case 1:
loadSegmentNum<uint8_t>();
break;
case 2:
loadSegmentNum<uint16_t>();
break;
case 4:
loadSegmentNum<uint32_t>();
break;
case 8:
loadSegmentNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENTDIR:
switch(colType.colWidth) {
case 1:
loadPartitionNum<uint8_t>();
break;
case 2:
loadPartitionNum<uint16_t>();
break;
case 4:
loadPartitionNum<uint32_t>();
break;
case 8:
loadPartitionNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_BLOCKID:
switch(colType.colWidth) {
case 1:
loadLBID<uint8_t>();
break;
case 2:
loadLBID<uint16_t>();
break;
case 4:
loadLBID<uint32_t>();
break;
case 8:
loadLBID<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_DBROOT:
switch(colType.colWidth) {
case 1:
loadDBRootNum<uint8_t>();
break;
case 2:
loadDBRootNum<uint16_t>();
break;
case 4:
loadDBRootNum<uint32_t>();
break;
case 8:
loadDBRootNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
/* cases where the value to use is sent from the UM */
case PSEUDO_EXTENTMAX:
case PSEUDO_EXTENTMIN:
case PSEUDO_EXTENTID:
switch(colType.colWidth) {
case 1:
loadSingleValue<int8_t>(valueFromUM);
break;
case 2:
loadSingleValue<int16_t>(valueFromUM);
break;
case 4:
loadSingleValue<int32_t>(valueFromUM);
break;
case 8:
loadSingleValue<int64_t>(valueFromUM);
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
default:
cout << "PC::loadData(): bad function" << endl;
break;
}
switch (function)
{
case PSEUDO_PM:
switch (colType.colWidth)
{
case 1:
loadPMNumber<uint8_t>();
break;
case 2:
loadPMNumber<uint16_t>();
break;
case 4:
loadPMNumber<uint32_t>();
break;
case 8:
loadPMNumber<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_EXTENTRELATIVERID:
switch (colType.colWidth)
{
case 1:
loadRIDs<uint8_t>();
break;
case 2:
loadRIDs<uint16_t>();
break;
case 4:
loadRIDs<uint32_t>();
break;
case 8:
loadRIDs<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENT:
switch (colType.colWidth)
{
case 1:
loadSegmentNum<uint8_t>();
break;
case 2:
loadSegmentNum<uint16_t>();
break;
case 4:
loadSegmentNum<uint32_t>();
break;
case 8:
loadSegmentNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENTDIR:
switch (colType.colWidth)
{
case 1:
loadPartitionNum<uint8_t>();
break;
case 2:
loadPartitionNum<uint16_t>();
break;
case 4:
loadPartitionNum<uint32_t>();
break;
case 8:
loadPartitionNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_BLOCKID:
switch (colType.colWidth)
{
case 1:
loadLBID<uint8_t>();
break;
case 2:
loadLBID<uint16_t>();
break;
case 4:
loadLBID<uint32_t>();
break;
case 8:
loadLBID<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_DBROOT:
switch (colType.colWidth)
{
case 1:
loadDBRootNum<uint8_t>();
break;
case 2:
loadDBRootNum<uint16_t>();
break;
case 4:
loadDBRootNum<uint32_t>();
break;
case 8:
loadDBRootNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
/* cases where the value to use is sent from the UM */
case PSEUDO_EXTENTMAX:
case PSEUDO_EXTENTMIN:
case PSEUDO_EXTENTID:
switch (colType.colWidth)
{
case 1:
loadSingleValue<int8_t>(valueFromUM);
break;
case 2:
loadSingleValue<int16_t>(valueFromUM);
break;
case 4:
loadSingleValue<int32_t>(valueFromUM);
break;
case 8:
loadSingleValue<int64_t>(valueFromUM);
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
default:
cout << "PC::loadData(): bad function" << endl;
break;
}
}
}

95
primitives/primproc/pseudocc.h
View File

@ -22,32 +22,33 @@
#include "columncommand.h"
#include "blocksize.h"
namespace primitiveprocessor {
namespace primitiveprocessor
{
class PseudoCC : public ColumnCommand
{
public:
PseudoCC();
virtual ~PseudoCC();
PseudoCC();
virtual ~PseudoCC();
virtual SCommand duplicate();
virtual void createCommand(messageqcpp::ByteStream &);
virtual void resetCommand(messageqcpp::ByteStream &);
virtual SCommand duplicate();
virtual void createCommand(messageqcpp::ByteStream&);
virtual void resetCommand(messageqcpp::ByteStream&);
protected:
virtual void loadData();
virtual void loadData();
private:
template<typename W> void loadSingleValue(W val);
template<typename W> void loadPMNumber();
template<typename W> void loadRIDs();
template<typename W> void loadSegmentNum();
template<typename W> void loadDBRootNum();
template<typename W> void loadPartitionNum();
template<typename W> void loadLBID();
template<typename W> void loadSingleValue(W val);
template<typename W> void loadPMNumber();
template<typename W> void loadRIDs();
template<typename W> void loadSegmentNum();
template<typename W> void loadDBRootNum();
template<typename W> void loadPartitionNum();
template<typename W> void loadLBID();
uint32_t function;
uint64_t valueFromUM;
uint32_t function;
uint64_t valueFromUM;
};
@ -61,65 +62,71 @@ __attribute__((optimize("no-tree-vectorize")))
#endif
void PseudoCC::loadSingleValue(W val)
{
W *bData = (W *) bpp->blockData;
for (uint32_t i = 0; i < BLOCK_SIZE; i++)
bData[i] = val; // this breaks when using tree-vectorization
//memcpy(&bData[i], &val, sizeof(W)); // this always works
W* bData = (W*) bpp->blockData;
for (uint32_t i = 0; i < BLOCK_SIZE; i++)
bData[i] = val; // this breaks when using tree-vectorization
//memcpy(&bData[i], &val, sizeof(W)); // this always works
}
template<typename W>
void PseudoCC::loadPMNumber()
{
uint32_t pmNum = oamCache->getLocalPMId();
loadSingleValue<W>(pmNum);
uint32_t pmNum = oamCache->getLocalPMId();
loadSingleValue<W>(pmNum);
}
/* This returns extent-relative rids */
template<typename W>
void PseudoCC::loadRIDs()
{
uint32_t i;
uint64_t baseRid = rowgroup::getExtentRelativeRid(bpp->baseRid);
W *bData = (W *) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++) {
//W val = baseRid + i;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = baseRid + i; // breaks with tree-vectorization
}
uint32_t i;
uint64_t baseRid = rowgroup::getExtentRelativeRid(bpp->baseRid);
W* bData = (W*) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++)
{
//W val = baseRid + i;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = baseRid + i; // breaks with tree-vectorization
}
}
template<typename W>
void PseudoCC::loadSegmentNum()
{
uint16_t segNum;
rowgroup::getLocationFromRid(bpp->baseRid, NULL, &segNum, NULL, NULL);
loadSingleValue<W>(segNum);
uint16_t segNum;
rowgroup::getLocationFromRid(bpp->baseRid, NULL, &segNum, NULL, NULL);
loadSingleValue<W>(segNum);
}
template<typename W>
void PseudoCC::loadPartitionNum()
{
uint32_t partNum;
rowgroup::getLocationFromRid(bpp->baseRid, &partNum, NULL, NULL, NULL);
loadSingleValue<W>(partNum);
uint32_t partNum;
rowgroup::getLocationFromRid(bpp->baseRid, &partNum, NULL, NULL, NULL);
loadSingleValue<W>(partNum);
}
template<typename W>
void PseudoCC::loadDBRootNum()
{
loadSingleValue<W>(bpp->dbRoot);
loadSingleValue<W>(bpp->dbRoot);
}
template<typename W>
void PseudoCC::loadLBID()
{
uint32_t i;
W *bData = (W *) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++) {
//W val = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; // breaks with tree-vectorization
}
uint32_t i;
W* bData = (W*) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++)
{
//W val = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; // breaks with tree-vectorization
}
}
}

28
primitives/primproc/resource.h
View File

@ -1,14 +1,14 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by PrimProc.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by PrimProc.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

254
primitives/primproc/rtscommand.cpp
View File

@ -51,174 +51,202 @@ RTSCommand::~RTSCommand()
void RTSCommand::execute()
{
throw logic_error("RTSCommand shouldn't be used for filter steps");
throw logic_error("RTSCommand shouldn't be used for filter steps");
}
void RTSCommand::project()
{
uint32_t i;
uint32_t i;
if (passThru) {
if (passThru)
{
// if (bpp->absRids.get() == NULL)
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
// need something in values
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
dict.project();
}
else {
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
col.execute(tmpValues);
// need something in values
if (old_rc != bpp->ridCount) {
ostringstream os;
dict.project();
}
else
{
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else {
throw PrimitiveColumnProjectResultExcept(os.str());
}
}
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
dict.project(tmpValues);
}
col.execute(tmpValues);
if (old_rc != bpp->ridCount)
{
ostringstream os;
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else
{
throw PrimitiveColumnProjectResultExcept(os.str());
}
}
dict.project(tmpValues);
}
}
void RTSCommand::projectIntoRowGroup(RowGroup &rg, uint32_t colNum)
void RTSCommand::projectIntoRowGroup(RowGroup& rg, uint32_t colNum)
{
uint32_t i;
uint32_t i;
if (passThru) {
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
dict.projectIntoRowGroup(rg, colNum);
}
else {
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
if (passThru)
{
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
col.execute(tmpValues);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
if (old_rc != bpp->ridCount) {
dict.projectIntoRowGroup(rg, colNum);
}
else
{
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
ostringstream os;
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else
throw PrimitiveColumnProjectResultExcept(os.str());
}
col.execute(tmpValues);
dict.projectIntoRowGroup(rg, tmpValues, colNum);
}
if (old_rc != bpp->ridCount)
{
ostringstream os;
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else
throw PrimitiveColumnProjectResultExcept(os.str());
}
dict.projectIntoRowGroup(rg, tmpValues, colNum);
}
}
uint64_t RTSCommand::getLBID()
{
if (!passThru)
return col.getLBID();
else
return 0;
if (!passThru)
return col.getLBID();
else
return 0;
}
void RTSCommand::nextLBID()
{
if (!passThru)
col.nextLBID();
if (!passThru)
col.nextLBID();
}
void RTSCommand::prep(int8_t outputType, bool makeAbsRids)
{
if (!passThru)
col.prep(OT_BOTH, true);
dict.prep(OT_DATAVALUE, true);
if (!passThru)
col.prep(OT_BOTH, true);
dict.prep(OT_DATAVALUE, true);
}
void RTSCommand::createCommand(ByteStream &bs)
void RTSCommand::createCommand(ByteStream& bs)
{
bs.advance(1);
bs >> passThru;
if (!passThru) {
col.createCommand(bs);
}
dict.createCommand(bs);
Command::createCommand(bs);
bs.advance(1);
bs >> passThru;
if (!passThru)
{
col.createCommand(bs);
}
dict.createCommand(bs);
Command::createCommand(bs);
}
void RTSCommand::resetCommand(ByteStream &bs)
void RTSCommand::resetCommand(ByteStream& bs)
{
if (!passThru)
col.resetCommand(bs);
dict.resetCommand(bs);
if (!passThru)
col.resetCommand(bs);
dict.resetCommand(bs);
}
SCommand RTSCommand::duplicate()
{
SCommand ret;
RTSCommand *rts;
SCommand ret;
RTSCommand* rts;
ret.reset(new RTSCommand());
rts = (RTSCommand *) ret.get();
rts->passThru = passThru;
if (!passThru)
rts->col = col;
rts->dict = dict;
rts->Command::duplicate(this);
return ret;
ret.reset(new RTSCommand());
rts = (RTSCommand*) ret.get();
rts->passThru = passThru;
if (!passThru)
rts->col = col;
rts->dict = dict;
rts->Command::duplicate(this);
return ret;
}
bool RTSCommand::operator==(const RTSCommand &r) const
bool RTSCommand::operator==(const RTSCommand& r) const
{
if (passThru != r.passThru)
return false;
if (!passThru)
if (col != r.col)
return false;
if (absNull != r.absNull)
return false;
if (dict != dict)
return false;
return true;
if (passThru != r.passThru)
return false;
if (!passThru)
if (col != r.col)
return false;
if (absNull != r.absNull)
return false;
if (dict != dict)
return false;
return true;
}
bool RTSCommand::operator!=(const RTSCommand &r) const
bool RTSCommand::operator!=(const RTSCommand& r) const
{
return !(*this == r);
return !(*this == r);
}
void RTSCommand::getLBIDList(uint32_t loopCount, vector<int64_t> *lbids)
void RTSCommand::getLBIDList(uint32_t loopCount, vector<int64_t>* lbids)
{
dict.getLBIDList(loopCount, lbids);
if (!passThru)
col.getLBIDList(loopCount, lbids);
dict.getLBIDList(loopCount, lbids);
if (!passThru)
col.getLBIDList(loopCount, lbids);
}
void RTSCommand::setBatchPrimitiveProcessor(BatchPrimitiveProcessor *b)
void RTSCommand::setBatchPrimitiveProcessor(BatchPrimitiveProcessor* b)
{
Command::setBatchPrimitiveProcessor(b);
dict.setBatchPrimitiveProcessor(b);
if (!passThru)
col.setBatchPrimitiveProcessor(b);
Command::setBatchPrimitiveProcessor(b);
dict.setBatchPrimitiveProcessor(b);
if (!passThru)
col.setBatchPrimitiveProcessor(b);
}
};

67
primitives/primproc/rtscommand.h
View File

@ -19,7 +19,7 @@
// $Id: rtscommand.h 2035 2013-01-21 14:12:19Z rdempsey $
// C++ Interface: rtscommand
//
// Description:
// Description:
//
//
// Author: Patrick <pleblanc@localhost.localdomain>, (C) 2008
@ -39,38 +39,47 @@ namespace primitiveprocessor
class RTSCommand : public Command
{
public:
RTSCommand();
virtual ~RTSCommand();
public:
RTSCommand();
virtual ~RTSCommand();
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
SCommand duplicate();
bool operator==(const RTSCommand &) const;
bool operator!=(const RTSCommand &) const;
void setBatchPrimitiveProcessor(BatchPrimitiveProcessor *);
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
bool operator==(const RTSCommand&) const;
bool operator!=(const RTSCommand&) const;
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
uint8_t isPassThru() { return passThru; }
void setAbsNull(bool a = true) { absNull = a; }
void getLBIDList(uint32_t loopCount, std::vector<int64_t> *lbids);
void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*);
//TODO: do we need to reference either col or dict?
int getCompType() const { return dict.getCompType(); }
private:
RTSCommand(const RTSCommand &);
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
uint8_t isPassThru()
{
return passThru;
}
void setAbsNull(bool a = true)
{
absNull = a;
}
void getLBIDList(uint32_t loopCount, std::vector<int64_t>* lbids);
ColumnCommand col;
DictStep dict;
uint8_t passThru;
bool absNull;
//TODO: do we need to reference either col or dict?
int getCompType() const
{
return dict.getCompType();
}
private:
RTSCommand(const RTSCommand&);
ColumnCommand col;
DictStep dict;
uint8_t passThru;
bool absNull;
};
}

287
primitives/primproc/tdriver-umsocksel.cpp
View File

@ -41,158 +41,171 @@ using namespace primitiveprocessor;
int main()
{
// Columnstore.xml file should be configured as follows:
// um1: 10.100.4.85 and 10.100.5.85
// um2: 10.101.4.85 and 10.101.5.85
sockaddr_in sa = { 1, 0, {0} , {' '} };
char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"};
// Columnstore.xml file should be configured as follows:
// um1: 10.100.4.85 and 10.100.5.85
// um2: 10.101.4.85 and 10.101.5.85
sockaddr_in sa = { 1, 0, {0}, {' '} };
char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"};
// These are the IP addresses we use to test runtime connections
// "not" in the Columnstore.xml file.
sockaddr_in saUnknown = { 1, 0, {0} , {' '} };
char* ipsUnknown[]={"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"};
// These are the IP addresses we use to test runtime connections
// "not" in the Columnstore.xml file.
sockaddr_in saUnknown = { 1, 0, {0}, {' '} };
char* ipsUnknown[] = {"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"};
//--------------------------------------------------------------------------
// Test initialization
//--------------------------------------------------------------------------
UmSocketSelector* sockSel = UmSocketSelector::instance();
//--------------------------------------------------------------------------
// Test initialization
//--------------------------------------------------------------------------
UmSocketSelector* sockSel = UmSocketSelector::instance();
std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl;
std::cout << std::endl << "----Dump1 after initialization..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump1 End...................." << std::endl << std::endl;
std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl;
std::cout << std::endl << "----Dump1 after initialization..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump1 End...................." << std::endl << std::endl;
IOSocket sock[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ips[i], &sa.sin_addr);
for (int j=0; j<4; j++)
{
sock[i][j].setSocketImpl(new InetStreamSocket());
sa.sin_port = htons((i*4)+j);
sock[i][j].sa( sa );
sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
IOSocket sock[4][4];
std::cout << std::endl << "----Dump2 after adding 16 connections..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump2 End...................." << std::endl << std::endl;
for (int i = 0; i < 4; i++)
{
inet_aton(ips[i], &sa.sin_addr);
//--------------------------------------------------------------------------
// Test socket/port selection
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<17; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
for (int j = 0; j < 4; j++)
{
sock[i][j].setSocketImpl(new InetStreamSocket());
sa.sin_port = htons((i * 4) + j);
sock[i][j].sa( sa );
sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << std::endl << "----Dump2 after adding 16 connections..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump2 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k = 0; k < 17; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
#if 1
if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
#else
if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
#endif
std::cout << "no nextIP found for " << sock[0][0] << std::endl;
}
for (unsigned k=0; k<7; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sock[2][0] << std::endl;
}
std::cout << std::endl;
std::cout << "----Dump3 after selecting 17 connections from IP " <<
ips[0] << "; and 7 connections from IP " << ips[2] << " ..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump3 End...................." << std::endl << std::endl;
std::cout << "no nextIP found for " << sock[0][0] << std::endl;
}
//--------------------------------------------------------------------------
// Test connection deletions
//--------------------------------------------------------------------------
for (unsigned k=0; k<4; k++)
{
sockSel->delConnection( sock[k][0] );
}
std::cout << "----Dump4 after deleting first connection for each IP..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump4 End...................." << std::endl << std::endl;
for (unsigned k = 0; k < 7; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
//--------------------------------------------------------------------------
// Test addition of unknown connections
//--------------------------------------------------------------------------
IOSocket sockUnknown[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ipsUnknown[i], &saUnknown.sin_addr);
for (int j=0; j<4; j++)
{
sockUnknown[i][j].setSocketImpl(new InetStreamSocket());
saUnknown.sin_port = htons((i*4)+j);
sockUnknown[i][j].sa( saUnknown );
sockSel->addConnection(
SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sock[2][0] << std::endl;
}
std::cout << "----Dump5 after adding connections for unknown IP's..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump5 End...................." << std::endl << std::endl;
std::cout << std::endl;
std::cout << "----Dump3 after selecting 17 connections from IP " <<
ips[0] << "; and 7 connections from IP " << ips[2] << " ..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump3 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test resetting of "next" indexes after deleting all sockets from a
// specific IP address for which the "next" index is pointing.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[1][1] );
sockSel->delConnection( sock[1][2] );
sockSel->delConnection( sock[1][3] );
//--------------------------------------------------------------------------
// Test connection deletions
//--------------------------------------------------------------------------
for (unsigned k = 0; k < 4; k++)
{
sockSel->delConnection( sock[k][0] );
}
std::cout << "----Dump6 after deleting all connections for IP " << ips[1] <<
" ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump6 End...................." << std::endl << std::endl;
std::cout << "----Dump4 after deleting first connection for each IP..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump4 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection for an unknown module
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<11; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sockUnknown[2][0]<<std::endl;
}
std::cout << std::endl;
std::cout << "----Dump7 after selecting 11 connections for IP " <<
ipsUnknown[2] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump7 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test addition of unknown connections
//--------------------------------------------------------------------------
IOSocket sockUnknown[4][4];
//--------------------------------------------------------------------------
// Test deletion of last socket/port connection and resetting if the
// "next" index that is pointing to it.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[3][3] );
std::cout << "----Dump8 after deleting last connection for IP " <<
ips[3] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump8 End...................." << std::endl << std::endl;
for (int i = 0; i < 4; i++)
{
inet_aton(ipsUnknown[i], &saUnknown.sin_addr);
return 0;
for (int j = 0; j < 4; j++)
{
sockUnknown[i][j].setSocketImpl(new InetStreamSocket());
saUnknown.sin_port = htons((i * 4) + j);
sockUnknown[i][j].sa( saUnknown );
sockSel->addConnection(
SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << "----Dump5 after adding connections for unknown IP's..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump5 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test resetting of "next" indexes after deleting all sockets from a
// specific IP address for which the "next" index is pointing.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[1][1] );
sockSel->delConnection( sock[1][2] );
sockSel->delConnection( sock[1][3] );
std::cout << "----Dump6 after deleting all connections for IP " << ips[1] <<
" ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump6 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection for an unknown module
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k = 0; k < 11; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sockUnknown[2][0] << std::endl;
}
std::cout << std::endl;
std::cout << "----Dump7 after selecting 11 connections for IP " <<
ipsUnknown[2] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump7 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test deletion of last socket/port connection and resetting if the
// "next" index that is pointing to it.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[3][3] );
std::cout << "----Dump8 after deleting last connection for IP " <<
ips[3] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump8 End...................." << std::endl << std::endl;
return 0;
}

48
primitives/primproc/tdriver.cpp
View File

@ -47,7 +47,7 @@ void testColByScan()
cout << "Sending COL_BY_SCAN primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ISMPacketHeader packetHeader;
packetHeader.Reserve = 0;
@ -91,6 +91,7 @@ void testColByScan()
ibs = proc.read();
int messageLen = ibs.length();
if (messageLen)
{
cout << "Received results" << endl << endl;
@ -131,12 +132,13 @@ void testColByScan()
char* data = new char[remaining];
memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining);
char *ptr = data;
char* ptr = data;
for(int i = 0; i < remaining; i++)
for (int i = 0; i < remaining; i++)
{
for (int j = 0; j < 10 && j < remaining; j++)
printf("%02x ", *ptr++);
printf("\n");
}
@ -154,7 +156,7 @@ void testColByRid()
cout << "Sending COL_BY_RID primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ISMPacketHeader packetHeader;
packetHeader.Reserve = 0;
@ -198,6 +200,7 @@ void testColByRid()
ibs = proc.read();
int messageLen = ibs.length();
if (messageLen)
{
@ -239,12 +242,13 @@ void testColByRid()
char* data = new char[remaining];
memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining);
char *ptr = data;
char* ptr = data;
for(int i = 0; i < remaining; i++)
for (int i = 0; i < remaining; i++)
{
for (int j = 0; j < 10 && j < remaining; j++)
printf("%02x ", *ptr++);
printf("\n");
}
@ -263,7 +267,7 @@ void testColAggByScan()
cout << "Sending COL_AGG_BY_SCAN primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ISMPacketHeader packetHeader;
packetHeader.Reserve = 0;
@ -298,7 +302,7 @@ void testColAggByScan()
memmove(message + sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader), &TempData, sizeof(Int64));
obs.append((messageqcpp::ByteStream::byte*)message, sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader)
+ sizeof(Int64));
+ sizeof(Int64));
cout << "Sending to Primitive Server" << endl;
@ -308,6 +312,7 @@ void testColAggByScan()
ibs = proc.read();
int messageLen = ibs.length();
if (messageLen)
{
ISMPacketHeader pktHeader;
@ -351,12 +356,13 @@ void testColAggByScan()
char* data = new char[remaining];
memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining);
char *ptr = data;
char* ptr = data;
for(int i = 0; i < remaining; i++)
for (int i = 0; i < remaining; i++)
{
for (int j = 0; j < 10 && j < remaining; j++)
printf("%02x ", *ptr++);
printf("\n");
}
@ -374,7 +380,7 @@ void testColAggByRid()
cout << "Sending COL_AGG_BY_RID primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ByteStream::octbyte value64;
ByteStream::quadbyte value32;
@ -496,6 +502,7 @@ void testColAggByRid()
cout << "Sent... awaiting results" << endl;
ibs = proc.read();
if (ibs.length() > 0)
{
ISMPacketHeader pktHeader;
@ -583,6 +590,7 @@ void testColAggByRid()
cout << "Data" << endl;
cout << "----" << endl << endl;
cout << "Data Size: " << ibs.length() << endl;
for (int i = 0; i < colAggResult.NVALS && ibs.length(); i++)
{
if (colAggByRID.OutputType == 1 || colAggByRID.OutputType == 3 )
@ -590,9 +598,11 @@ void testColAggByRid()
ibs >> value16;
cout << "RID: " << value16 << endl;
}
if (colAggByRID.OutputType == 2 || colAggByRID.OutputType == 3)
{
cout << "Token: ";
switch (colAggByRID.DataSize)
{
case 1:
@ -629,7 +639,7 @@ void testDictTokenByIndexCompare()
cout << "Sending DICT_TOKEN_BY_INDEX_COMPARE primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ByteStream::octbyte value64;
ByteStream::quadbyte value32;
@ -741,9 +751,11 @@ void testDictTokenByIndexCompare()
ibs = proc.read();
cout << "Data" << endl;
cout << "----" << endl << endl;
if (ibs.length() > 0)
{
cout << "Data Size: " << ibs.length() << endl;
while (ibs.length())
{
ibs >> value16;
@ -764,7 +776,7 @@ void testDictSignature()
cout << "DICT_SIGNATURE primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ByteStream::octbyte value64;
ByteStream::quadbyte value32;
@ -877,9 +889,11 @@ void testDictSignature()
cout << "Data" << endl;
cout << "----" << endl << endl;
if (ibs.length() > 0)
{
cout << "Data Size: " << ibs.length() << endl;
while (ibs.length())
{
ibs >> value16;
@ -914,9 +928,9 @@ int main(int argc, char* argv[])
po::options_description desc ("Allowed options");
desc.add_options ()
("help", "produce help message")
("all", "process all tests" )
("loop", "loop processing all tests");
("help", "produce help message")
("all", "process all tests" )
("loop", "loop processing all tests");
po::variables_map vm;
po::store (po::parse_command_line (argc, argv, desc), vm);
po::notify (vm);
@ -932,7 +946,7 @@ int main(int argc, char* argv[])
}
else if (vm.count ("loop"))
{
while(1)
while (1)
{
testColByScan();
testColByRid();

40
primitives/primproc/udf.cpp
View File

@ -39,29 +39,33 @@ namespace primitiveprocessor
void loadUDFs()
{
#ifndef _MSC_VER
int flags = RTLD_NOW;
int flags = RTLD_NOW;
void* libPtr = 0;
void* libPtr = 0;
UDFFcnPtr_t fcnPtr;
UDFFcnPtr_t fcnPtr;
libPtr = dlopen(UdfLibName.c_str(), flags);
libPtr = dlopen(UdfLibName.c_str(), flags);
if (libPtr == 0)
return;
if (libPtr == 0)
return;
unsigned i = 1;
for (;;)
{
ostringstream oss;
oss << "cpfunc" << i;
fcnPtr = (UDFFcnPtr_t)dlsym(libPtr, oss.str().c_str());
if (fcnPtr == 0)
break;
UDFFcnMap[i] = fcnPtr;
i++;
}
cout << "loaded " << UDFFcnMap.size() << " UDF's" << endl;
unsigned i = 1;
for (;;)
{
ostringstream oss;
oss << "cpfunc" << i;
fcnPtr = (UDFFcnPtr_t)dlsym(libPtr, oss.str().c_str());
if (fcnPtr == 0)
break;
UDFFcnMap[i] = fcnPtr;
i++;
}
cout << "loaded " << UDFFcnMap.size() << " UDF's" << endl;
#endif
}

637
primitives/primproc/umsocketselector.cpp
View File

@ -55,9 +55,9 @@ using namespace oam;
namespace primitiveprocessor
{
/*static*/ const int32_t UmIPSocketConns::NEXT_IOSOCKET_UNASSIGNED = -1;
/*static*/ const int32_t UmModuleIPs::NEXT_IP_SOCKET_UNASSIGNED = -1;
/*static*/ UmSocketSelector* UmSocketSelector::fpUmSocketSelector = 0;
/*static*/ const int32_t UmIPSocketConns::NEXT_IOSOCKET_UNASSIGNED = -1;
/*static*/ const int32_t UmModuleIPs::NEXT_IP_SOCKET_UNASSIGNED = -1;
/*static*/ UmSocketSelector* UmSocketSelector::fpUmSocketSelector = 0;
//------------------------------------------------------------------------------
// UmSocketSelector methods
@ -67,12 +67,12 @@ namespace primitiveprocessor
/* static */ UmSocketSelector*
UmSocketSelector::instance()
{
if (fpUmSocketSelector == 0)
{
fpUmSocketSelector = new UmSocketSelector();
}
if (fpUmSocketSelector == 0)
{
fpUmSocketSelector = new UmSocketSelector();
}
return fpUmSocketSelector;
return fpUmSocketSelector;
}
//------------------------------------------------------------------------------
@ -80,7 +80,7 @@ UmSocketSelector::instance()
//------------------------------------------------------------------------------
UmSocketSelector::UmSocketSelector()
{
loadUMModuleInfo();
loadUMModuleInfo();
}
//------------------------------------------------------------------------------
@ -90,14 +90,14 @@ UmSocketSelector::UmSocketSelector()
uint32_t
UmSocketSelector::ipAddressCount() const
{
uint32_t ipCount = 0;
uint32_t ipCount = 0;
for (unsigned int i=0; i<fUmModuleIPs.size(); ++i)
{
ipCount += fUmModuleIPs[i]->ipAddressCount();
}
for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i)
{
ipCount += fUmModuleIPs[i]->ipAddressCount();
}
return ipCount;
return ipCount;
}
//------------------------------------------------------------------------------
@ -109,67 +109,68 @@ UmSocketSelector::ipAddressCount() const
void
UmSocketSelector::loadUMModuleInfo()
{
Oam oam;
ModuleTypeConfig moduleTypeConfig;
const std::string UM_MODTYPE("um");
Oam oam;
ModuleTypeConfig moduleTypeConfig;
const std::string UM_MODTYPE("um");
oam.getSystemConfig(UM_MODTYPE, moduleTypeConfig);
oam.getSystemConfig(UM_MODTYPE, moduleTypeConfig);
int moduleCount= moduleTypeConfig.ModuleCount;
std::string moduleType = moduleTypeConfig.ModuleType;
int moduleCount = moduleTypeConfig.ModuleCount;
std::string moduleType = moduleTypeConfig.ModuleType;
#ifdef LOAD_MODULE_DEBUG
std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl;
std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl;
std::cout << "ModuleCount = " << moduleCount << std::endl;
std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl;
std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl;
std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl;
std::cout << "ModuleCount = " << moduleCount << std::endl;
std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl;
#endif
if ( moduleCount > 0 )
{
//..Loop through the list of UM modules
for (DeviceNetworkList::iterator iter1 =
moduleTypeConfig.ModuleNetworkList.begin();
(iter1 != moduleTypeConfig.ModuleNetworkList.end());
++iter1)
{
std::string moduleName = iter1->DeviceName;
if ( moduleCount > 0 )
{
//..Loop through the list of UM modules
for (DeviceNetworkList::iterator iter1 =
moduleTypeConfig.ModuleNetworkList.begin();
(iter1 != moduleTypeConfig.ModuleNetworkList.end());
++iter1)
{
std::string moduleName = iter1->DeviceName;
#ifdef LOAD_MODULE_DEBUG
std::cout << "ModuleName-" << moduleName << std::endl;
std::cout << "ModuleName-" << moduleName << std::endl;
#endif
//..Assign the UM index based on whether it is a new UM or one
// we have seen before
unsigned int umIdx = findOrAddUm( moduleName );
//..Assign the UM index based on whether it is a new UM or one
// we have seen before
unsigned int umIdx = findOrAddUm( moduleName );
//..Get the list of IP addresses (NIC's) for this UM module
for (HostConfigList::iterator iter2 = iter1->hostConfigList.begin();
(iter2 != iter1->hostConfigList.end());
++iter2)
{
std::string ipAddr = iter2->IPAddr;
//..Get the list of IP addresses (NIC's) for this UM module
for (HostConfigList::iterator iter2 = iter1->hostConfigList.begin();
(iter2 != iter1->hostConfigList.end());
++iter2)
{
std::string ipAddr = iter2->IPAddr;
#ifdef LOAD_MODULE_DEBUG
std::cout << " NIC-" << iter2->NicID <<
"; host-" << iter2->HostName <<
"; IP-" << ipAddr << std::endl;
std::cout << " NIC-" << iter2->NicID <<
"; host-" << iter2->HostName <<
"; IP-" << ipAddr << std::endl;
#endif
struct in_addr ip;
if ( inet_aton(ipAddr.c_str(), &ip ) )
{
fIpAddressUmMap[ ip.s_addr ] = umIdx;
fUmModuleIPs[umIdx]->addIP( ip.s_addr );
}
else
{
std::cerr << "Invalid IP address in SystemModuleConfig "
"section: " << ipAddr << std::endl;
}
} // loop through the IP addresses for a UM module
} // loop through the list of UM modules
} // moduleCount > 0
struct in_addr ip;
if ( inet_aton(ipAddr.c_str(), &ip ) )
{
fIpAddressUmMap[ ip.s_addr ] = umIdx;
fUmModuleIPs[umIdx]->addIP( ip.s_addr );
}
else
{
std::cerr << "Invalid IP address in SystemModuleConfig "
"section: " << ipAddr << std::endl;
}
} // loop through the IP addresses for a UM module
} // loop through the list of UM modules
} // moduleCount > 0
}
//------------------------------------------------------------------------------
@ -180,21 +181,22 @@ UmSocketSelector::loadUMModuleInfo()
unsigned int
UmSocketSelector::findOrAddUm( const std::string& moduleName )
{
unsigned int umIdx = std::numeric_limits<unsigned int>::max();
for (unsigned int i=0; i<fUmModuleIPs.size(); ++i)
{
if (fUmModuleIPs[i]->moduleName() == moduleName)
{
umIdx = i;
return umIdx;
}
}
unsigned int umIdx = std::numeric_limits<unsigned int>::max();
//..We have encountered a new UM module we should add to the list
fUmModuleIPs.push_back( SP_UM_MODIPS(new UmModuleIPs(moduleName)) );
umIdx = fUmModuleIPs.size() - 1;
for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i)
{
if (fUmModuleIPs[i]->moduleName() == moduleName)
{
umIdx = i;
return umIdx;
}
}
return umIdx;
//..We have encountered a new UM module we should add to the list
fUmModuleIPs.push_back( SP_UM_MODIPS(new UmModuleIPs(moduleName)) );
umIdx = fUmModuleIPs.size() - 1;
return umIdx;
}
//------------------------------------------------------------------------------
@ -206,34 +208,34 @@ UmSocketSelector::findOrAddUm( const std::string& moduleName )
//------------------------------------------------------------------------------
bool
UmSocketSelector::addConnection(
const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock )
const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock )
{
bool bConnAdded = false;
bool bConnAdded = false;
sockaddr sa = ios->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
sockaddr sa = ios->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
// Add this socket/port connection to the UM connection list it belongs to.
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
bConnAdded = fUmModuleIPs[umIdx]->addSocketConn( ios, writeLock );
}
// Add this socket/port connection to the UM connection list it belongs to.
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
bConnAdded = fUmModuleIPs[umIdx]->addSocketConn( ios, writeLock );
}
if (!bConnAdded)
{
if (!bConnAdded)
{
#ifdef SEL_CONN_DEBUG
std::ostringstream oss;
oss << "No UM/IP match found to add connection " << ios->toString() <<
std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "No UM/IP match found to add connection " << ios->toString() <<
std::endl;
std::cout << oss.str();
#endif
}
}
return bConnAdded;
return bConnAdded;
}
//------------------------------------------------------------------------------
@ -243,16 +245,16 @@ UmSocketSelector::addConnection(
void
UmSocketSelector::delConnection( const IOSocket& ios )
{
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
fUmModuleIPs[umIdx]->delSocketConn( ios );
}
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
fUmModuleIPs[umIdx]->delSocketConn( ios );
}
}
//------------------------------------------------------------------------------
@ -269,36 +271,37 @@ UmSocketSelector::delConnection( const IOSocket& ios )
//------------------------------------------------------------------------------
bool
UmSocketSelector::nextIOSocket(
const IOSocket& ios,
SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock )
const IOSocket& ios,
SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock )
{
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
if (fUmModuleIPs[umIdx]->nextIOSocket( outIos, writeLock ))
{
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
if (fUmModuleIPs[umIdx]->nextIOSocket( outIos, writeLock ))
{
#ifdef SEL_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleIPs[umIdx]->moduleName() <<
"; in: " << ios.toString() <<
"; selected out: " << outIos->toString() << std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "UM " << fUmModuleIPs[umIdx]->moduleName() <<
"; in: " << ios.toString() <<
"; selected out: " << outIos->toString() << std::endl;
std::cout << oss.str();
#endif
return true;
}
}
return true;
}
}
//..This should not happen. Application is asking for next socket/port for
// a connection not in our UM module list.
return false;
//..This should not happen. Application is asking for next socket/port for
// a connection not in our UM module list.
return false;
}
//------------------------------------------------------------------------------
@ -307,25 +310,26 @@ UmSocketSelector::nextIOSocket(
const std::string
UmSocketSelector::toString() const
{
std::ostringstream oss;
std::ostringstream oss;
oss << "IP Address to UM index map:" << std::endl;
for (IpAddressUmMap_t::const_iterator mapIter = fIpAddressUmMap.begin();
(mapIter != fIpAddressUmMap.end());
++mapIter)
{
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " <<
UmIPSocketConns::nwToString(mapIter->first, ipString) <<
" maps to UM: " << mapIter->second << std::endl;
}
oss << "IP Address to UM index map:" << std::endl;
for (unsigned int i=0; i<fUmModuleIPs.size(); ++i)
{
oss << std::endl << fUmModuleIPs[i]->toString();
}
for (IpAddressUmMap_t::const_iterator mapIter = fIpAddressUmMap.begin();
(mapIter != fIpAddressUmMap.end());
++mapIter)
{
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " <<
UmIPSocketConns::nwToString(mapIter->first, ipString) <<
" maps to UM: " << mapIter->second << std::endl;
}
return oss.str();
for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i)
{
oss << std::endl << fUmModuleIPs[i]->toString();
}
return oss.str();
}
//------------------------------------------------------------------------------
@ -337,17 +341,17 @@ UmSocketSelector::toString() const
void
UmModuleIPs::addIP( in_addr_t ip )
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
boost::mutex::scoped_lock lock( fUmModuleMutex );
#ifdef MOD_CONN_DEBUG
std::ostringstream oss;
char ipString[INET_ADDRSTRLEN];
oss << " UM " << fUmModuleName << "; adding IP: " <<
UmIPSocketConns::nwToString(ip,ipString) << std::endl;
std::cout << oss.str();
std::ostringstream oss;
char ipString[INET_ADDRSTRLEN];
oss << " UM " << fUmModuleName << "; adding IP: " <<
UmIPSocketConns::nwToString(ip, ipString) << std::endl;
std::cout << oss.str();
#endif
fUmIPSocketConns.push_back( SP_UM_IPCONNS(new UmIPSocketConns(ip)) );
fUmIPSocketConns.push_back( SP_UM_IPCONNS(new UmIPSocketConns(ip)) );
}
//------------------------------------------------------------------------------
@ -359,38 +363,41 @@ UmModuleIPs::addIP( in_addr_t ip )
//------------------------------------------------------------------------------
bool
UmModuleIPs::addSocketConn(
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
{
bool bConnAdded = false;
bool bConnAdded = false;
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i=0; i<fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
#ifdef MOD_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; adding connection " <<
ioSock->toString() << std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; adding connection " <<
ioSock->toString() << std::endl;
std::cout << oss.str();
#endif
fUmIPSocketConns[i]->addSocketConn ( ioSock, writeLock );
bConnAdded = true;
fUmIPSocketConns[i]->addSocketConn ( ioSock, writeLock );
bConnAdded = true;
//..Initialize fNextUmIPSocketIdx if this is the first socket/port
// connection for this UM.
if ( fNextUmIPSocketIdx == NEXT_IP_SOCKET_UNASSIGNED)
fNextUmIPSocketIdx = i;
break;
}
}
//..Initialize fNextUmIPSocketIdx if this is the first socket/port
// connection for this UM.
if ( fNextUmIPSocketIdx == NEXT_IP_SOCKET_UNASSIGNED)
fNextUmIPSocketIdx = i;
return bConnAdded;
break;
}
}
return bConnAdded;
}
//------------------------------------------------------------------------------
@ -400,32 +407,35 @@ UmModuleIPs::addSocketConn(
void
UmModuleIPs::delSocketConn( const IOSocket& ioSock )
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i=0; i<fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
#ifdef MOD_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; deleting connection "<<
ioSock.toString() << std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; deleting connection " <<
ioSock.toString() << std::endl;
std::cout << oss.str();
#endif
fUmIPSocketConns[i]->delSocketConn ( ioSock );
fUmIPSocketConns[i]->delSocketConn ( ioSock );
//..If we just deleted the last connection for this IP, then ad-
// vance fNextUmIPSocketIdx to an IP address having connections.
if (fUmIPSocketConns[i]->count() == 0)
{
advanceToNextIP();
}
break;
}
}
//..If we just deleted the last connection for this IP, then ad-
// vance fNextUmIPSocketIdx to an IP address having connections.
if (fUmIPSocketConns[i]->count() == 0)
{
advanceToNextIP();
}
break;
}
}
}
//------------------------------------------------------------------------------
@ -439,19 +449,20 @@ UmModuleIPs::delSocketConn( const IOSocket& ioSock )
bool
UmModuleIPs::nextIOSocket( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
{
bool found = false;
bool found = false;
boost::mutex::scoped_lock lock( fUmModuleMutex );
if ((fUmIPSocketConns.size() > 0) &&
(fNextUmIPSocketIdx != NEXT_IP_SOCKET_UNASSIGNED))
{
assert (fNextUmIPSocketIdx < static_cast<int>(fUmIPSocketConns.size()));
fUmIPSocketConns[fNextUmIPSocketIdx]->nextIOSocket( outIos, writeLock );
advanceToNextIP();
found = true;
}
boost::mutex::scoped_lock lock( fUmModuleMutex );
return found;
if ((fUmIPSocketConns.size() > 0) &&
(fNextUmIPSocketIdx != NEXT_IP_SOCKET_UNASSIGNED))
{
assert (fNextUmIPSocketIdx < static_cast<int>(fUmIPSocketConns.size()));
fUmIPSocketConns[fNextUmIPSocketIdx]->nextIOSocket( outIos, writeLock );
advanceToNextIP();
found = true;
}
return found;
}
//------------------------------------------------------------------------------
@ -462,40 +473,40 @@ UmModuleIPs::nextIOSocket( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
void
UmModuleIPs::advanceToNextIP()
{
if ( fUmIPSocketConns.size() > 1 )
{
//..Search to end of IP list for an IP having 1 or more connections
for (int i=fNextUmIPSocketIdx+1;
i<static_cast<int>(fUmIPSocketConns.size());
++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
if ( fUmIPSocketConns.size() > 1 )
{
//..Search to end of IP list for an IP having 1 or more connections
for (int i = fNextUmIPSocketIdx + 1;
i < static_cast<int>(fUmIPSocketConns.size());
++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
//..Wrap back around to the start of the list, to continue the search
for (int i=0; i<fNextUmIPSocketIdx; ++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
//..Wrap back around to the start of the list, to continue the search
for (int i = 0; i < fNextUmIPSocketIdx; ++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
else // special logic to handle 0 fUmIPSocketConns, or a single empty one
{
if ((fUmIPSocketConns.size() == 0) ||
(fUmIPSocketConns[0]->count() == 0))
{
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
}
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
else // special logic to handle 0 fUmIPSocketConns, or a single empty one
{
if ((fUmIPSocketConns.size() == 0) ||
(fUmIPSocketConns[0]->count() == 0))
{
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
}
}
//------------------------------------------------------------------------------
@ -504,17 +515,18 @@ UmModuleIPs::advanceToNextIP()
const std::string
UmModuleIPs::toString()
{
std::ostringstream oss;
std::ostringstream oss;
boost::mutex::scoped_lock lock( fUmModuleMutex );
oss << "UM module name: " << fUmModuleName <<
"; nextUmIPIdx: " << fNextUmIPSocketIdx << std::endl;
for (unsigned int i=0; i<fUmIPSocketConns.size(); ++i)
{
oss << fUmIPSocketConns[i]->toString();
}
boost::mutex::scoped_lock lock( fUmModuleMutex );
oss << "UM module name: " << fUmModuleName <<
"; nextUmIPIdx: " << fNextUmIPSocketIdx << std::endl;
return oss.str();
for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i)
{
oss << fUmIPSocketConns[i]->toString();
}
return oss.str();
}
//------------------------------------------------------------------------------
@ -527,16 +539,18 @@ UmModuleIPs::toString()
//------------------------------------------------------------------------------
void
UmIPSocketConns::addSocketConn(
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
{
UmIOSocketData sockData = {
SP_UM_IOSOCK(ioSock), SP_UM_MUTEX(writeLock) };
fIOSockets.push_back( sockData );
UmIOSocketData sockData =
{
SP_UM_IOSOCK(ioSock), SP_UM_MUTEX(writeLock)
};
fIOSockets.push_back( sockData );
//..Initialize fNextIOSocketIdx when we add first connection
if (fIOSockets.size() == 1)
fNextIOSocketIdx = 0;
//..Initialize fNextIOSocketIdx when we add first connection
if (fIOSockets.size() == 1)
fNextIOSocketIdx = 0;
}
//------------------------------------------------------------------------------
@ -553,36 +567,37 @@ UmIPSocketConns::addSocketConn(
void
UmIPSocketConns::delSocketConn( const IOSocket& ioSock )
{
for (unsigned int i=0; i<fIOSockets.size(); ++i)
{
sockaddr sa1 = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp1 = reinterpret_cast<const sockaddr_in*>(&sa1);
sockaddr sa2 = ioSock.sa();
const sockaddr_in* sinp2 = reinterpret_cast<const sockaddr_in*>(&sa2);
if (sinp1->sin_port == sinp2->sin_port)
{
fIOSockets.erase ( fIOSockets.begin() + i );
for (unsigned int i = 0; i < fIOSockets.size(); ++i)
{
sockaddr sa1 = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp1 = reinterpret_cast<const sockaddr_in*>(&sa1);
sockaddr sa2 = ioSock.sa();
const sockaddr_in* sinp2 = reinterpret_cast<const sockaddr_in*>(&sa2);
//..Adjust fNextIOSocketIdx
// 1a. decrement if fNextIOSocketIdx is after deleted connection
// 1b. reset to start of vector if we deleted the last connection
// 2. reset fNextIOSocketIdx to -1 if we have no more connections
if (fIOSockets.size() > 0)
{
if (fNextIOSocketIdx > static_cast<int>(i))
fNextIOSocketIdx--;
if ( fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()) )
fNextIOSocketIdx = 0;
}
else
{
fNextIOSocketIdx = NEXT_IOSOCKET_UNASSIGNED;
}
if (sinp1->sin_port == sinp2->sin_port)
{
fIOSockets.erase ( fIOSockets.begin() + i );
break;
}
}
//..Adjust fNextIOSocketIdx
// 1a. decrement if fNextIOSocketIdx is after deleted connection
// 1b. reset to start of vector if we deleted the last connection
// 2. reset fNextIOSocketIdx to -1 if we have no more connections
if (fIOSockets.size() > 0)
{
if (fNextIOSocketIdx > static_cast<int>(i))
fNextIOSocketIdx--;
if ( fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()) )
fNextIOSocketIdx = 0;
}
else
{
fNextIOSocketIdx = NEXT_IOSOCKET_UNASSIGNED;
}
break;
}
}
}
//------------------------------------------------------------------------------
@ -594,18 +609,19 @@ UmIPSocketConns::delSocketConn( const IOSocket& ioSock )
void
UmIPSocketConns::nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
{
assert (fIOSockets.size() > 0);
assert (fNextIOSocketIdx != NEXT_IOSOCKET_UNASSIGNED);
assert (fNextIOSocketIdx < static_cast<int>(fIOSockets.size()));
assert (fIOSockets.size() > 0);
assert (fNextIOSocketIdx != NEXT_IOSOCKET_UNASSIGNED);
assert (fNextIOSocketIdx < static_cast<int>(fIOSockets.size()));
outIos = fIOSockets[fNextIOSocketIdx].fSock;
writeLock = fIOSockets[fNextIOSocketIdx].fMutex;
outIos = fIOSockets[fNextIOSocketIdx].fSock;
writeLock = fIOSockets[fNextIOSocketIdx].fMutex;
//..Update "next" index, being sure to wrap around to the start
// whenever we reach the end of the vector.
fNextIOSocketIdx++;
if (fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()))
fNextIOSocketIdx = 0;
//..Update "next" index, being sure to wrap around to the start
// whenever we reach the end of the vector.
fNextIOSocketIdx++;
if (fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()))
fNextIOSocketIdx = 0;
}
//------------------------------------------------------------------------------
@ -615,13 +631,14 @@ UmIPSocketConns::nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
/* static */ char*
UmIPSocketConns::nwToString( in_addr_t addr, char* ipString )
{
in_addr addrStruct = { addr };
in_addr addrStruct = { addr };
#ifndef _MSC_VER
if (!inet_ntop(AF_INET, &addrStruct, ipString, INET_ADDRSTRLEN))
#endif
strcpy(ipString,"unknown");
return ipString;
if (!inet_ntop(AF_INET, &addrStruct, ipString, INET_ADDRSTRLEN))
#endif
strcpy(ipString, "unknown");
return ipString;
}
//------------------------------------------------------------------------------
@ -630,21 +647,21 @@ UmIPSocketConns::nwToString( in_addr_t addr, char* ipString )
const std::string
UmIPSocketConns::toString() const
{
std::ostringstream oss;
std::ostringstream oss;
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " << UmIPSocketConns::nwToString(fIpAddress,ipString)<<
"; nextIOSocketIdx: " << fNextIOSocketIdx << std::endl;
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " << UmIPSocketConns::nwToString(fIpAddress, ipString) <<
"; nextIOSocketIdx: " << fNextIOSocketIdx << std::endl;
for (unsigned int i=0; i<fIOSockets.size(); ++i)
{
sockaddr sa = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
oss << " port: " << ntohs(sinp->sin_port) <<
std::endl;
}
for (unsigned int i = 0; i < fIOSockets.size(); ++i)
{
sockaddr sa = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
oss << " port: " << ntohs(sinp->sin_port) <<
std::endl;
}
return oss.str();
return oss.str();
}
} // end of primitiveprocessor namespace

376
primitives/primproc/umsocketselector.h
View File

@ -81,73 +81,73 @@ typedef boost::shared_ptr<boost::mutex> SP_UM_MUTEX;
//------------------------------------------------------------------------------
class UmSocketSelector
{
public:
typedef std::map<in_addr_t,unsigned int> IpAddressUmMap_t;
public:
typedef std::map<in_addr_t, unsigned int> IpAddressUmMap_t;
/** @brief Singleton accessor to UmSocketSelector instance.
*
* This method should be called once from the main thread to perform
* initialization from a single threaded environment.
*/
static UmSocketSelector* instance();
/** @brief Singleton accessor to UmSocketSelector instance.
*
* This method should be called once from the main thread to perform
* initialization from a single threaded environment.
*/
static UmSocketSelector* instance();
/** @brief UmSocketSelector destructor
*
*/
~UmSocketSelector()
{ };
/** @brief UmSocketSelector destructor
*
*/
~UmSocketSelector()
{ };
/** @brief Accessor to total number of UM IP's in Columnstore.xml.
*
* @return Number of UM IP addresses read from Columnstore.xml.
*/
uint32_t ipAddressCount() const;
/** @brief Accessor to total number of UM IP's in Columnstore.xml.
*
* @return Number of UM IP addresses read from Columnstore.xml.
*/
uint32_t ipAddressCount() const;
/** @brief Add a socket/port connection to the connection list.
*
* @param ios (in) socket/port connection to be added.
* @param writeLock (in) mutex to use when writing to ios.
* @return boolean indicating if socket/port connection was added.
*/
bool addConnection( const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock );
/** @brief Add a socket/port connection to the connection list.
*
* @param ios (in) socket/port connection to be added.
* @param writeLock (in) mutex to use when writing to ios.
* @return boolean indicating if socket/port connection was added.
*/
bool addConnection( const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock );
/** @brief Delete a socket/port connection from the connection list.
*
* @param ios (in) socket/port connection to be removed.
*/
void delConnection( const IOSocket& ios );
/** @brief Delete a socket/port connection from the connection list.
*
* @param ios (in) socket/port connection to be removed.
*/
void delConnection( const IOSocket& ios );
/** @brief Get the next output IOSocket to use for the specified ios.
*
* @param ios (in) socket/port connection for the incoming message.
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return boolean indicating if operation was successful.
*/
bool nextIOSocket( const IOSocket& ios, SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock );
/** @brief Get the next output IOSocket to use for the specified ios.
*
* @param ios (in) socket/port connection for the incoming message.
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return boolean indicating if operation was successful.
*/
bool nextIOSocket( const IOSocket& ios, SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock );
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
private:
//...Disable default copy constructor and assignment operator by
// declaring but not defining
UmSocketSelector (const UmSocketSelector& rhs);
UmSocketSelector& operator=(const UmSocketSelector& rhs);
private:
//...Disable default copy constructor and assignment operator by
// declaring but not defining
UmSocketSelector (const UmSocketSelector& rhs);
UmSocketSelector& operator=(const UmSocketSelector& rhs);
UmSocketSelector();
void loadUMModuleInfo();
unsigned int findOrAddUm( const std::string& moduleName );
UmSocketSelector();
void loadUMModuleInfo();
unsigned int findOrAddUm( const std::string& moduleName );
static UmSocketSelector* fpUmSocketSelector;
std::vector<SP_UM_MODIPS> fUmModuleIPs; // UM's and their sockets
static UmSocketSelector* fpUmSocketSelector;
std::vector<SP_UM_MODIPS> fUmModuleIPs; // UM's and their sockets
// std::map that maps an IP address to an index into fUmModuleIPs
IpAddressUmMap_t fIpAddressUmMap;
// std::map that maps an IP address to an index into fUmModuleIPs
IpAddressUmMap_t fIpAddressUmMap;
};
//------------------------------------------------------------------------------
@ -164,80 +164,84 @@ class UmSocketSelector
//------------------------------------------------------------------------------
class UmModuleIPs
{
public:
/** @brief UmModuleIPs constructor.
*
* @param moduleName (in) UM module name for this UmModuleIPs object.
*/
explicit UmModuleIPs ( const std::string& moduleName ) :
fUmModuleName(moduleName),
fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED)
{ }
public:
/** @brief UmModuleIPs constructor.
*
* @param moduleName (in) UM module name for this UmModuleIPs object.
*/
explicit UmModuleIPs ( const std::string& moduleName ) :
fUmModuleName(moduleName),
fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED)
{ }
/** @brief UmModuleIPs destructor.
*
*/
~UmModuleIPs ( )
{ };
/** @brief UmModuleIPs destructor.
*
*/
~UmModuleIPs ( )
{ };
/** @brief Accessor to number of IP's from Columnstore.xml for this UM.
*
* @return Number of IP addresses read from Columnstore.xml for this UM.
*/
uint32_t ipAddressCount() const
{ return fUmIPSocketConns.size(); }
/** @brief Accessor to number of IP's from Columnstore.xml for this UM.
*
* @return Number of IP addresses read from Columnstore.xml for this UM.
*/
uint32_t ipAddressCount() const
{
return fUmIPSocketConns.size();
}
/** @brief Accessor to the module name for this UmModuleIPs object.
*
* @return UM module name.
*/
const std::string& moduleName() const
{ return fUmModuleName; }
/** @brief Accessor to the module name for this UmModuleIPs object.
*
* @return UM module name.
*/
const std::string& moduleName() const
{
return fUmModuleName;
}
/** @brief Add an IP address to this UM module.
*
* @param ip (in) IP address to be added (in network byte order)
*/
void addIP ( in_addr_t ip );
/** @brief Add an IP address to this UM module.
*
* @param ip (in) IP address to be added (in network byte order)
*/
void addIP ( in_addr_t ip );
/** @brief Add specified socket/port to the connection list for this UM.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
* @return boolean indicating if socket/port connection was added.
*/
bool addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Add specified socket/port to the connection list for this UM.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
* @return boolean indicating if socket/port connection was added.
*/
bool addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Get the "next" available socket/port for this UM module.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return bool flag indicating whether a socket/port was available.
*/
bool nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief Get the "next" available socket/port for this UM module.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return bool flag indicating whether a socket/port was available.
*/
bool nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString();
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString();
private:
void advanceToNextIP();
private:
void advanceToNextIP();
static const int32_t NEXT_IP_SOCKET_UNASSIGNED;
std::string fUmModuleName; // UM module name
int32_t fNextUmIPSocketIdx; //index to "next" IP address
boost::mutex fUmModuleMutex;
static const int32_t NEXT_IP_SOCKET_UNASSIGNED;
std::string fUmModuleName; // UM module name
int32_t fNextUmIPSocketIdx; //index to "next" IP address
boost::mutex fUmModuleMutex;
// collection of IP addresses and their corresponding socket/port conns
std::vector<SP_UM_IPCONNS> fUmIPSocketConns;
// collection of IP addresses and their corresponding socket/port conns
std::vector<SP_UM_IPCONNS> fUmIPSocketConns;
};
//------------------------------------------------------------------------------
@ -255,81 +259,85 @@ class UmModuleIPs
//------------------------------------------------------------------------------
class UmIPSocketConns
{
public:
struct UmIOSocketData
{
SP_UM_IOSOCK fSock; // an output IOSocket
SP_UM_MUTEX fMutex; // mutex to be use when writing to fSock
};
public:
struct UmIOSocketData
{
SP_UM_IOSOCK fSock; // an output IOSocket
SP_UM_MUTEX fMutex; // mutex to be use when writing to fSock
};
/** @brief UmIPSocketConns constructor.
*
* @param ip (in) IP address for this UmIPSocketConns object.
*/
explicit UmIPSocketConns( in_addr_t ip ) :
fIpAddress(ip),
fNextIOSocketIdx(NEXT_IOSOCKET_UNASSIGNED)
{ }
/** @brief UmIPSocketConns constructor.
*
* @param ip (in) IP address for this UmIPSocketConns object.
*/
explicit UmIPSocketConns( in_addr_t ip ) :
fIpAddress(ip),
fNextIOSocketIdx(NEXT_IOSOCKET_UNASSIGNED)
{ }
/** @brief UmIPSocketConns destructor.
*
*/
~UmIPSocketConns( )
{ }
/** @brief UmIPSocketConns destructor.
*
*/
~UmIPSocketConns( )
{ }
/** @brief Accessor to the IP address for this UmIPSocketConns object.
*
* @return IP address (in network byte order).
*/
in_addr_t ipAddress ( )
{ return fIpAddress; }
/** @brief Accessor to the IP address for this UmIPSocketConns object.
*
* @return IP address (in network byte order).
*/
in_addr_t ipAddress ( )
{
return fIpAddress;
}
/** @brief Accessor to socket/port connection count for this IP address.
*
* @return socket/port connection count.
*/
uint32_t count( )
{ return fIOSockets.size(); }
/** @brief Accessor to socket/port connection count for this IP address.
*
* @return socket/port connection count.
*/
uint32_t count( )
{
return fIOSockets.size();
}
/** @brief Add specified socket/port to the connection list.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
*/
void addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Add specified socket/port to the connection list.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
*/
void addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Get the "next" available socket/port for this IP address.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
*/
void nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief Get the "next" available socket/port for this IP address.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
*/
void nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief Convert network byte ordered IP address to string
*
* @param ipString (out) IP address string;
* ipString must be an array of size INET_ADDRSTRLEN.
* @return IP address string (same as ipString)
*/
static char* nwToString( in_addr_t addr, char* ipString );
/** @brief Convert network byte ordered IP address to string
*
* @param ipString (out) IP address string;
* ipString must be an array of size INET_ADDRSTRLEN.
* @return IP address string (same as ipString)
*/
static char* nwToString( in_addr_t addr, char* ipString );
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
private:
static const int32_t NEXT_IOSOCKET_UNASSIGNED;
in_addr_t fIpAddress; // IP address in network byte order
int32_t fNextIOSocketIdx;//index to "next" socket/port
std::vector<UmIOSocketData> fIOSockets;//socket/port list for fIpAddress
private:
static const int32_t NEXT_IOSOCKET_UNASSIGNED;
in_addr_t fIpAddress; // IP address in network byte order
int32_t fNextIOSocketIdx;//index to "next" socket/port
std::vector<UmIOSocketData> fIOSockets;//socket/port list for fIpAddress
};
}