database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

631
primitives/blockcache/bcTest.cpp
View File

@ -40,346 +40,405 @@ using namespace BRM;
using namespace dbbc;
using namespace std;
Stats* gPMStatsPtr=NULL;
bool gPMProfOn=false;
uint32_t gSession=0;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
int fLoops=1;
int thr_cnt=1;
uint64_t bfoundTot=0;
uint64_t bnfoundTot=0;
uint64_t rfoundTot=0;
uint64_t rnfoundTot=0;
uint64_t rangeOpCountTot=0;
uint64_t blockOpCountTot=0;
uint64_t noOpCountTot=0;
int fLoops = 1;
int thr_cnt = 1;
uint64_t bfoundTot = 0;
uint64_t bnfoundTot = 0;
uint64_t rfoundTot = 0;
uint64_t rnfoundTot = 0;
uint64_t rangeOpCountTot = 0;
uint64_t blockOpCountTot = 0;
uint64_t noOpCountTot = 0;
struct thr_wait_struct {
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
struct thr_wait_struct
{
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
};
typedef thr_wait_struct thr_wait_t;
uint32_t cacheSize=10240;
uint32_t cacheSize = 10240;
// BlockRequestProcessor BRP(cacheSize, 4, 16);
BlockRequestProcessor* BRP;
BRM::VER_t ver=0xFFFF;
u_int64_t totBlocks=0;
BRM::VER_t ver = 0xFFFF;
u_int64_t totBlocks = 0;
void* thr_client(void* clientArgs) {
blockCacheClient bc(*BRP);
uint64_t bfound=0;
uint64_t bnfound=0;
uint64_t rfound=0;
uint64_t rnfound=0;
uint64_t rangeOpCount=0;
uint64_t blockOpCount=0;
uint64_t noOpCount=0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate=0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
void* thr_client(void* clientArgs)
{
blockCacheClient bc(*BRP);
uint64_t bfound = 0;
uint64_t bnfound = 0;
uint64_t rfound = 0;
uint64_t rnfound = 0;
uint64_t rangeOpCount = 0;
uint64_t blockOpCount = 0;
uint64_t noOpCount = 0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate = 0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
gettimeofday(&tv, NULL);
uint8_t fbData[8192]={0};
FileBuffer* fbPtr=NULL;
int ret=0;
int idx=0;
uint32_t jdx=0;
uint32_t l=0;
uint32_t m=0;
uint32_t start=0, max=0, size=0;
LBIDRange_v& r=range_thr[0];
for(idx=0; idx<fLoops; idx++) {
for (jdx=0; jdx<range_thr.size(); jdx++) {
ret=0;
r = range_thr[jdx];
for(l=0; l<r.size(); l++) {
start = r[l].start;
size = r[l].size;
max = r[l].start+r[l].size;
//cout << "readThr " <<start << " " << max << endl;
for(m=start;m<max; m++) {
//const FileBuffer* ptr=bc.read(m, ver, fbPtr);
bool ptr=bc.read(m, ver, fbData);
//cout << "bc ptr " << ptr << endl;
if (ptr) {
bfound++;
//memcpy(fbData, ptr->getData(), ptr->datLen());
} else {
bnfound++;
//cout << "bc fail:" << m << " " <<start << " " << size << " " << max << endl;
}
ptr=false;
}
}
}
}
gettimeofday(&tv, NULL);
uint8_t fbData[8192] = {0};
FileBuffer* fbPtr = NULL;
int ret = 0;
int idx = 0;
uint32_t jdx = 0;
uint32_t l = 0;
uint32_t m = 0;
uint32_t start = 0, max = 0, size = 0;
LBIDRange_v& r = range_thr[0];
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
for (idx = 0; idx < fLoops; idx++)
{
for (jdx = 0; jdx < range_thr.size(); jdx++)
{
ret = 0;
r = range_thr[jdx];
if (elTime>0) {
avgTot=(bfound+rfound)/elTime;
rangeAvg=(rfound)/elTime;
blkAvg=(bfound)/elTime;
} else {
avgTot=bfound+rfound;
rangeAvg=rfound;
blkAvg=bfound;
}
for (l = 0; l < r.size(); l++)
{
start = r[l].start;
size = r[l].size;
max = r[l].start + r[l].size;
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
//cout << "readThr " <<start << " " << max << endl;
for (m = start; m < max; m++)
{
//const FileBuffer* ptr=bc.read(m, ver, fbPtr);
bool ptr = bc.read(m, ver, fbData);
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot+=bfound;
bnfoundTot+=bnfound;
rfoundTot+=rfound;
rnfoundTot+=rnfound;
rangeOpCountTot+=rangeOpCount;
blockOpCountTot+=blockOpCount;
noOpCountTot+=noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
//cout << "bc ptr " << ptr << endl;
if (ptr)
{
bfound++;
//memcpy(fbData, ptr->getData(), ptr->datLen());
}
else
{
bnfound++;
//cout << "bc fail:" << m << " " <<start << " " << size << " " << max << endl;
}
ptr = false;
}
}
}
}
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfound + rfound) / elTime;
rangeAvg = (rfound) / elTime;
blkAvg = (bfound) / elTime;
}
else
{
avgTot = bfound + rfound;
rangeAvg = rfound;
blkAvg = bfound;
}
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot += bfound;
bnfoundTot += bnfound;
rfoundTot += rfound;
rnfoundTot += rnfound;
rangeOpCountTot += rangeOpCount;
blockOpCountTot += blockOpCount;
noOpCountTot += noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
return NULL;
return NULL;
} // end thr_client
void LoadRange(const LBIDRange_v& v, uint32_t& loadCount)
{
blockCacheClient bc(*BRP);
blockCacheClient bc(*BRP);
uint32_t rCount=0;
for (uint32_t i =0; i<v.size() ; i++)
{
InlineLBIDRange r={v[i].start, v[i].size};
if (r.size+loadCount>cacheSize)
r.size=(r.size+loadCount)-cacheSize;
if (r.size<=1024) {
//cout << "check() " << r.start << " " << r.size << endl;
bc.check(r, ver, rCount );
loadCount+=rCount;
}
rCount=0;
uint32_t rCount = 0;
}
for (uint32_t i = 0; i < v.size() ; i++)
{
InlineLBIDRange r = {v[i].start, v[i].size};
if (r.size + loadCount > cacheSize)
r.size = (r.size + loadCount) - cacheSize;
if (r.size <= 1024)
{
//cout << "check() " << r.start << " " << r.size << endl;
bc.check(r, ver, rCount );
loadCount += rCount;
}
rCount = 0;
}
}
void ReadRange(const LBIDRange_v& v)
{
blockCacheClient bc(*BRP);
int found=0;
int notfound=0;
int ret=0;
for(uint32_t i=0; i<v.size(); i++)
{
const InlineLBIDRange r={v[i].start, v[i].size};
FileBuffer fb(-1, -1);
//cout << "read() " << r.start << " " << r.size << endl;
for(int j=r.start; j<r.start+r.size; j++)
{
if (r.size > 1024)
continue;
ret=bc.read(j, ver, fb);
if (ret)
found++;
else {
notfound++;
}
ret=0;
}
totBlocks+=found;
totBlocks+=notfound;
found=0;
notfound=0;
}
blockCacheClient bc(*BRP);
int found = 0;
int notfound = 0;
int ret = 0;
for (uint32_t i = 0; i < v.size(); i++)
{
const InlineLBIDRange r = {v[i].start, v[i].size};
FileBuffer fb(-1, -1);
//cout << "read() " << r.start << " " << r.size << endl;
for (int j = r.start; j < r.start + r.size; j++)
{
if (r.size > 1024)
continue;
ret = bc.read(j, ver, fb);
if (ret)
found++;
else
{
notfound++;
}
ret = 0;
}
totBlocks += found;
totBlocks += notfound;
found = 0;
notfound = 0;
}
}
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(*BRP);
bool b;
bc.check(lbid, ver, false, b);
blockCacheClient bc(*BRP);
bool b;
bc.check(lbid, ver, false, b);
}
void ReadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
static int found=0, notfound=0;
uint8_t** d=NULL; //[8192];
blockCacheClient bc(*BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if (ret)
found++;
else
notfound++;
static int found = 0, notfound = 0;
uint8_t** d = NULL; //[8192];
blockCacheClient bc(*BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if ((found+notfound)%10000==0)
cout << "found " << found << " notfound " << notfound << endl;
if (ret)
found++;
else
notfound++;
if ((found + notfound) % 10000 == 0)
cout << "found " << found << " notfound " << notfound << endl;
}
//
int main(int argc, char *argv[]) {
int main(int argc, char* argv[])
{
if (argc>=2) thr_cnt=atoi(argv[1]);
if (argc>=3) fLoops=atoi(argv[2]);
if (argc>=4) cacheSize=atoi(argv[3]);
if (thr_cnt<=0) thr_cnt=1;
if (thr_cnt>1024) thr_cnt=1024;
if (fLoops<=0) fLoops=1;
if (argc >= 2) thr_cnt = atoi(argv[1]);
BlockRequestProcessor brp(cacheSize, 4, 16);
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
BRP=&brp;
if (argc >= 3) fLoops = atoi(argv[2]);
cout << "Reading Ranges " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid=3000;
uint32_t totalExt=0;
do {
int ret = dbrm.lookup(oid, r);
if (ret==0 && r.size() > 0) {
lowlbid = (r[0].start/extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start=lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx=0; idx<r.size(); idx++)
totalExt+=r[idx].size;
ranges.push_back(r);
cout << ".";
if (ranges.size()%50==0)
cout <<endl;
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
if (argc >= 4) cacheSize = atoi(argv[3]);
cout << endl << ranges.size() << " ranges found" << endl;
if (thr_cnt <= 0) thr_cnt = 1;
gettimeofday(&tv, NULL);
uint32_t blksLoaded=0;
uint32_t rangesLoaded=0;
cout << "Loading Ranges " << endl;
for (uint32_t i =0; i<ranges.size() && blksLoaded < (cacheSize-1024); i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
cout << ".";
if (i%50==0&&i)
cout <<endl;
}
cout << endl;
if (thr_cnt > 1024) thr_cnt = 1024;
gettimeofday(&tv2, NULL);
cout << endl << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec <<endl;
if (fLoops <= 0) fLoops = 1;
while (ranges.size()>=rangesLoaded) ranges.pop_back();
BlockRequestProcessor brp(cacheSize, 4, 16);
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
BRP = &brp;
ranges.pop_back();
cout << "Reading Ranges " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid = 3000;
uint32_t totalExt = 0;
do
{
int ret = dbrm.lookup(oid, r);
if (ret == 0 && r.size() > 0)
{
lowlbid = (r[0].start / extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start = lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx = 0; idx < r.size(); idx++)
totalExt += r[idx].size;
ranges.push_back(r);
cout << ".";
if (ranges.size() % 50 == 0)
cout << endl;
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
cout << endl << ranges.size() << " ranges found" << endl;
gettimeofday(&tv, NULL);
uint32_t blksLoaded = 0;
uint32_t rangesLoaded = 0;
cout << "Loading Ranges " << endl;
for (uint32_t i = 0; i < ranges.size() && blksLoaded < (cacheSize - 1024); i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
cout << ".";
if (i % 50 == 0 && i)
cout << endl;
}
cout << endl;
gettimeofday(&tv2, NULL);
cout << endl << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec << endl;
while (ranges.size() >= rangesLoaded) ranges.pop_back();
ranges.pop_back();
#ifdef BLAH
for (uint32_t i =0; i<ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i =0; i<ranges.size(); i++)
{
LBIDRange_v rv=ranges[i];
for(uint32_t j=0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for(uint32_t k=l.start; k<l.start+l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
for (uint32_t i = 0; i < ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i = 0; i < ranges.size(); i++)
{
LBIDRange_v rv = ranges[i];
for (uint32_t j = 0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for (uint32_t k = l.start; k < l.start + l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
#endif
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait={0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
//start threads running
cout << "Starting reader threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt*(sizeof(pthread_t)));
for(int i=0; i<thr_cnt; i++) {
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
while (thr_wait.predicate>0) {
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
//start threads running
cout << "Starting reader threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt * (sizeof(pthread_t)));
// join threads back to main
for(int i=0; i<thr_cnt; i++) {
pthread_join(thr_id[i], NULL);
}
for (int i = 0; i < thr_cnt; i++)
{
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
//uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
if (elTime>0) {
avgTot=(bfoundTot+rfoundTot)/elTime;
rangeAvg=(rfoundTot)/elTime;
blkAvg=(bfoundTot)/elTime;
} else {
avgTot=bfoundTot+rfoundTot;
rangeAvg=rfoundTot;
blkAvg=bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot/thr_cnt << endl << endl;
while (thr_wait.predicate > 0)
{
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
// join threads back to main
for (int i = 0; i < thr_cnt; i++)
{
pthread_join(thr_id[i], NULL);
}
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
//uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfoundTot + rfoundTot) / elTime;
rangeAvg = (rfoundTot) / elTime;
blkAvg = (bfoundTot) / elTime;
}
else
{
avgTot = bfoundTot + rfoundTot;
rangeAvg = rfoundTot;
blkAvg = bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl << endl;
return 0;
return 0;
} // end main

5
primitives/blockcache/blockcacheclient.cpp
View File

@ -21,11 +21,12 @@
*
* jrodriguez@calpont.com *
***************************************************************************/
#include <sstream>
#include <string>
#include "blockcacheclient.h"
namespace dbbc {
namespace dbbc
{
}

194
primitives/blockcache/blockcacheclient.h
View File

@ -35,7 +35,7 @@
/**
* @brief API for the Disk Block Buffer Cache
*
*
*
*/
namespace dbbc
@ -45,103 +45,131 @@ class blockCacheClient
{
public:
/**
* @brief ctor requires reference to BlockRequestProcessor object
**/
blockCacheClient(BlockRequestProcessor& brp) : fBCCBrp(&brp) {}
/**
* @brief ctor requires reference to BlockRequestProcessor object
**/
blockCacheClient(BlockRequestProcessor& brp) : fBCCBrp(&brp) {}
/**
* @brief dtor
**/
virtual ~blockCacheClient() {}
/**
* @brief dtor
**/
virtual ~blockCacheClient() {}
/**
* @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache.
**/
inline void check(BRM::LBID_t lbid, const BRM::QueryContext &ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache) {
fBCCBrp->check(lbid, ver, txn, flg, compType, wasBlockInCache); }
/**
* @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache.
**/
inline void check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache)
{
fBCCBrp->check(lbid, ver, txn, flg, compType, wasBlockInCache);
}
/**
* @brief verify all Disk Blocks for the LBID range are loaded into the Cache
**/
inline void check(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType,
uint32_t& rCount) {
fBCCBrp->check(range, ver, txn, compType, rCount); }
/**
* @brief verify all Disk Blocks for the LBID range are loaded into the Cache
**/
inline void check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType,
uint32_t& rCount)
{
fBCCBrp->check(range, ver, txn, compType, rCount);
}
inline FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver, bool flg) {
return fBCCBrp->getBlockPtr(lbid, ver, flg); }
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) {
return fBCCBrp->read(lbid, ver, fb); }
inline FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver, bool flg)
{
return fBCCBrp->getBlockPtr(lbid, ver, flg);
}
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) {
return fBCCBrp->read(lbid, ver, bufferPtr); }
inline const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType,
void* bufferPtr, bool flg, bool &wasCached, bool *wasVersioned = NULL, bool insertIntoCache = true,
bool readFromCache = true) {
return fBCCBrp->getBlock(lbid, ver, txn, compType, bufferPtr, flg, wasCached, wasVersioned, insertIntoCache,
readFromCache); }
inline int getCachedBlocks(const BRM::LBID_t *lbids, const BRM::VER_t *vers, uint8_t **bufferPtrs,
bool *wasCached, uint32_t blockCount)
{ return fBCCBrp->getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount); }
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
return fBCCBrp->read(lbid, ver, fb);
}
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) {
return fBCCBrp->exists(lbid, ver); }
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
return fBCCBrp->read(lbid, ver, bufferPtr);
}
/**
* @brief flush the cache
**/
inline void flushCache() {
fBCCBrp->flushCache(); }
inline const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType,
void* bufferPtr, bool flg, bool& wasCached, bool* wasVersioned = NULL, bool insertIntoCache = true,
bool readFromCache = true)
{
return fBCCBrp->getBlock(lbid, ver, txn, compType, bufferPtr, flg, wasCached, wasVersioned, insertIntoCache,
readFromCache);
}
/**
* @brief flush one LBID@Ver from the cache
**/
inline void flushOne(const BRM::LBID_t& lbid, const BRM::VER_t& ver) {
fBCCBrp->flushOne(lbid, ver); }
inline int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** bufferPtrs,
bool* wasCached, uint32_t blockCount)
{
return fBCCBrp->getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount);
}
/**
* @brief flush specific LBID/version pairs from the cache
**/
inline void flushMany(const LbidAtVer* laVptr, uint32_t cnt) {
fBCCBrp->flushMany(laVptr, cnt); }
/**
* @brief flush all versions of the given lbids from the cache.
**/
inline void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) {
fBCCBrp->flushManyAllversion(laVptr, cnt); }
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
return fBCCBrp->exists(lbid, ver);
}
/**
* @brief Flush all versions of all LBIDs belonging to the given OIDs.
*/
inline void flushOIDs(const uint32_t *oids, uint32_t count) {
fBCCBrp->flushOIDs(oids, count); }
/**
* @brief flush the cache
**/
inline void flushCache()
{
fBCCBrp->flushCache();
}
/**
* @brief Flush all versions of a partition from the given OIDs.
*/
inline void flushPartition(const std::vector<BRM::OID_t> &oids, const std::set<BRM::LogicalPartition> partitions) {
fBCCBrp->flushPartition(oids, partitions); }
/**
* @brief flush one LBID@Ver from the cache
**/
inline void flushOne(const BRM::LBID_t& lbid, const BRM::VER_t& ver)
{
fBCCBrp->flushOne(lbid, ver);
}
/**
* @brief flush specific LBID/version pairs from the cache
**/
inline void flushMany(const LbidAtVer* laVptr, uint32_t cnt)
{
fBCCBrp->flushMany(laVptr, cnt);
}
/**
* @brief flush all versions of the given lbids from the cache.
**/
inline void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt)
{
fBCCBrp->flushManyAllversion(laVptr, cnt);
}
/**
* @brief Flush all versions of all LBIDs belonging to the given OIDs.
*/
inline void flushOIDs(const uint32_t* oids, uint32_t count)
{
fBCCBrp->flushOIDs(oids, count);
}
/**
* @brief Flush all versions of a partition from the given OIDs.
*/
inline void flushPartition(const std::vector<BRM::OID_t>& oids, const std::set<BRM::LogicalPartition> partitions)
{
fBCCBrp->flushPartition(oids, partitions);
}
private:
/**
* @brief pointer to the BlockRequestProcessor object on which the API will operate
**/
BlockRequestProcessor* fBCCBrp;
/**
* @brief pointer to the BlockRequestProcessor object on which the API will operate
**/
BlockRequestProcessor* fBCCBrp;
//do not implement
blockCacheClient(const blockCacheClient& bc);
blockCacheClient& operator=(const blockCacheClient& blk);
//do not implement
blockCacheClient(const blockCacheClient& bc);
blockCacheClient& operator=(const blockCacheClient& blk);
};

349
primitives/blockcache/blockrequestprocessor.cpp
View File

@ -35,210 +35,233 @@ using namespace std;
#include "dbrm.h"
#include "pp_logger.h"
namespace dbbc {
namespace dbbc
{
BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks,
int thrCount,
int blocksPerRead,
uint32_t deleteBlocks,
uint32_t blckSz) :
fbMgr(numBlcks, blckSz, deleteBlocks),
fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead)
int thrCount,
int blocksPerRead,
uint32_t deleteBlocks,
uint32_t blckSz) :
fbMgr(numBlcks, blckSz, deleteBlocks),
fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead)
{
//pthread_mutex_init(&check_mutex, NULL);
config::Config* fConfig=config::Config::makeConfig();
string val = fConfig->getConfig("DBBC", "BRPTracing");
int temp=0;
//pthread_mutex_init(&check_mutex, NULL);
config::Config* fConfig = config::Config::makeConfig();
string val = fConfig->getConfig("DBBC", "BRPTracing");
int temp = 0;
#ifdef _MSC_VER
int tid = GetCurrentThreadId();
int tid = GetCurrentThreadId();
#else
pthread_t tid = pthread_self();
pthread_t tid = pthread_self();
#endif
if (val.length()>0) temp=static_cast<int>(config::Config::fromText(val));
if (val.length() > 0) temp = static_cast<int>(config::Config::fromText(val));
if (temp > 0)
fTrace=true;
else
fTrace=false;
if (temp > 0)
fTrace = true;
else
fTrace = false;
if (fTrace)
{
ostringstream brpLogFileName;
if (fTrace)
{
ostringstream brpLogFileName;
#ifdef _MSC_VER
brpLogFileName << "C:/Calpont/log/trace/brp." << tid;
brpLogFileName << "C:/Calpont/log/trace/brp." << tid;
#else
brpLogFileName << "/var/log/mariadb/columnstore/trace/brp." << tid;
brpLogFileName << "/var/log/mariadb/columnstore/trace/brp." << tid;
#endif
fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate);
}
fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate);
}
}
BlockRequestProcessor::~BlockRequestProcessor()
{
//pthread_mutex_destroy(&check_mutex);
if (fTrace)
fLogFile.close();
//pthread_mutex_destroy(&check_mutex);
if (fTrace)
fLogFile.close();
}
void BlockRequestProcessor::stop() {
fBRPRequestQueue.stop();
fIOMgr.stop();
void BlockRequestProcessor::stop()
{
fBRPRequestQueue.stop();
fIOMgr.stop();
}
int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType, uint32_t& lbidCount) {
uint64_t maxLbid = range.start; // highest existent lbid
uint64_t rangeLen = range.size;
uint64_t idx;
uint64_t adjSz;
struct timespec start_tm;
lbidCount = 0;
int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType, uint32_t& lbidCount)
{
uint64_t maxLbid = range.start; // highest existent lbid
uint64_t rangeLen = range.size;
uint64_t idx;
uint64_t adjSz;
struct timespec start_tm;
lbidCount = 0;
if (fTrace)
clock_gettime(CLOCK_MONOTONIC, &start_tm);
if (fTrace)
clock_gettime(CLOCK_MONOTONIC, &start_tm);
for (idx = 0; fbMgr.exists(maxLbid, ver.currentScn) == true && idx<rangeLen; maxLbid++, idx++)
(void)0;
for (idx = 0; fbMgr.exists(maxLbid, ver.currentScn) == true && idx < rangeLen; maxLbid++, idx++)
(void)0;
if (idx == rangeLen) { // range is already loaded
if (fTrace)
{
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< 0 << " "
<< 0 << " "
<< 0 << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
}
return 0;
}
if (idx == rangeLen) // range is already loaded
{
if (fTrace)
{
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< 0 << " "
<< 0 << " "
<< 0 << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
}
adjSz=rangeLen-idx;
BRM::InlineLBIDRange adjRange;
adjRange.start=maxLbid;
adjRange.size=adjSz;
fileRequest rqstBlk(adjRange, ver, txn, compType);
check(rqstBlk);
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
throw runtime_error(rqstBlk.RequestStatusStr());
lbidCount=rqstBlk.BlocksRead();
return 0;
}
if (fTrace) {
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< adjSz << " "
<< rqstBlk.BlocksRead() << " "
<< rqstBlk.BlocksLoaded() << " "
<< right << fixed << ((double)(start_tm.tv_sec+(1.e-9*start_tm.tv_nsec)))
<< endl;
}
adjSz = rangeLen - idx;
BRM::InlineLBIDRange adjRange;
adjRange.start = maxLbid;
adjRange.size = adjSz;
fileRequest rqstBlk(adjRange, ver, txn, compType);
check(rqstBlk);
return rqstBlk.BlocksLoaded();
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
throw runtime_error(rqstBlk.RequestStatusStr());
lbidCount = rqstBlk.BlocksRead();
if (fTrace)
{
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< adjSz << " "
<< rqstBlk.BlocksRead() << " "
<< rqstBlk.BlocksLoaded() << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
}
return rqstBlk.BlocksLoaded();
} // check
int BlockRequestProcessor::check(fileRequest& rqstBlk) {
rqstBlk.frMutex().lock();
rqstBlk.SetPredicate(fileRequest::SENDING);
sendRequest(rqstBlk); // start file read request
int BlockRequestProcessor::check(fileRequest& rqstBlk)
{
rqstBlk.frMutex().lock();
rqstBlk.SetPredicate(fileRequest::SENDING);
sendRequest(rqstBlk); // start file read request
while(rqstBlk.frPredicate()<fileRequest::COMPLETE)
rqstBlk.frCond().wait(rqstBlk.frMutex());
rqstBlk.frMutex().unlock();
while (rqstBlk.frPredicate() < fileRequest::COMPLETE)
rqstBlk.frCond().wait(rqstBlk.frMutex());
return 0;
rqstBlk.frMutex().unlock();
return 0;
}
// For future use. Not currently used.
int BlockRequestProcessor::check(BRM::LBID_t lbid, const BRM::QueryContext &ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache) {
if (fbMgr.exists(lbid, ver.currentScn)==true) {
wasBlockInCache = true;
return 0;
} else {
wasBlockInCache = false;
fileRequest rqstBlk(lbid, ver, flg, txn, compType);
int ret=check(rqstBlk);
if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) {
throw runtime_error(rqstBlk.RequestStatusStr());
}
return ret;
}
int BlockRequestProcessor::check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache)
{
if (fbMgr.exists(lbid, ver.currentScn) == true)
{
wasBlockInCache = true;
return 0;
}
else
{
wasBlockInCache = false;
fileRequest rqstBlk(lbid, ver, flg, txn, compType);
int ret = check(rqstBlk);
if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
{
throw runtime_error(rqstBlk.RequestStatusStr());
}
return ret;
}
}
const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext &ver, BRM::VER_t txn,
int compType, void* bufferPtr, bool vbFlg, bool &wasCached, bool *versioned, bool insertIntoCache,
bool readFromCache)
const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, BRM::VER_t txn,
int compType, void* bufferPtr, bool vbFlg, bool& wasCached, bool* versioned, bool insertIntoCache,
bool readFromCache)
{
if (readFromCache) {
HashObject_t hashObj(lbid, ver.currentScn, 0);
wasCached = fbMgr.find(hashObj, bufferPtr);
if (wasCached)
return 1;
}
if (readFromCache)
{
HashObject_t hashObj(lbid, ver.currentScn, 0);
wasCached = fbMgr.find(hashObj, bufferPtr);
wasCached = false;
fileRequest rqstBlk(lbid, ver, vbFlg, txn, compType, (uint8_t *) bufferPtr, insertIntoCache);
check(rqstBlk);
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
{
ostringstream os;
os << "BRP::getBlock(): got a BRM lookup error. LBID=" << lbid << " ver=" << ver << " txn="
<< txn << " vbFlg=" << (int) vbFlg;
primitiveprocessor::Logger logger;
logger.logMessage(os.str(), false);
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
{
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
{
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
}
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) {
throw runtime_error(rqstBlk.RequestStatusStr());
}
if (versioned)
*versioned = rqstBlk.versioned();
return 1;
if (wasCached)
return 1;
}
wasCached = false;
fileRequest rqstBlk(lbid, ver, vbFlg, txn, compType, (uint8_t*) bufferPtr, insertIntoCache);
check(rqstBlk);
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
{
ostringstream os;
os << "BRP::getBlock(): got a BRM lookup error. LBID=" << lbid << " ver=" << ver << " txn="
<< txn << " vbFlg=" << (int) vbFlg;
primitiveprocessor::Logger logger;
logger.logMessage(os.str(), false);
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
{
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
{
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
}
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
{
throw runtime_error(rqstBlk.RequestStatusStr());
}
if (versioned)
*versioned = rqstBlk.versioned();
return 1;
}
int BlockRequestProcessor::getCachedBlocks(const BRM::LBID_t *lbids, const BRM::VER_t *vers,
uint8_t **ptrs, bool *wasCached, uint32_t count)
int BlockRequestProcessor::getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers,
uint8_t** ptrs, bool* wasCached, uint32_t count)
{
return fbMgr.bulkFind(lbids, vers, ptrs, wasCached, count);
return fbMgr.bulkFind(lbids, vers, ptrs, wasCached, count);
}

215
primitives/blockcache/blockrequestprocessor.h
View File

@ -38,129 +38,160 @@
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
namespace dbbc {
namespace dbbc
{
typedef std::list<FileBuffer> FileBufferList_t;
/**
* @brief class to control the populating of the Disk Block Buffer Cache and manage Block requests.
**/
class BlockRequestProcessor {
class BlockRequestProcessor
{
public:
/**
* @brief default ctor
**/
BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks=0,
uint32_t blckSz=BLOCK_SIZE);
/**
* @brief default dtor
**/
virtual ~BlockRequestProcessor();
/**
* @brief default ctor
**/
BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0,
uint32_t blckSz = BLOCK_SIZE);
/**
* @brief send a request for disk blocks to the IO manager
**/
int sendRequest(fileRequest& blk) {
return fBRPRequestQueue.push(blk); }
/**
* @brief default dtor
**/
virtual ~BlockRequestProcessor();
/**
* @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not
**/
int check(BRM::LBID_t lbid, const BRM::QueryContext &ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache);
/**
* @brief send a request for disk blocks to the IO manager
**/
int sendRequest(fileRequest& blk)
{
return fBRPRequestQueue.push(blk);
}
/**
* @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not
**/
int check(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType,
uint32_t& lbidCount);
/**
* @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not
**/
int check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver, bool flg) {
return fbMgr.findPtr(HashObject_t(lbid, ver, flg)); }
/**
* @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not
**/
int check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType,
uint32_t& lbidCount);
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) {
return (fbMgr.find(HashObject_t(lbid, ver, 0), fb) ? 1 : 0); }
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver, bool flg)
{
return fbMgr.findPtr(HashObject_t(lbid, ver, flg));
}
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t &ver, void* bufferPtr) {
return (fbMgr.find(HashObject_t(lbid, ver, 0), bufferPtr) ? 1 : 0); }
const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext &ver, BRM::VER_t txn, int compType,
void* bufferPtr, bool flg, bool &wasCached, bool *wasVersioned, bool insertIntoCache,
bool readFromCache);
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
return (fbMgr.find(HashObject_t(lbid, ver, 0), fb) ? 1 : 0);
}
int getCachedBlocks(const BRM::LBID_t *lbids, const BRM::VER_t *vers, uint8_t **ptrs,
bool *wasCached, uint32_t count);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
return (fbMgr.find(HashObject_t(lbid, ver, 0), bufferPtr) ? 1 : 0);
}
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) {
return fbMgr.exists(HashObject_t(lbid, ver, 0)); }
const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, BRM::VER_t txn, int compType,
void* bufferPtr, bool flg, bool& wasCached, bool* wasVersioned, bool insertIntoCache,
bool readFromCache);
/**
* @brief
**/
void flushCache() {
fbMgr.flushCache(); }
int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** ptrs,
bool* wasCached, uint32_t count);
/**
* @brief
**/
void flushOne(BRM::LBID_t lbid, BRM::VER_t ver) {
fbMgr.flushOne(lbid, ver); }
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
return fbMgr.exists(HashObject_t(lbid, ver, 0));
}
void flushMany(const LbidAtVer* laVptr, uint32_t cnt) {
fbMgr.flushMany(laVptr, cnt); }
void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) {
fbMgr.flushManyAllversion(laVptr, cnt); }
/**
* @brief
**/
void flushCache()
{
fbMgr.flushCache();
}
void flushOIDs(const uint32_t *oids, uint32_t count) {
fbMgr.flushOIDs(oids, count); }
/**
* @brief
**/
void flushOne(BRM::LBID_t lbid, BRM::VER_t ver)
{
fbMgr.flushOne(lbid, ver);
}
void flushPartition(const std::vector<BRM::OID_t> &oids, const std::set<BRM::LogicalPartition> &partitions) {
fbMgr.flushPartition(oids, partitions); }
void flushMany(const LbidAtVer* laVptr, uint32_t cnt)
{
fbMgr.flushMany(laVptr, cnt);
}
void setReportingFrequency(const uint32_t d) {
fbMgr.setReportingFrequency(d); }
void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt)
{
fbMgr.flushManyAllversion(laVptr, cnt);
}
uint32_t ReportingFrequency() const {return fbMgr.ReportingFrequency();}
void flushOIDs(const uint32_t* oids, uint32_t count)
{
fbMgr.flushOIDs(oids, count);
}
void flushPartition(const std::vector<BRM::OID_t>& oids, const std::set<BRM::LogicalPartition>& partitions)
{
fbMgr.flushPartition(oids, partitions);
}
void setReportingFrequency(const uint32_t d)
{
fbMgr.setReportingFrequency(d);
}
uint32_t ReportingFrequency() const
{
return fbMgr.ReportingFrequency();
}
std::ostream& formatLRUList(std::ostream& os) const
{
return fbMgr.formatLRUList(os);
}
std::ostream& formatLRUList(std::ostream& os) const {
return fbMgr.formatLRUList(os); }
private:
FileBufferMgr fbMgr;
fileBlockRequestQueue fBRPRequestQueue;
ioManager fIOMgr;
boost::mutex check_mutex;
/**
* helper function for public check functions
**/
int check(fileRequest& rqstBlk);
FileBufferMgr fbMgr;
fileBlockRequestQueue fBRPRequestQueue;
ioManager fIOMgr;
boost::mutex check_mutex;
/**
* send stop requests for IOmanager and request Q
**/
void stop();
/**
* helper function for public check functions
**/
int check(fileRequest& rqstBlk);
std::ofstream fLogFile;
bool fTrace;
/**
* send stop requests for IOmanager and request Q
**/
void stop();
BRM::DBRM fdbrm;
std::ofstream fLogFile;
bool fTrace;
BRM::DBRM fdbrm;
// do not implement
BlockRequestProcessor(const BlockRequestProcessor& brp);
BlockRequestProcessor& operator=(const BlockRequestProcessor& brp);
// do not implement
BlockRequestProcessor(const BlockRequestProcessor& brp);
BlockRequestProcessor& operator=(const BlockRequestProcessor& brp);
};
}

69
primitives/blockcache/fileblockrequestqueue.cpp
View File

@ -30,62 +30,69 @@
using namespace std;
namespace dbbc {
namespace dbbc
{
fileBlockRequestQueue::fileBlockRequestQueue() : queueSize(0), readersWaiting(0)
{
//pthread_mutex_init(&mutex, NULL);
//pthread_cond_init(&notEmpty, NULL);
//pthread_mutex_init(&mutex, NULL);
//pthread_cond_init(&notEmpty, NULL);
}
fileBlockRequestQueue::~fileBlockRequestQueue()
{
//pthread_cond_destroy(&notEmpty);
//pthread_mutex_destroy(&mutex);
//pthread_cond_destroy(&notEmpty);
//pthread_mutex_destroy(&mutex);
}
bool fileBlockRequestQueue::empty() const
{
return (queueSize == 0);
return (queueSize == 0);
}
fileRequest* fileBlockRequestQueue::top() const
{
return fbQueue.front();
return fbQueue.front();
}
int fileBlockRequestQueue::push(fileRequest& blk) {
mutex.lock(); //pthread_mutex_lock(&mutex);
fbQueue.push_back(&blk);
int fileBlockRequestQueue::push(fileRequest& blk)
{
mutex.lock(); //pthread_mutex_lock(&mutex);
fbQueue.push_back(&blk);
// @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which
// caused only one i/o thread to be working at a time.
if (++queueSize >= 1 && readersWaiting > 0)
notEmpty.notify_one(); //pthread_cond_signal(&notEmpty);
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return 0;
// @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which
// caused only one i/o thread to be working at a time.
if (++queueSize >= 1 && readersWaiting > 0)
notEmpty.notify_one(); //pthread_cond_signal(&notEmpty);
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return 0;
}
void fileBlockRequestQueue::stop() {
notEmpty.notify_all(); //pthread_cond_broadcast(&notEmpty);
void fileBlockRequestQueue::stop()
{
notEmpty.notify_all(); //pthread_cond_broadcast(&notEmpty);
}
fileRequest* fileBlockRequestQueue::pop(void) {
mutex.lock(); //pthread_mutex_lock(&mutex);
while (queueSize == 0) {
readersWaiting++;
notEmpty.wait(mutex); //pthread_cond_wait(&notEmpty, &mutex);
readersWaiting--;
}
fileRequest* fileBlockRequestQueue::pop(void)
{
mutex.lock(); //pthread_mutex_lock(&mutex);
fileRequest* blk = fbQueue.front();
fbQueue.pop_front();
--queueSize;
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return blk;
while (queueSize == 0)
{
readersWaiting++;
notEmpty.wait(mutex); //pthread_cond_wait(&notEmpty, &mutex);
readersWaiting--;
}
fileRequest* blk = fbQueue.front();
fbQueue.pop_front();
--queueSize;
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return blk;
}
}

101
primitives/blockcache/fileblockrequestqueue.h
View File

@ -25,7 +25,7 @@
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include <deque>
#include <boost/thread.hpp>
@ -41,8 +41,9 @@
/**
* @brief definition of the block request queue as stl std::priority_queue
**/
namespace dbbc {
namespace dbbc
{
typedef std::deque<fileRequest*> fileBlockRequestQueue_t;
/**
@ -50,63 +51,67 @@ typedef std::deque<fileRequest*> fileBlockRequestQueue_t;
**/
class fileBlockRequestQueue {
class fileBlockRequestQueue
{
public:
/**
* @brief default ctor
**/
fileBlockRequestQueue();
/**
* @brief dtor
**/
virtual ~fileBlockRequestQueue();
/**
* @brief default ctor
**/
fileBlockRequestQueue();
/**
* @brief add a request to the queue
**/
int push(fileRequest& blk);
/**
* @brief dtor
**/
virtual ~fileBlockRequestQueue();
/**
* @brief get the next request from the queue and delete it from the queue
**/
fileRequest* pop(void);
/**
* @brief add a request to the queue
**/
int push(fileRequest& blk);
/**
* @brief true if no reuquests are in the queue. false if there are requests in the queue
**/
bool empty() const;
/**
* @brief get the next request from the queue and delete it from the queue
**/
fileRequest* pop(void);
/**
* @brief number of requests in the queue
**/
uint32_t size() const {return queueSize;}
/**
* @brief true if no reuquests are in the queue. false if there are requests in the queue
**/
bool empty() const;
/**
* @brief number of requests in the queue
**/
uint32_t size() const
{
return queueSize;
}
/**
* @brief queue will stop accecpting requests in preparation for the dtor
**/
void stop();
/**
* @brief queue will stop accecpting requests in preparation for the dtor
**/
void stop();
protected:
boost::mutex mutex;
boost::condition notEmpty;
fileBlockRequestQueue_t fbQueue;
uint32_t queueSize;
uint32_t readersWaiting;
boost::mutex mutex;
boost::condition notEmpty;
fileBlockRequestQueue_t fbQueue;
uint32_t queueSize;
uint32_t readersWaiting;
private:
// do not implement
fileBlockRequestQueue(const fileBlockRequestQueue& Q){}
const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q);
// do not implement
fileBlockRequestQueue(const fileBlockRequestQueue& Q) {}
const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q);
/**
* @brief pointer to the next request to be popped from the queue
**/
fileRequest* top() const;
/**
* @brief pointer to the next request to be popped from the queue
**/
fileRequest* top() const;
};
}
#endif

78
primitives/blockcache/filebuffer.cpp
View File

@ -30,65 +30,71 @@
using namespace std;
namespace dbbc {
namespace dbbc
{
FileBuffer::FileBuffer() : fDataLen(0), fLbid(-1), fVerid(0)
{
}
FileBuffer::FileBuffer(const FileBuffer& rhs) {
FileBuffer::FileBuffer(const FileBuffer& rhs)
{
if (this==NULL || this==&rhs)
return;
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
setData(rhs.fByteData, rhs.fDataLen);
fListLoc=rhs.listLoc();
fDataLen=rhs.fDataLen;
if (this == NULL || this == &rhs)
return;
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
setData(rhs.fByteData, rhs.fDataLen);
fListLoc = rhs.listLoc();
fDataLen = rhs.fDataLen;
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len) {
fLbid = lbid;
fVerid = ver;
fDataLen=len;
setData(data, fDataLen);
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len)
{
fLbid = lbid;
fVerid = ver;
fDataLen = len;
setData(data, fDataLen);
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver) {
fLbid=lbid;
fVerid=ver;
fDataLen=0;
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
fLbid = lbid;
fVerid = ver;
fDataLen = 0;
}
FileBuffer& FileBuffer::operator= (const FileBuffer& rhs) {
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
fDataLen=rhs.fDataLen;
setData(rhs.fByteData, fDataLen);
fListLoc=rhs.listLoc();
return *this;
FileBuffer& FileBuffer::operator= (const FileBuffer& rhs)
{
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
fDataLen = rhs.fDataLen;
setData(rhs.fByteData, fDataLen);
fListLoc = rhs.listLoc();
return *this;
}
void FileBuffer::setData(const uint8_t* d, const int len)
{
if (d==NULL || len <=0)
return;
fDataLen=len;
memcpy(fByteData, d, len);
if (d == NULL || len <= 0)
return;
fDataLen = len;
memcpy(fByteData, d, len);
}
void FileBuffer::setData(const uint8_t *d)
void FileBuffer::setData(const uint8_t* d)
{
fDataLen = 8192;
memcpy(fByteData, d, 8192);
fDataLen = 8192;
memcpy(fByteData, d, 8192);
}
FileBuffer::~FileBuffer() {
FileBuffer::~FileBuffer()
{
}
}

181
primitives/blockcache/filebuffer.h
View File

@ -45,104 +45,139 @@
/**
* @brief represents a disk blockrequest
**/
namespace dbbc {
namespace dbbc
{
typedef struct FBData{
BRM::LBID_t lbid;
BRM::VER_t ver;
uint8_t hits;
typedef struct FBData
{
BRM::LBID_t lbid;
BRM::VER_t ver;
uint8_t hits;
} FBData_t;
//@bug 669 Change to list for least recently used cache
//@bug 669 Change to list for least recently used cache
typedef std::list<FBData_t> filebuffer_list_t;
typedef std::list<FBData_t>::iterator filebuffer_list_iter_t;
class FileBuffer {
class FileBuffer
{
public:
FileBuffer();
/**
* @brief copy ctor
**/
FileBuffer(const FileBuffer& fb);
public:
/**
* @brief the disk block from lbid@ver, and a data block len bytes long
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len);
FileBuffer();
/**
* @brief copy ctor
**/
FileBuffer(const FileBuffer& fb);
/**
* @brief disk block lbid@ver and empty data
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief the disk block from lbid@ver, and a data block len bytes long
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len);
/**
* @brief class dtor
**/
/**
* @brief disk block lbid@ver and empty data
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief class dtor
**/
~FileBuffer();
/**
* @brief set the data value of this block to d have len bytestream
**/
void setData(const uint8_t* d, const int len);
void setData(const uint8_t *d); // assumes len = 8192
/**
* @brief set the data value of this block to d have len bytestream
**/
void setData(const uint8_t* d, const int len);
void setData(const uint8_t* d); // assumes len = 8192
/**
* @brief retrieve the data in byte* format from this data block
**/
inline const uint8_t* getData() const {return fByteData;}
inline uint8_t* getData() {return fByteData;}
/**
* @brief retrieve the data in byte* format from this data block
**/
inline const uint8_t* getData() const
{
return fByteData;
}
inline uint8_t* getData()
{
return fByteData;
}
inline const uint32_t datLen() const {return fDataLen;}
inline const uint32_t datLen() const
{
return fDataLen;
}
/**
* @brief assignment operator
**/
FileBuffer& operator= (const FileBuffer& rhs);
/**
* @brief assignment operator
**/
FileBuffer& operator= (const FileBuffer& rhs);
/**
* @brief equality operator is based on lbid@ver
**/
inline bool operator==(const FileBuffer& rhs) const {
return (fLbid == rhs.fLbid && fVerid == rhs.fVerid);
}
/**
* @brief equality operator is based on lbid@ver
**/
inline bool operator==(const FileBuffer& rhs) const
{
return (fLbid == rhs.fLbid && fVerid == rhs.fVerid);
}
/**
* @brief inequality operator
**/
inline bool operator!=(const FileBuffer& rhs) const {
return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid));
}
/**
* @brief inequality operator
**/
inline bool operator!=(const FileBuffer& rhs) const
{
return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid));
}
FileBuffer* thisPtr() {return this;}
/**
* @brief return the lbid value of disk bloc
**/
inline const BRM::LBID_t Lbid() const {return fLbid;}
inline void Lbid(const BRM::LBID_t l) {fLbid=l;}
FileBuffer* thisPtr()
{
return this;
}
/**
* @brief return the lbid value of disk bloc
**/
inline const BRM::LBID_t Lbid() const
{
return fLbid;
}
inline void Lbid(const BRM::LBID_t l)
{
fLbid = l;
}
/**
* @brief return the version of this disk block. ignored for range retrievals
**/
inline const BRM::VER_t Verid() const {return fVerid;}
inline void Verid(BRM::VER_t v) {fVerid=v;}
/**
* @brief return the version of this disk block. ignored for range retrievals
**/
inline const BRM::VER_t Verid() const
{
return fVerid;
}
inline void Verid(BRM::VER_t v)
{
fVerid = v;
}
/**
* @brief return the number of bytes in this disk blockrequest
**/
/**
* @brief return the number of bytes in this disk blockrequest
**/
inline void listLoc(const filebuffer_list_iter_t& loc) {fListLoc = loc;}
inline void listLoc(const filebuffer_list_iter_t& loc)
{
fListLoc = loc;
}
inline const filebuffer_list_iter_t& listLoc() const {return fListLoc;}
inline const filebuffer_list_iter_t& listLoc() const
{
return fListLoc;
}
private:
uint8_t fByteData[BLOCK_SIZE];
uint32_t fDataLen;
BRM::LBID_t fLbid;
BRM::VER_t fVerid;
filebuffer_list_iter_t fListLoc;
uint8_t fByteData[BLOCK_SIZE];
uint32_t fDataLen;
BRM::LBID_t fLbid;
BRM::VER_t fVerid;
filebuffer_list_iter_t fListLoc;
};
typedef std::vector<FileBuffer> FileBufferPool_t;

1069
primitives/blockcache/filebuffermgr.cpp
View File

File diff suppressed because it is too large Load Diff

244
primitives/blockcache/filebuffermgr.h
View File

@ -46,10 +46,11 @@
/**
* @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and list.
*
*
**/
namespace dbbc {
namespace dbbc
{
/**
* @brief used as the hasher algorithm for the unordered_set used to store the disk blocks
@ -57,168 +58,185 @@ namespace dbbc {
struct FileBufferIndex
{
FileBufferIndex(BRM::LBID_t l, BRM::VER_t v, uint32_t p) : lbid(l), ver(v), poolIdx(p) {}
BRM::LBID_t lbid;
BRM::VER_t ver;
uint32_t poolIdx;
FileBufferIndex(BRM::LBID_t l, BRM::VER_t v, uint32_t p) : lbid(l), ver(v), poolIdx(p) {}
BRM::LBID_t lbid;
BRM::VER_t ver;
uint32_t poolIdx;
};
struct CacheInsert_t {
CacheInsert_t(const BRM::LBID_t &l, const BRM::VER_t &v, const uint8_t *d) :
lbid(l), ver(v), data(d) { }
BRM::LBID_t lbid;
BRM::VER_t ver;
const uint8_t *data;
struct CacheInsert_t
{
CacheInsert_t(const BRM::LBID_t& l, const BRM::VER_t& v, const uint8_t* d) :
lbid(l), ver(v), data(d) { }
BRM::LBID_t lbid;
BRM::VER_t ver;
const uint8_t* data;
};
typedef FileBufferIndex HashObject_t;
class bcHasher
{
public:
inline size_t operator()(const HashObject_t& rhs) const
{
return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL));
}
public:
inline size_t operator()(const HashObject_t& rhs) const
{
return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL));
}
};
class bcEqual
{
public:
inline bool operator()(const HashObject_t& f1, const HashObject_t& f2) const
{
return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver));
}
public:
inline bool operator()(const HashObject_t& f1, const HashObject_t& f2) const
{
return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver));
}
};
inline bool operator<(const HashObject_t& f1, const HashObject_t& f2)
{
return ((f1.lbid < f2.lbid) || (f1.lbid == f2.lbid && f1.ver < f2.ver));
return ((f1.lbid < f2.lbid) || (f1.lbid == f2.lbid && f1.ver < f2.ver));
}
class FileBufferMgr {
class FileBufferMgr
{
public:
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual> filebuffer_uset_t;
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual>::const_iterator filebuffer_uset_iter_t;
typedef std::pair<filebuffer_uset_t::iterator, bool> filebuffer_pair_t; // return type for insert
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual> filebuffer_uset_t;
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual>::const_iterator filebuffer_uset_iter_t;
typedef std::pair<filebuffer_uset_t::iterator, bool> filebuffer_pair_t; // return type for insert
typedef std::deque<uint32_t> emptylist_t;
typedef std::deque<uint32_t> emptylist_t;
/**
* @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz
**/
/**
* @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz
**/
FileBufferMgr(uint32_t numBlcks, uint32_t blckSz=BLOCK_SIZE, uint32_t deleteBlocks = 0);
FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0);
/**
* @brief default dtor
**/
/**
* @brief default dtor
**/
virtual ~FileBufferMgr();
/**
* @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const;
/**
* @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const HashObject_t& fb) const;
/**
* @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const;
/**
* @brief add the Disk Block reference by fb into the Disk Block Buffer Cache
**/
int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data);
/**
* @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const HashObject_t& fb) const;
int bulkInsert(const std::vector<CacheInsert_t> &);
/**
* @brief add the Disk Block reference by fb into the Disk Block Buffer Cache
**/
int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data);
/**
* @brief returns the total number of Disk Blocks in the Cache
**/
uint32_t size() const {return fbSet.size();}
int bulkInsert(const std::vector<CacheInsert_t>&);
/**
* @brief
**/
void flushCache();
/**
* @brief returns the total number of Disk Blocks in the Cache
**/
uint32_t size() const
{
return fbSet.size();
}
/**
* @brief
**/
void flushOne(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief
**/
void flushCache();
/**
* @brief
**/
void flushMany(const LbidAtVer* laVptr, uint32_t cnt);
/**
* @brief flush all versions
**/
void flushManyAllversion(const BRM::LBID_t *laVptr, uint32_t cnt);
/**
* @brief
**/
void flushOne(const BRM::LBID_t lbid, const BRM::VER_t ver);
void flushOIDs(const uint32_t *oids, uint32_t count);
void flushPartition(const std::vector<BRM::OID_t> &oids, const std::set<BRM::LogicalPartition> &partitions);
/**
* @brief
**/
void flushMany(const LbidAtVer* laVptr, uint32_t cnt);
/**
* @brief return the disk Block referenced by fb
**/
/**
* @brief flush all versions
**/
void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt);
FileBuffer* findPtr(const HashObject_t& keyFb);
void flushOIDs(const uint32_t* oids, uint32_t count);
void flushPartition(const std::vector<BRM::OID_t>& oids, const std::set<BRM::LogicalPartition>& partitions);
bool find(const HashObject_t& keyFb, FileBuffer& fb);
/**
* @brief return the disk Block referenced by fb
**/
/**
* @brief return the disk Block referenced by bufferPtr
**/
FileBuffer* findPtr(const HashObject_t& keyFb);
bool find(const HashObject_t& keyFb, void* bufferPtr);
uint32_t bulkFind(const BRM::LBID_t *lbids, const BRM::VER_t *vers, uint8_t **buffers,
bool *wasCached, uint32_t blockCount);
uint32_t maxCacheSize() const {return fMaxNumBlocks;}
bool find(const HashObject_t& keyFb, FileBuffer& fb);
uint32_t listSize() const {return fbList.size();}
/**
* @brief return the disk Block referenced by bufferPtr
**/
const filebuffer_uset_iter_t end() const {return fbSet.end();}
bool find(const HashObject_t& keyFb, void* bufferPtr);
uint32_t bulkFind(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** buffers,
bool* wasCached, uint32_t blockCount);
void setReportingFrequency(const uint32_t d);
const uint32_t ReportingFrequency() const {return fReportFrequency;}
uint32_t maxCacheSize() const
{
return fMaxNumBlocks;
}
std::ostream& formatLRUList(std::ostream& os) const;
uint32_t listSize() const
{
return fbList.size();
}
const filebuffer_uset_iter_t end() const
{
return fbSet.end();
}
void setReportingFrequency(const uint32_t d);
const uint32_t ReportingFrequency() const
{
return fReportFrequency;
}
std::ostream& formatLRUList(std::ostream& os) const;
private:
uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list
uint32_t fBlockSz; // size in bytes size of a data block - probably 8
uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list
uint32_t fBlockSz; // size in bytes size of a data block - probably 8
mutable boost::mutex fWLock;
mutable filebuffer_uset_t fbSet;
mutable boost::mutex fWLock;
mutable filebuffer_uset_t fbSet;
mutable filebuffer_list_t fbList; // rename this
uint32_t fCacheSize;
mutable filebuffer_list_t fbList; // rename this
uint32_t fCacheSize;
FileBufferPool_t fFBPool; // vector<FileBuffer>
uint32_t fDeleteBlocks;
emptylist_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused
FileBufferPool_t fFBPool; // vector<FileBuffer>
uint32_t fDeleteBlocks;
emptylist_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused
void depleteCache();
uint64_t fBlksLoaded; // number of blocks inserted into cache
uint64_t fBlksNotUsed; // number of blocks inserted and not used
uint64_t fReportFrequency; // how many blocks are read between reports
std::ofstream fLog;
config::Config* fConfig;
// do not implement
FileBufferMgr(const FileBufferMgr& fbm);
const FileBufferMgr& operator =(const FileBufferMgr& fbm);
// used by bulkInsert
void updateLRU(const FBData_t &f);
uint32_t doBlockCopy(const BRM::LBID_t &lbid, const BRM::VER_t &ver, const uint8_t *data);
void depleteCache();
uint64_t fBlksLoaded; // number of blocks inserted into cache
uint64_t fBlksNotUsed; // number of blocks inserted and not used
uint64_t fReportFrequency; // how many blocks are read between reports
std::ofstream fLog;
config::Config* fConfig;
// do not implement
FileBufferMgr(const FileBufferMgr& fbm);
const FileBufferMgr& operator =(const FileBufferMgr& fbm);
// used by bulkInsert
void updateLRU(const FBData_t& f);
uint32_t doBlockCopy(const BRM::LBID_t& lbid, const BRM::VER_t& ver, const uint8_t* data);
};
}

83
primitives/blockcache/filerequest.cpp
View File

@ -27,68 +27,69 @@ using namespace std;
#include "filerequest.h"
namespace dbbc {
namespace dbbc
{
fileRequest::fileRequest() :
data(0), fLBID(-1), fVer(-1), fFlg(false), fTxn(-1), fRqstType(LBIDREQUEST), fCompType(0),
cache(true), wasVersioned(false)
data(0), fLBID(-1), fVer(-1), fFlg(false), fTxn(-1), fRqstType(LBIDREQUEST), fCompType(0),
cache(true), wasVersioned(false)
{
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
}
fileRequest::fileRequest(BRM::LBID_t lbid, const BRM::QueryContext &ver, bool flg, BRM::VER_t txn, int compType,
uint8_t *ptr, bool cacheIt) :
data(ptr), fLBID(lbid), fVer(ver), fFlg(flg), fTxn(txn), fRqstType(LBIDREQUEST), fCompType(compType),
cache(cacheIt), wasVersioned(false)
fileRequest::fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, int compType,
uint8_t* ptr, bool cacheIt) :
data(ptr), fLBID(lbid), fVer(ver), fFlg(flg), fTxn(txn), fRqstType(LBIDREQUEST), fCompType(compType),
cache(cacheIt), wasVersioned(false)
{
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = 1;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = 1;
}
fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, BRM::VER_t txn, int compType) :
data(0), fLBID(range.start), fVer(ver), fFlg(false), fTxn(txn), fLength(range.size),
fRqstType(RANGEREQUEST), fCompType(compType), cache(true), wasVersioned(false)
fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, int compType) :
data(0), fLBID(range.start), fVer(ver), fFlg(false), fTxn(txn), fLength(range.size),
fRqstType(RANGEREQUEST), fCompType(compType), cache(true), wasVersioned(false)
{
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = range.size;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = range.size;
}
fileRequest::fileRequest(const fileRequest& blk)
{
fLBID = blk.fLBID;
fVer = blk.fVer;
fTxn = blk.fTxn;
fFlg = blk.fFlg;
fRqstType = blk.fRqstType;
fRqstStatusString = blk.fRqstStatusString;
data = blk.data;
fCompType = blk.fCompType;
cache = blk.cache;
wasVersioned = blk.wasVersioned;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLBID = blk.fLBID;
fVer = blk.fVer;
fTxn = blk.fTxn;
fFlg = blk.fFlg;
fRqstType = blk.fRqstType;
fRqstStatusString = blk.fRqstStatusString;
data = blk.data;
fCompType = blk.fCompType;
cache = blk.cache;
wasVersioned = blk.wasVersioned;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
}
void fileRequest::init()
{
fFRPredicate = INIT;
fLength = 0;
fblksRead = 0;
fblksLoaded = 0;
fRqstStatus = SUCCESSFUL;
fFRPredicate = INIT;
fLength = 0;
fblksRead = 0;
fblksLoaded = 0;
fRqstStatus = SUCCESSFUL;
}
ostream& operator<<(ostream& os, const fileRequest& rhs)
{
os
<< "LBID: " << rhs.fLBID
<< " ver: " << rhs.fVer
<< " Txn: " << rhs.fTxn
<< " len: " << rhs.fLength
<< " read: " << rhs.fblksRead
<< " load: " << rhs.fblksLoaded
<< " ct: " << rhs.fCompType
;
return os;
os
<< "LBID: " << rhs.fLBID
<< " ver: " << rhs.fVer
<< " Txn: " << rhs.fTxn
<< " len: " << rhs.fLength
<< " read: " << rhs.fblksRead
<< " load: " << rhs.fblksLoaded
<< " ct: " << rhs.fCompType
;
return os;
}
}

395
primitives/blockcache/filerequest.h
View File

@ -41,202 +41,285 @@
/**
* @brief request class for the fileblockrequestqueue and the iomanager
**/
namespace dbbc {
namespace dbbc
{
class fileRequest {
class fileRequest
{
public:
/**
* @brief default ctor
**/
fileRequest();
/**
* @brief default ctor
**/
fileRequest();
/**
* @brief copy constructor
**/
fileRequest(const fileRequest& blk);
/**
* @brief copy constructor
**/
fileRequest(const fileRequest& blk);
/**
* @brief request for the disk block lbid@ver
* note, useCache tells IOManager to cache the loaded blocks.
**/
fileRequest(BRM::LBID_t lbid, const BRM::QueryContext &ver, bool flg, BRM::VER_t txn, int compType,
uint8_t *ptr=0, bool useCache = true);
/**
* @brief request for the disk block lbid@ver
* note, useCache tells IOManager to cache the loaded blocks.
**/
fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, int compType,
uint8_t* ptr = 0, bool useCache = true);
/**
* @brief request a range of disk blocks
**/
fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, BRM::VER_t txn, int compType);
/**
* @brief request a range of disk blocks
**/
fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, int compType);
/**
* @brief class dtor
**/
/**
* @brief class dtor
**/
virtual ~fileRequest() {};
/**
* @brief less-than operator
**/
bool operator< (const fileRequest & rhs) const {
return fLength < rhs.fLength;
}
/**
* @brief less-than operator
**/
bool operator< (const fileRequest& rhs) const
{
return fLength < rhs.fLength;
}
/**
* @brief greater-than operator
**/
bool operator> (const fileRequest & rhs) const {
return fLength > rhs.fLength;
}
/**
* @brief greater-than operator
**/
bool operator> (const fileRequest& rhs) const
{
return fLength > rhs.fLength;
}
/**
* @brief equality operator
**/
bool operator== (const fileRequest & rhs) const {
return fLength == rhs.fLength;
}
/**
* @brief equality operator
**/
bool operator== (const fileRequest& rhs) const
{
return fLength == rhs.fLength;
}
enum request_status_enum {
SUCCESSFUL,
FAILED,
BRM_LOOKUP_ERROR,
FS_EINVAL,
FS_ENOENT
};
enum request_status_enum
{
SUCCESSFUL,
FAILED,
BRM_LOOKUP_ERROR,
FS_EINVAL,
FS_ENOENT
};
enum request_type_enum {
LBIDREQUEST,
RANGEREQUEST
};
enum request_type_enum
{
LBIDREQUEST,
RANGEREQUEST
};
/**
* @brief used to manage request processing synchronzation
**/
enum predicate_status_enum {
INIT,
SENDING,
READING,
COMPLETE,
STOP
};
/**
* @brief used to manage request processing synchronzation
**/
enum predicate_status_enum
{
INIT,
SENDING,
READING,
COMPLETE,
STOP
};
/**
* @brief lbid requested
**/
const BRM::LBID_t Lbid() const {return fLBID;}
/**
* @brief lbid requested
**/
const BRM::LBID_t Lbid() const
{
return fLBID;
}
/**
* @brief version of the lbid requested
**/
const BRM::QueryContext Ver() const {return fVer;}
/**
* @brief version of the lbid requested
**/
const BRM::QueryContext Ver() const
{
return fVer;
}
/**
* @brief VBBM flag of the LBID/Ver
**/
const bool Flg() const {return fFlg;}
/**
* @brief VBBM flag of the LBID/Ver
**/
const bool Flg() const
{
return fFlg;
}
const BRM::VER_t Txn() const { return fTxn;}
const BRM::VER_t Txn() const
{
return fTxn;
}
const int CompType() const { return fCompType;}
const int CompType() const
{
return fCompType;
}
/**
* @brief number of blocks requested
**/
const uint32_t BlocksRequested() const {return fLength;}
/**
* @brief number of blocks requested
**/
const uint32_t BlocksRequested() const
{
return fLength;
}
/**
* @brief setter for blocks requested
**/
void BlocksRequested(const int l) {fLength=l;}
/**
* @brief setter for blocks requested
**/
void BlocksRequested(const int l)
{
fLength = l;
}
/**
* @brief number of blocks read from disk
**/
const uint32_t BlocksRead() const {return fblksRead;}
const uint32_t BlocksLoaded() const {return fblksLoaded;}
/**
* @brief number of blocks read from disk
**/
const uint32_t BlocksRead() const
{
return fblksRead;
}
const uint32_t BlocksLoaded() const
{
return fblksLoaded;
}
/**
* @brief setter for blocks read from disk
**/
void BlocksRead(const int l) {fblksRead=l;}
void BlocksLoaded(const int l) {fblksLoaded=l;}
/**
* @brief setter for blocks read from disk
**/
void BlocksRead(const int l)
{
fblksRead = l;
}
void BlocksLoaded(const int l)
{
fblksLoaded = l;
}
/**
* @brief did the request succeed for fail (0-success; else failure)
**/
int RequestStatus() const {return fRqstStatus;}
/**
* @brief did the request succeed for fail (0-success; else failure)
**/
int RequestStatus() const
{
return fRqstStatus;
}
/**
* @brief setter for the request status
**/
void RequestStatus(int s) {fRqstStatus=s;}
/**
* @brief setter for the request status
**/
void RequestStatus(int s)
{
fRqstStatus = s;
}
void CompType(int compType) { fCompType = compType; }
void CompType(int compType)
{
fCompType = compType;
}
/**
* @brief if request failed, this is the error message string
**/
std::string RequestStatusStr() const {return fRqstStatusString;}
/**
* @brief if request failed, this is the error message string
**/
std::string RequestStatusStr() const
{
return fRqstStatusString;
}
/**
* @brief setter for the request status error message string
**/
void RequestStatusStr(const std::string& s) {fRqstStatusString=s;}
/**
* @brief setter for the request status error message string
**/
void RequestStatusStr(const std::string& s)
{
fRqstStatusString = s;
}
/**
* @brief return BLOCK or RANGE requested
**/
int RequestType() const {return fRqstType;}
/**
* @brief return BLOCK or RANGE requested
**/
int RequestType() const
{
return fRqstType;
}
/**
* @brief mutex to control synchronzation of request processing
**/
boost::mutex& frMutex() const {return fFRMutex;}
/**
* @brief mutex to control synchronzation of request processing
**/
boost::mutex& frMutex() const
{
return fFRMutex;
}
/**
* @brief condition variable. signal when request is complete
**/
boost::condition& frCond() const {return fFRCond;}
/**
* @brief condition variable. signal when request is complete
**/
boost::condition& frCond() const
{
return fFRCond;
}
/**
* @brief
**/
const enum predicate_status_enum& frPredicate() const {return fFRPredicate;}
/**
* @brief
**/
const enum predicate_status_enum& frPredicate() const
{
return fFRPredicate;
}
/**
* @brief setter for the predicate
**/
void SetPredicate(const enum predicate_status_enum& p) {fFRPredicate=p;}
/**
* @brief setter for the predicate
**/
void SetPredicate(const enum predicate_status_enum& p)
{
fFRPredicate = p;
}
uint8_t *data;
uint8_t* data;
friend std::ostream& operator<<(std::ostream& os, const fileRequest& rhs);
friend std::ostream& operator<<(std::ostream& os, const fileRequest& rhs);
// tells IOManager to cache the loaded blocks or not
bool useCache() { return cache; }
void useCache(bool c) { cache = c; }
bool versioned() { return wasVersioned; }
void versioned(bool b) { wasVersioned = b; }
// tells IOManager to cache the loaded blocks or not
bool useCache()
{
return cache;
}
void useCache(bool c)
{
cache = c;
}
bool versioned()
{
return wasVersioned;
}
void versioned(bool b)
{
wasVersioned = b;
}
private:
void init();
void init();
BRM::LBID_t fLBID;
BRM::QueryContext fVer;
bool fFlg;
BRM::VER_t fTxn;
mutable boost::mutex fFRMutex;
mutable boost::condition fFRCond;
predicate_status_enum fFRPredicate;
uint32_t fLength; // lbids requested
uint32_t fblksRead; // lbids read
uint32_t fblksLoaded; // lbids loaded into cache
int fRqstStatus;
std::string fRqstStatusString;
enum request_type_enum fRqstType;
int fCompType;
bool cache;
bool wasVersioned;
BRM::LBID_t fLBID;
BRM::QueryContext fVer;
bool fFlg;
BRM::VER_t fTxn;
mutable boost::mutex fFRMutex;
mutable boost::condition fFRCond;
predicate_status_enum fFRPredicate;
uint32_t fLength; // lbids requested
uint32_t fblksRead; // lbids read
uint32_t fblksLoaded; // lbids loaded into cache
int fRqstStatus;
std::string fRqstStatusString;
enum request_type_enum fRqstType;
int fCompType;
bool cache;
bool wasVersioned;
};
}

102
primitives/blockcache/fsutils.cpp
View File

@ -25,7 +25,7 @@ using namespace std;
#include <boost/tokenizer.hpp>
#include <boost/filesystem.hpp>
using namespace boost;
namespace fs=boost::filesystem;
namespace fs = boost::filesystem;
#include "fsutils.h"
#include "exceptclasses.h"
@ -35,41 +35,49 @@ namespace
const string resolveInDir(const string& dir, const string& name)
{
idbassert(!dir.empty() && !name.empty());
string ret;
fs::path path(dir);
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
path /= name;
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
idbassert(!dir.empty() && !name.empty());
string ret;
fs::path path(dir);
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
path /= name;
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
#ifndef _MSC_VER
if (!fs::is_symlink(path))
return ret;
idbassert(fs::is_symlink(path));
char* realname = (char*)alloca(PATH_MAX+1);
ssize_t realnamelen = readlink(path.string().c_str(), realname, PATH_MAX);
if (realnamelen <= 0)
return ret;
realname[realnamelen] = 0;
fs::path linkname(realname);
fs::path realpath("/dev");
realpath /= linkname.filename();
ret = realpath.string();
if (!fs::is_symlink(path))
return ret;
idbassert(fs::is_symlink(path));
char* realname = (char*)alloca(PATH_MAX + 1);
ssize_t realnamelen = readlink(path.string().c_str(), realname, PATH_MAX);
if (realnamelen <= 0)
return ret;
realname[realnamelen] = 0;
fs::path linkname(realname);
fs::path realpath("/dev");
realpath /= linkname.filename();
ret = realpath.string();
#endif
return ret;
return ret;
}
inline const string label2dev(const string& name)
{
return resolveInDir("/dev/disk/by-label", name);
return resolveInDir("/dev/disk/by-label", name);
}
inline const string uuid2dev(const string& name)
{
return resolveInDir("/dev/disk/by-uuid", name);
return resolveInDir("/dev/disk/by-uuid", name);
}
}
@ -79,33 +87,35 @@ namespace fsutils
const string symname2devname(const string& sympath)
{
string ret;
string ret;
#ifndef _MSC_VER
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
boost::char_separator<char> sep("=");
tokenizer tokens(sympath, sep);
tokenizer::iterator tok_iter = tokens.begin();
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
boost::char_separator<char> sep("=");
tokenizer tokens(sympath, sep);
tokenizer::iterator tok_iter = tokens.begin();
idbassert(tok_iter != tokens.end());
string symtype = *tok_iter;
if (symtype != "LABEL" && symtype != "UUID")
return ret;
idbassert(tok_iter != tokens.end());
string symtype = *tok_iter;
idbassert(symtype == "LABEL" || symtype == "UUID");
if (symtype != "LABEL" && symtype != "UUID")
return ret;
++tok_iter;
idbassert(tok_iter != tokens.end());
string symname = *tok_iter;
idbassert(symtype == "LABEL" || symtype == "UUID");
++tok_iter;
idbassert(tok_iter == tokens.end());
++tok_iter;
idbassert(tok_iter != tokens.end());
string symname = *tok_iter;
++tok_iter;
idbassert(tok_iter == tokens.end());
if (symtype == "LABEL")
ret = label2dev(symname);
else if (symtype == "UUID")
ret = uuid2dev(symname);
if (symtype == "LABEL")
ret = label2dev(symname);
else if (symtype == "UUID")
ret = uuid2dev(symname);
#endif
return ret;
return ret;
}
}

2172
primitives/blockcache/iomanager.cpp
View File

File diff suppressed because it is too large Load Diff

134
primitives/blockcache/iomanager.h
View File

@ -21,7 +21,7 @@
//
// C++ Interface: iomanager
//
// Description:
// Description:
//
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>
@ -45,25 +45,36 @@
//#define SHARED_NOTHING_DEMO_2
namespace dbbc {
namespace dbbc
{
class ioManager
{
class ioManager {
public:
ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount,
int bsPerRead);
//ioManager(FileBufferMgr& fbm, int thrCount);
~ioManager();
int readerCount() const {return fThreadCount;}
fileRequest* getNextRequest();
void go(void);
void stop();
FileBufferMgr& fileBufferManager() {return fIOMfbMgr;}
config::Config* configPtr() {return fConfig;}
ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount,
int bsPerRead);
//ioManager(FileBufferMgr& fbm, int thrCount);
~ioManager();
int readerCount() const
{
return fThreadCount;
}
fileRequest* getNextRequest();
void go(void);
void stop();
FileBufferMgr& fileBufferManager()
{
return fIOMfbMgr;
}
config::Config* configPtr()
{
return fConfig;
}
const int localLbidLookup(BRM::LBID_t lbid,
BRM::VER_t verid,
BRM::VER_t verid,
bool vbFlag,
BRM::OID_t& oid,
uint16_t& dbRoot,
@ -72,55 +83,76 @@ public:
uint32_t& fileBlockOffset);
void buildOidFileName(const BRM::OID_t oid,
const uint16_t dbRoot,
const uint16_t partNum,
const uint32_t segNum,
char* file_name);
const uint16_t dbRoot,
const uint16_t partNum,
const uint32_t segNum,
char* file_name);
const uint32_t getExtentRows() { return fdbrm.getExtentRows(); }
const uint32_t getExtentRows()
{
return fdbrm.getExtentRows();
}
uint32_t blocksPerRead;
uint32_t blocksPerRead;
bool IOTrace() const { return fIOTrace;}
bool IOTrace() const
{
return fIOTrace;
}
uint32_t MaxOpenFiles() const { return fMaxOpenFiles;}
uint32_t MaxOpenFiles() const
{
return fMaxOpenFiles;
}
uint32_t DecreaseOpenFilesCount() const { return fDecreaseOpenFilesCount;}
uint32_t DecreaseOpenFilesCount() const
{
return fDecreaseOpenFilesCount;
}
bool FDCacheTrace() const { return fFDCacheTrace;}
bool FDCacheTrace() const
{
return fFDCacheTrace;
}
void handleBlockReadError ( fileRequest* fr,
const std::string& errMsg, bool *copyLocked, int errorCode=fileRequest::FAILED );
void handleBlockReadError ( fileRequest* fr,
const std::string& errMsg, bool* copyLocked, int errorCode = fileRequest::FAILED );
std::ofstream& FDTraceFile()
{
return fFDTraceFile;
}
BRM::DBRM* dbrm()
{
return &fdbrm;
}
std::ofstream& FDTraceFile() {return fFDTraceFile;}
BRM::DBRM* dbrm() { return &fdbrm;}
#ifdef SHARED_NOTHING_DEMO_2
uint32_t pmCount;
uint32_t pmCount;
#endif
private:
FileBufferMgr& fIOMfbMgr;
fileBlockRequestQueue& fIOMRequestQueue;
int fThreadCount;
boost::thread_group fThreadArr;
void createReaders();
config::Config* fConfig;
BRM::DBRM fdbrm;
WriteEngine::FileOp fFileOp;
// do not implement
ioManager();
ioManager(const ioManager& iom);
const ioManager& operator=(const ioManager& iom);
bool fIOTrace;
uint32_t fMaxOpenFiles;
uint32_t fDecreaseOpenFilesCount;
bool fFDCacheTrace;
std::ofstream fFDTraceFile;
FileBufferMgr& fIOMfbMgr;
fileBlockRequestQueue& fIOMRequestQueue;
int fThreadCount;
boost::thread_group fThreadArr;
void createReaders();
config::Config* fConfig;
BRM::DBRM fdbrm;
WriteEngine::FileOp fFileOp;
// do not implement
ioManager();
ioManager(const ioManager& iom);
const ioManager& operator=(const ioManager& iom);
bool fIOTrace;
uint32_t fMaxOpenFiles;
uint32_t fDecreaseOpenFilesCount;
bool fFDCacheTrace;
std::ofstream fFDTraceFile;
};

302
primitives/blockcache/stats.cpp
View File

@ -50,118 +50,128 @@ namespace
void pause_(unsigned delay)
{
struct timespec req;
struct timespec rem;
struct timespec req;
struct timespec rem;
req.tv_sec = delay;
req.tv_nsec = 0;
req.tv_sec = delay;
req.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
#ifdef _MSC_VER
Sleep(req.tv_sec * 1000);
Sleep(req.tv_sec * 1000);
#else
again:
if (nanosleep(&req, &rem) != 0)
if (rem.tv_sec > 0 || rem.tv_nsec > 0) {
req = rem;
goto again;
}
if (nanosleep(&req, &rem) != 0)
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
goto again;
}
#endif
}
const string timestr()
{
// Get a timestamp for output.
struct tm tm;
struct timeval tv;
// Get a timestamp for output.
struct tm tm;
struct timeval tv;
gettimeofday(&tv, 0);
localtime_r(reinterpret_cast<time_t*>(&tv.tv_sec), &tm);
gettimeofday(&tv, 0);
localtime_r(reinterpret_cast<time_t*>(&tv.tv_sec), &tm);
ostringstream oss;
oss << setfill('0')
<< setw(2) << tm.tm_hour << ':'
<< setw(2) << tm.tm_min << ':'
<< setw(2) << tm.tm_sec << '.'
<< setw(4) << tv.tv_usec/100;
ostringstream oss;
oss << setfill('0')
<< setw(2) << tm.tm_hour << ':'
<< setw(2) << tm.tm_min << ':'
<< setw(2) << tm.tm_sec << '.'
<< setw(4) << tv.tv_usec / 100;
return oss.str();
return oss.str();
}
class TraceFile
{
public:
TraceFile(uint32_t sessionID, const char* name)
{
if (sessionID > 0 )
{
const char* outName;
if (name == 0)
outName = "lbids";
else
outName = name;
ostringstream oss;
TraceFile(uint32_t sessionID, const char* name)
{
if (sessionID > 0 )
{
const char* outName;
if (name == 0)
outName = "lbids";
else
outName = name;
ostringstream oss;
#ifdef _MSC_VER
oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID;
oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID;
#else
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID;
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID;
#endif
oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
}
}
oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
}
}
~TraceFile()
{
}
~TraceFile()
{
}
void close()
{
if (oFile)
{
oFile->close();
}
}
void close()
{
if (oFile)
{
oFile->close();
}
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event='\0')
{
*oFile << oid << ' ' << timestr() << ' ' << lbid
<< ' ' << thdid;
if (event != '\0')
*oFile << ' ' << event;
*oFile << endl;
oFile->flush();
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
*oFile << oid << ' ' << timestr() << ' ' << lbid
<< ' ' << thdid;
if (event != '\0')
*oFile << ' ' << event;
*oFile << endl;
oFile->flush();
}
private:
//Compiler defaults okay
//TraceFile(const TraceFile& rhs);
//TraceFile operator=(const TraceFile& rhs);
boost::shared_ptr<ofstream> oFile;
//Compiler defaults okay
//TraceFile(const TraceFile& rhs);
//TraceFile operator=(const TraceFile& rhs);
boost::shared_ptr<ofstream> oFile;
};
struct TraceFileInfo
{
TraceFileInfo(uint32_t session=0, const char* name=0) : traceFile(session, name), lastTouched(0) { }
~TraceFileInfo() { }
TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) { }
~TraceFileInfo() { }
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event='\0')
{
traceFile.log(oid, lbid, thdid, event);
lastTouched = time(0);
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
traceFile.log(oid, lbid, thdid, event);
lastTouched = time(0);
}
void close() { traceFile.close(); }
void close()
{
traceFile.close();
}
TraceFile traceFile;
time_t lastTouched;
TraceFile traceFile;
time_t lastTouched;
private:
//Compiler defaults okay
//TraceFileInfo(const TraceFileInfo& rhs);
//TraceFileInfo operator=(const TraceFileInfo& rhs);
//Compiler defaults okay
//TraceFileInfo(const TraceFileInfo& rhs);
//TraceFileInfo operator=(const TraceFileInfo& rhs);
};
//map a session id to a trace file
@ -174,49 +184,51 @@ mutex traceFileMapMutex;
class StatMon
{
public:
StatMon()
{
StatMon()
{
#ifndef _MSC_VER
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
pthread_sigmask(SIG_BLOCK, &sigset, 0);
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
pthread_sigmask(SIG_BLOCK, &sigset, 0);
#endif
}
void operator()() const
{
//struct timespec ts = { 60 * 1, 0 };
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter;
TraceFileMap_t::iterator end;
for (;;)
{
lk.unlock();
time_t beforeSleep = time(0);
//nanosleep(&ts, 0);
pause_(60);
lk.lock();
iter = traceFileMap.begin();
end = traceFileMap.end();
while (iter != end)
{
if (iter->second.lastTouched < beforeSleep)
{
//remove this session trace file
iter->second.close();
traceFileMap.erase(iter++);
}
else
++iter;
}
}
}
}
void operator()() const
{
//struct timespec ts = { 60 * 1, 0 };
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter;
TraceFileMap_t::iterator end;
for (;;)
{
lk.unlock();
time_t beforeSleep = time(0);
//nanosleep(&ts, 0);
pause_(60);
lk.lock();
iter = traceFileMap.begin();
end = traceFileMap.end();
while (iter != end)
{
if (iter->second.lastTouched < beforeSleep)
{
//remove this session trace file
iter->second.close();
traceFileMap.erase(iter++);
}
else
++iter;
}
}
}
private:
//Compiler defaults okay
//StatMon(const StatMon& rhs);
//StatMon operator=(const StatMon& rhs);
//Compiler defaults okay
//StatMon(const StatMon& rhs);
//StatMon operator=(const StatMon& rhs);
};
}
@ -225,52 +237,56 @@ namespace dbbc
{
Stats::Stats() :
fMonitorp(0)
fMonitorp(0)
{
fMonitorp = new boost::thread(StatMon());
fMonitorp = new boost::thread(StatMon());
}
Stats::Stats(const char *name) :
fMonitorp(0), fName(name)
Stats::Stats(const char* name) :
fMonitorp(0), fName(name)
{
fMonitorp = new boost::thread(StatMon());
//fName << name;
fMonitorp = new boost::thread(StatMon());
//fName << name;
}
Stats::~Stats()
{
delete fMonitorp;
delete fMonitorp;
}
void Stats::touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session)
{
if (lbid < 0 || session == 0) return;
if (lbid < 0 || session == 0) return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid);
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid);
}
void Stats::markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event)
{
if (lbid < 0 || session == 0) return;
if (lbid < 0 || session == 0) return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session, fName);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid, event);
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session, fName);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid, event);
}
}

42
primitives/blockcache/stats.h
View File

@ -42,32 +42,32 @@ namespace dbbc
class Stats
{
public:
Stats();
Stats(const char *name);
virtual ~Stats();
Stats();
Stats(const char* name);
virtual ~Stats();
void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session=0);
void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event);
void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0);
void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event);
inline BRM::OID_t lbid2oid(uint64_t lbid)
{
BRM::OID_t oid;
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo);
return oid;
}
inline BRM::OID_t lbid2oid(uint64_t lbid)
{
BRM::OID_t oid;
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo);
return oid;
}
private:
Stats(const Stats& rhs);
Stats& operator=(const Stats& rhs);
Stats(const Stats& rhs);
Stats& operator=(const Stats& rhs);
boost::thread* fMonitorp;
BRM::DBRM brm;
//ostringstream fName;
const char* fName;
boost::thread* fMonitorp;
BRM::DBRM brm;
//ostringstream fName;
const char* fName;
};

569
primitives/blockcache/tdriver.cpp
View File

@ -40,316 +40,369 @@ using namespace BRM;
using namespace dbbc;
using namespace std;
Stats* gPMStatsPtr=NULL;
bool gPMProfOn=false;
uint32_t gSession=0;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
int fLoops=1;
int thr_cnt=1;
uint64_t bfoundTot=0;
uint64_t bnfoundTot=0;
uint64_t rfoundTot=0;
uint64_t rnfoundTot=0;
uint64_t rangeOpCountTot=0;
uint64_t blockOpCountTot=0;
uint64_t noOpCountTot=0;
int fLoops = 1;
int thr_cnt = 1;
uint64_t bfoundTot = 0;
uint64_t bnfoundTot = 0;
uint64_t rfoundTot = 0;
uint64_t rnfoundTot = 0;
uint64_t rangeOpCountTot = 0;
uint64_t blockOpCountTot = 0;
uint64_t noOpCountTot = 0;
struct thr_wait_struct {
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
struct thr_wait_struct
{
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
};
typedef thr_wait_struct thr_wait_t;
const int32_t cacheSize=175000;
const int32_t cacheSize = 175000;
BlockRequestProcessor BRP(cacheSize, 4, 16);
BRM::VER_t ver=0xFFFF;
u_int64_t totBlocks=0;
BRM::VER_t ver = 0xFFFF;
u_int64_t totBlocks = 0;
void* thr_client(void* clientArgs) {
blockCacheClient bc(BRP);
uint64_t bfound=0;
uint64_t bnfound=0;
uint64_t rfound=0;
uint64_t rnfound=0;
uint64_t rangeOpCount=0;
uint64_t blockOpCount=0;
uint64_t noOpCount=0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate=0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
void* thr_client(void* clientArgs)
{
blockCacheClient bc(BRP);
uint64_t bfound = 0;
uint64_t bnfound = 0;
uint64_t rfound = 0;
uint64_t rnfound = 0;
uint64_t rangeOpCount = 0;
uint64_t blockOpCount = 0;
uint64_t noOpCount = 0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate = 0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
gettimeofday(&tv, NULL);
int j=0;
int s=0;
uint8_t fb[8192]={0};
LBIDRange_v& r=range_thr[0];
for(int idx=0; idx<fLoops; idx++) {
for (int jdx=0; jdx<range_thr.size(); jdx++) {
uint32_t lbid=0;
bool b;
int ret=0;
r = range_thr[jdx];
for(int l=0; l<r.size(); l++) {
for(int m=r[l].start;m<r[l].start+r[l].size; m++) {
//ret=bc.getBlock(m, ver, &fb, false, b);
ret=bc.read(m, ver, &fb);
if (ret) {
bfound++;
} else {
bnfound++;
}
}
}
}
}
gettimeofday(&tv, NULL);
int j = 0;
int s = 0;
uint8_t fb[8192] = {0};
LBIDRange_v& r = range_thr[0];
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
for (int idx = 0; idx < fLoops; idx++)
{
for (int jdx = 0; jdx < range_thr.size(); jdx++)
{
uint32_t lbid = 0;
bool b;
int ret = 0;
r = range_thr[jdx];
if (elTime>0) {
avgTot=(bfound+rfound)/elTime;
rangeAvg=(rfound)/elTime;
blkAvg=(bfound)/elTime;
} else {
avgTot=bfound+rfound;
rangeAvg=rfound;
blkAvg=bfound;
}
for (int l = 0; l < r.size(); l++)
{
for (int m = r[l].start; m < r[l].start + r[l].size; m++)
{
//ret=bc.getBlock(m, ver, &fb, false, b);
ret = bc.read(m, ver, &fb);
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
if (ret)
{
bfound++;
}
else
{
bnfound++;
}
}
}
}
}
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot+=bfound;
bnfoundTot+=bnfound;
rfoundTot+=rfound;
rnfoundTot+=rnfound;
rangeOpCountTot+=rangeOpCount;
blockOpCountTot+=blockOpCount;
noOpCountTot+=noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfound + rfound) / elTime;
rangeAvg = (rfound) / elTime;
blkAvg = (bfound) / elTime;
}
else
{
avgTot = bfound + rfound;
rangeAvg = rfound;
blkAvg = bfound;
}
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot += bfound;
bnfoundTot += bnfound;
rfoundTot += rfound;
rnfoundTot += rnfound;
rangeOpCountTot += rangeOpCount;
blockOpCountTot += blockOpCount;
noOpCountTot += noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
return NULL;
return NULL;
} // end thr_client
void LoadRange(const LBIDRange_v& v, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
blockCacheClient bc(BRP);
uint32_t rCount=0;
for (uint32_t i =0; i<v.size() ; i++)
{
const InlineLBIDRange r={v[i].start, v[i].size};
if (r.size<=1024) {
bc.check(r, ver, rCount );
loadCount+=rCount;
}
rCount=0;
uint32_t rCount = 0;
}
for (uint32_t i = 0; i < v.size() ; i++)
{
const InlineLBIDRange r = {v[i].start, v[i].size};
if (r.size <= 1024)
{
bc.check(r, ver, rCount );
loadCount += rCount;
}
rCount = 0;
}
}
void ReadRange(const LBIDRange_v& v)
{
blockCacheClient bc(BRP);
int found=0;
int notfound=0;
int ret=0;
for(uint32_t i=0; i<v.size(); i++)
{
const InlineLBIDRange r={v[i].start, v[i].size};
FileBuffer fb(-1, -1);
for(int j=r.start; j<r.start+r.size; j++)
{
if (r.size > 1024)
continue;
ret=bc.read(j, ver, fb);
if (ret)
found++;
else
notfound++;
ret=0;
}
totBlocks+=found;
totBlocks+=notfound;
found=0;
notfound=0;
}
blockCacheClient bc(BRP);
int found = 0;
int notfound = 0;
int ret = 0;
for (uint32_t i = 0; i < v.size(); i++)
{
const InlineLBIDRange r = {v[i].start, v[i].size};
FileBuffer fb(-1, -1);
for (int j = r.start; j < r.start + r.size; j++)
{
if (r.size > 1024)
continue;
ret = bc.read(j, ver, fb);
if (ret)
found++;
else
notfound++;
ret = 0;
}
totBlocks += found;
totBlocks += notfound;
found = 0;
notfound = 0;
}
}
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
}
void ReadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
static int found=0, notfound=0;
uint8_t d[8192];
blockCacheClient bc(BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if (ret)
found++;
else
notfound++;
static int found = 0, notfound = 0;
uint8_t d[8192];
blockCacheClient bc(BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if ((found+notfound)%10000==0)
cout << "found " << found << " notfound " << notfound << endl;
if (ret)
found++;
else
notfound++;
if ((found + notfound) % 10000 == 0)
cout << "found " << found << " notfound " << notfound << endl;
}
//
int main(int argc, char *argv[]) {
int main(int argc, char* argv[])
{
if (argc>=2) thr_cnt=atoi(argv[1]);
if (argc>=3) fLoops=atoi(argv[2]);
if (thr_cnt<=0) thr_cnt=1;
if (thr_cnt>1024) thr_cnt=1024;
if (fLoops<=0) fLoops=1;
if (argc >= 2) thr_cnt = atoi(argv[1]);
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
if (argc >= 3) fLoops = atoi(argv[2]);
cout << "Starting " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid=3000;
uint32_t totalExt=0;
do {
int ret = dbrm.lookup(oid, r);
if (ret==0 && r.size() > 0) {
lowlbid = (r[0].start/extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start=lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx=0; idx<r.size(); idx++)
totalExt+=r[idx].size;
ranges.push_back(r);
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
if (thr_cnt <= 0) thr_cnt = 1;
cout << ranges.size() << " ranges found" << endl;
if (thr_cnt > 1024) thr_cnt = 1024;
gettimeofday(&tv, NULL);
uint32_t blksLoaded=0;
int rangesLoaded=0;
for (uint32_t i =0; i<ranges.size() && blksLoaded < cacheSize; i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
}
cout << endl;
if (fLoops <= 0) fLoops = 1;
gettimeofday(&tv2, NULL);
cout << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec <<endl;
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
while (ranges.size() > rangesLoaded) ranges.pop_back();
cout << "Starting " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid = 3000;
uint32_t totalExt = 0;
do
{
int ret = dbrm.lookup(oid, r);
if (ret == 0 && r.size() > 0)
{
lowlbid = (r[0].start / extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start = lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx = 0; idx < r.size(); idx++)
totalExt += r[idx].size;
ranges.push_back(r);
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
cout << ranges.size() << " ranges found" << endl;
gettimeofday(&tv, NULL);
uint32_t blksLoaded = 0;
int rangesLoaded = 0;
for (uint32_t i = 0; i < ranges.size() && blksLoaded < cacheSize; i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
}
cout << endl;
gettimeofday(&tv2, NULL);
cout << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec << endl;
while (ranges.size() > rangesLoaded) ranges.pop_back();
#ifdef BLAH
for (uint32_t i =0; i<ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i =0; i<ranges.size(); i++)
{
LBIDRange_v rv=ranges[i];
for(uint32_t j=0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for(uint32_t k=l.start; k<l.start+l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
for (uint32_t i = 0; i < ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i = 0; i < ranges.size(); i++)
{
LBIDRange_v rv = ranges[i];
for (uint32_t j = 0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for (uint32_t k = l.start; k < l.start + l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
#endif
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait={0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
//start threads running
cout << "Starting driver threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt*(sizeof(pthread_t)));
for(int i=0; i<thr_cnt; i++) {
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
while (thr_wait.predicate>0) {
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
//start threads running
cout << "Starting driver threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt * (sizeof(pthread_t)));
// join threads back to main
for(int i=0; i<thr_cnt; i++) {
pthread_join(thr_id[i], NULL);
}
for (int i = 0; i < thr_cnt; i++)
{
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
if (elTime>0) {
avgTot=(bfoundTot+rfoundTot)/elTime;
rangeAvg=(rfoundTot)/elTime;
blkAvg=(bfoundTot)/elTime;
} else {
avgTot=bfoundTot+rfoundTot;
rangeAvg=rfoundTot;
blkAvg=bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot/thr_cnt << endl << endl;
while (thr_wait.predicate > 0)
{
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
// join threads back to main
for (int i = 0; i < thr_cnt; i++)
{
pthread_join(thr_id[i], NULL);
}
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfoundTot + rfoundTot) / elTime;
rangeAvg = (rfoundTot) / elTime;
blkAvg = (bfoundTot) / elTime;
}
else
{
avgTot = bfoundTot + rfoundTot;
rangeAvg = rfoundTot;
blkAvg = bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl << endl;
return 0;
return 0;
} // end main

2386
primitives/linux-port/column.cpp
View File

File diff suppressed because it is too large Load Diff

1590
primitives/linux-port/dictionary.cpp
View File

File diff suppressed because it is too large Load Diff

1521
primitives/linux-port/index.cpp
View File

File diff suppressed because it is too large Load Diff

15
primitives/linux-port/primitiveprocessor.cpp
View File

@ -33,17 +33,18 @@ namespace primitives
{
PrimitiveProcessor::PrimitiveProcessor(int debugLevel) :
fDebugLevel(debugLevel), fStatsPtr(NULL), logicalBlockMode(false)
fDebugLevel(debugLevel), fStatsPtr(NULL), logicalBlockMode(false)
{
// This does
// masks[11] = { 0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023 };
int acc, i;
int acc, i;
for (acc = 0, i = 0; i < 11; i++) {
masks[i] = acc;
acc = acc << 1 | 1;
}
for (acc = 0, i = 0; i < 11; i++)
{
masks[i] = acc;
acc = acc << 1 | 1;
}
}
@ -54,7 +55,7 @@ PrimitiveProcessor::~PrimitiveProcessor()
void PrimitiveProcessor::setParsedColumnFilter(boost::shared_ptr<ParsedColumnFilter> pcf)
{
parsedColumnFilter = pcf;
parsedColumnFilter = pcf;
}
ParsedColumnFilter::ParsedColumnFilter() : columnFilterMode(STANDARD), likeOps(0)

394
primitives/linux-port/primitiveprocessor.h
View File

@ -57,28 +57,29 @@ class PrimTest;
namespace primitives
{
enum ColumnFilterMode {
STANDARD,
TWO_ARRAYS,
UNORDERED_SET
enum ColumnFilterMode
{
STANDARD,
TWO_ARRAYS,
UNORDERED_SET
};
class pcfHasher
{
public:
inline size_t operator()(const int64_t i) const
{
return i;
}
public:
inline size_t operator()(const int64_t i) const
{
return i;
}
};
class pcfEqual
{
public:
inline size_t operator()(const int64_t f1, const int64_t f2) const
{
return f1 == f2;
}
public:
inline size_t operator()(const int64_t f1, const int64_t f2) const
{
return f1 == f2;
}
};
typedef std::tr1::unordered_set<int64_t, pcfHasher, pcfEqual> prestored_set_t;
@ -87,41 +88,46 @@ typedef std::tr1::unordered_set<std::string, utils::Hasher> DictEqualityFilter;
struct idb_regex_t
{
#ifdef POSIX_REGEX
regex_t regex;
regex_t regex;
#else
boost::regex regex;
boost::regex regex;
#endif
bool used;
idb_regex_t() : used(false) { }
~idb_regex_t() {
bool used;
idb_regex_t() : used(false) { }
~idb_regex_t()
{
#ifdef POSIX_REGEX
if (used)
regfree(&regex);
if (used)
regfree(&regex);
#endif
}
}
};
struct ParsedColumnFilter {
ColumnFilterMode columnFilterMode;
boost::shared_array<int64_t> prestored_argVals;
boost::shared_array<uint8_t> prestored_cops;
boost::shared_array<uint8_t> prestored_rfs;
boost::shared_ptr<prestored_set_t> prestored_set;
boost::shared_array<idb_regex_t> prestored_regex;
uint8_t likeOps;
struct ParsedColumnFilter
{
ColumnFilterMode columnFilterMode;
boost::shared_array<int64_t> prestored_argVals;
boost::shared_array<uint8_t> prestored_cops;
boost::shared_array<uint8_t> prestored_rfs;
boost::shared_ptr<prestored_set_t> prestored_set;
boost::shared_array<idb_regex_t> prestored_regex;
uint8_t likeOps;
ParsedColumnFilter();
~ParsedColumnFilter();
ParsedColumnFilter();
~ParsedColumnFilter();
};
//@bug 1828 These need to be public so that column operations can use it for 'like'
struct p_DataValue {
int len;
const uint8_t *data;
struct p_DataValue
{
int len;
const uint8_t* data;
};
boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t *filterString,
uint32_t colWidth, uint32_t colType, uint32_t filterCount, uint32_t BOP);
boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t* filterString,
uint32_t colWidth, uint32_t colType, uint32_t filterCount, uint32_t BOP);
/** @brief This class encapsulates the primitive processing functionality of the system.
*
@ -130,190 +136,196 @@ boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t *filterStr
class PrimitiveProcessor
{
public:
PrimitiveProcessor(int debugLevel=0);
virtual ~PrimitiveProcessor();
PrimitiveProcessor(int debugLevel = 0);
virtual ~PrimitiveProcessor();
/** @brief Sets the block to operate on
*
* The primitive processing functions operate on one block at a time. The caller
* sets which block to operate on next with this function.
*/
void setBlockPtr(int *data)
/** @brief Sets the block to operate on
*
* The primitive processing functions operate on one block at a time. The caller
* sets which block to operate on next with this function.
*/
void setBlockPtr(int* data)
{
block = data;
block = data;
}
void setPMStatsPtr(dbbc::Stats* p)
{
fStatsPtr = p;
}
void setPMStatsPtr(dbbc::Stats* p)
{
fStatsPtr=p;
}
/** @brief The interface to Mark's NIOS primitive processing code.
*
* The interface to Mark's NIOS primitive processing code. Instead of reading
* and writing to a bus, it will read/write to buffers specified by inBuf
* and outBuf. The primitives implemented this way are:
* - p_Col and p_ColAggregate
* - p_GetSignature
*
* @param inBuf (in) The buffer containing a command to execute
* @param inLength (in) The size of inBuf in 4-byte words
* @param outBuf (in) The buffer to store the output in
* @param outLength (in) The size of outBuf in 4-byte words
* @param written (out) The number of bytes written to outBuf.
* @note Throws logic_error if the output buffer is too small for the result.
*/
void processBuffer(int *inBuf, unsigned inLength, int *outBuf, unsigned outLength,
unsigned *written);
/** @brief The interface to Mark's NIOS primitive processing code.
*
* The interface to Mark's NIOS primitive processing code. Instead of reading
* and writing to a bus, it will read/write to buffers specified by inBuf
* and outBuf. The primitives implemented this way are:
* - p_Col and p_ColAggregate
* - p_GetSignature
*
* @param inBuf (in) The buffer containing a command to execute
* @param inLength (in) The size of inBuf in 4-byte words
* @param outBuf (in) The buffer to store the output in
* @param outLength (in) The size of outBuf in 4-byte words
* @param written (out) The number of bytes written to outBuf.
* @note Throws logic_error if the output buffer is too small for the result.
*/
void processBuffer(int* inBuf, unsigned inLength, int* outBuf, unsigned outLength,
unsigned* written);
/* Patrick */
/* Patrick */
/** @brief The p_TokenByScan primitive processor
*
* The p_TokenByScan primitive processor. It relies on the caller setting
* the block to operate on with setBlockPtr(). It assumes the continuation
* pointer is not used.
* @param t (in) The arguments to the primitive
* @param out (out) This must point to memory of some currently unknown max size
* @param outSize (in) The size of the output buffer in bytes.
* @note Throws logic_error if the output buffer is too small for the result.
*/
void p_TokenByScan(const TokenByScanRequestHeader *t,
TokenByScanResultHeader *out, unsigned outSize,bool utf8,
boost::shared_ptr<DictEqualityFilter> eqFilter);
/** @brief The p_TokenByScan primitive processor
*
* The p_TokenByScan primitive processor. It relies on the caller setting
* the block to operate on with setBlockPtr(). It assumes the continuation
* pointer is not used.
* @param t (in) The arguments to the primitive
* @param out (out) This must point to memory of some currently unknown max size
* @param outSize (in) The size of the output buffer in bytes.
* @note Throws logic_error if the output buffer is too small for the result.
*/
void p_TokenByScan(const TokenByScanRequestHeader* t,
TokenByScanResultHeader* out, unsigned outSize, bool utf8,
boost::shared_ptr<DictEqualityFilter> eqFilter);
/** @brief The p_IdxWalk primitive processor
*
* The p_IdxWalk primitive processor. The caller must set the block to operate
* on with setBlockPtr(). This primitive can return intermediate results.
* All results returned will have an different LBID than the input. They can
* also be in varying states of completion. A result is final when
* Shift >= SSlen, otherwise it is intermediate and needs to be reissued with
* the specified LBID loaded.
* @note If in->NVALS > 2, new vectors may be returned in the result set, which
* will have to be deleted by the caller. The test to use right now is
* ({element}->NVALS > 2 && {element}->State == 0). If that condition is true,
* delete the vector, otherwise don't. This kludginess is for efficiency's sake
* and may go away for the sake of sanity later.
* @note It is safe to delete any vector passed in after the call.
* @param out The caller should pass in an empty vector. The results
* will be returned as elements of this vector.
*/
void p_IdxWalk(const IndexWalkHeader *in, std::vector<IndexWalkHeader *> *out) throw();
/** @brief The p_IdxWalk primitive processor
*
* The p_IdxWalk primitive processor. The caller must set the block to operate
* on with setBlockPtr(). This primitive can return intermediate results.
* All results returned will have an different LBID than the input. They can
* also be in varying states of completion. A result is final when
* Shift >= SSlen, otherwise it is intermediate and needs to be reissued with
* the specified LBID loaded.
* @note If in->NVALS > 2, new vectors may be returned in the result set, which
* will have to be deleted by the caller. The test to use right now is
* ({element}->NVALS > 2 && {element}->State == 0). If that condition is true,
* delete the vector, otherwise don't. This kludginess is for efficiency's sake
* and may go away for the sake of sanity later.
* @note It is safe to delete any vector passed in after the call.
* @param out The caller should pass in an empty vector. The results
* will be returned as elements of this vector.
*/
void p_IdxWalk(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
/** @brief The p_IdxList primitive processor.
*
* The p_IdxList primitive processor. The caller must set the block to operate
* on with setBlockPtr(). This primitive can return one intermediate result
* for every call made. If there is an intermediate result returned, it will
* be the first element, distinguished by its type field. If the
* first element has a type == RID (3) , there is no intermediate result. If
* the first element had a type == LLP_SUBBLK (4) or type == LLP_BLK (5),
* that element is the intermediate result. Its value field will be a pointer
* to the next section of the list.
*
* @param rqst (in) The request header followed by NVALS IndexWalkParams
* @param rslt (out) The caller passes in a buffer which will be filled
* by the primitive on return. It will consist of an IndexListHeader,
* followed by NVALS IndexListEntrys.
* @param mode (optional, in) 0 specifies old behavior (the last entry of a block might
* be a pointer). 1 specifies new behavior (the last entry should be ignored).
*/
void p_IdxList(const IndexListHeader *rqst, IndexListHeader *rslt, int mode = 1);
/** @brief The p_IdxList primitive processor.
*
* The p_IdxList primitive processor. The caller must set the block to operate
* on with setBlockPtr(). This primitive can return one intermediate result
* for every call made. If there is an intermediate result returned, it will
* be the first element, distinguished by its type field. If the
* first element has a type == RID (3) , there is no intermediate result. If
* the first element had a type == LLP_SUBBLK (4) or type == LLP_BLK (5),
* that element is the intermediate result. Its value field will be a pointer
* to the next section of the list.
*
* @param rqst (in) The request header followed by NVALS IndexWalkParams
* @param rslt (out) The caller passes in a buffer which will be filled
* by the primitive on return. It will consist of an IndexListHeader,
* followed by NVALS IndexListEntrys.
* @param mode (optional, in) 0 specifies old behavior (the last entry of a block might
* be a pointer). 1 specifies new behavior (the last entry should be ignored).
*/
void p_IdxList(const IndexListHeader* rqst, IndexListHeader* rslt, int mode = 1);
/** @brief The p_AggregateSignature primitive processor.
*
* The p_AggregateSignature primitive processor. It operates on a dictionary
* block and assumes the continuation pointer is not used.
* @param in The input parameters
* @param out A pointer to a buffer where the result will be written.
* @param outSize The size of the output buffer in bytes.
* @param written (out parameter) A pointer to 1 int, which will contain the
* number of bytes written to out.
*/
void p_AggregateSignature(const AggregateSignatureRequestHeader *in,
AggregateSignatureResultHeader *out, unsigned outSize, unsigned *written, bool utf8);
/** @brief The p_AggregateSignature primitive processor.
*
* The p_AggregateSignature primitive processor. It operates on a dictionary
* block and assumes the continuation pointer is not used.
* @param in The input parameters
* @param out A pointer to a buffer where the result will be written.
* @param outSize The size of the output buffer in bytes.
* @param written (out parameter) A pointer to 1 int, which will contain the
* number of bytes written to out.
*/
void p_AggregateSignature(const AggregateSignatureRequestHeader* in,
AggregateSignatureResultHeader* out, unsigned outSize, unsigned* written, bool utf8);
/** @brief The p_Col primitive processor.
*
* The p_Col primitive processor. It operates on a column block specified using setBlockPtr().
* @param in The buffer containing the command parameters.
* The buffer should begin with a NewColRequestHeader structure, followed by
* an array of 'NOPS' defining the filter to apply (optional),
* followed by an array of RIDs to apply the filter to (optional).
* @param out The buffer that will contain the results. On return, it will start with
* a NewColResultHeader, followed by the output type specified by in->OutputType.
* \li If OT_RID, it will be an array of RIDs
* \li If OT_DATAVALUE, it will be an array of matching data values stored in the column
* \li If OT_BOTH, it will be an array of <DataValue, RID> pairs
* @param outSize The size of the output buffer in bytes.
* @param written (out parameter) A pointer to 1 int, which will contain the
* number of bytes written to out.
* @note See PrimitiveMsg.h for the type definitions.
*/
void p_Col(NewColRequestHeader *in, NewColResultHeader *out, unsigned outSize,
unsigned *written);
/** @brief The p_Col primitive processor.
*
* The p_Col primitive processor. It operates on a column block specified using setBlockPtr().
* @param in The buffer containing the command parameters.
* The buffer should begin with a NewColRequestHeader structure, followed by
* an array of 'NOPS' defining the filter to apply (optional),
* followed by an array of RIDs to apply the filter to (optional).
* @param out The buffer that will contain the results. On return, it will start with
* a NewColResultHeader, followed by the output type specified by in->OutputType.
* \li If OT_RID, it will be an array of RIDs
* \li If OT_DATAVALUE, it will be an array of matching data values stored in the column
* \li If OT_BOTH, it will be an array of <DataValue, RID> pairs
* @param outSize The size of the output buffer in bytes.
* @param written (out parameter) A pointer to 1 int, which will contain the
* number of bytes written to out.
* @note See PrimitiveMsg.h for the type definitions.
*/
void p_Col(NewColRequestHeader* in, NewColResultHeader* out, unsigned outSize,
unsigned* written);
boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t *filterString,
uint32_t colWidth, uint32_t colType, uint32_t filterCount, uint32_t BOP);
void setParsedColumnFilter(boost::shared_ptr<ParsedColumnFilter>);
boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t* filterString,
uint32_t colWidth, uint32_t colType, uint32_t filterCount, uint32_t BOP);
void setParsedColumnFilter(boost::shared_ptr<ParsedColumnFilter>);
/** @brief The p_ColAggregate primitive processor.
*
* The p_ColAggregate primitive processor. It operates on a column block
* specified using setBlockPtr().
* @param in The buffer containing the command parameters. The buffer should begin
* with a NewColAggRequestHeader, followed by an array of RIDs to generate
* the data for (optional).
* @param out The buffer to put the result in. On return, it will contain a
* NewCollAggResultHeader.
* @note See PrimitiveMsg.h for the type definitions.
*/
/** @brief The p_ColAggregate primitive processor.
*
* The p_ColAggregate primitive processor. It operates on a column block
* specified using setBlockPtr().
* @param in The buffer containing the command parameters. The buffer should begin
* with a NewColAggRequestHeader, followed by an array of RIDs to generate
* the data for (optional).
* @param out The buffer to put the result in. On return, it will contain a
* NewCollAggResultHeader.
* @note See PrimitiveMsg.h for the type definitions.
*/
// void p_ColAggregate(const NewColAggRequestHeader *in, NewColAggResultHeader *out);
void p_Dictionary(const DictInput *in, std::vector<uint8_t> *out, bool utf8,
bool skipNulls, boost::shared_ptr<DictEqualityFilter> eqFilter,
uint8_t eqOp);
void p_Dictionary(const DictInput* in, std::vector<uint8_t>* out, bool utf8,
bool skipNulls, boost::shared_ptr<DictEqualityFilter> eqFilter,
uint8_t eqOp);
inline void setLogicalBlockMode(bool b) { logicalBlockMode = b; }
inline void setLogicalBlockMode(bool b)
{
logicalBlockMode = b;
}
static int convertToRegexp(idb_regex_t *regex, const p_DataValue *str);
inline static bool isEscapedChar(char c);
boost::shared_array<idb_regex_t> makeLikeFilter(const DictFilterElement *inputMsg, uint32_t count);
void setLikeFilter(boost::shared_array<idb_regex_t> filter) { parsedLikeFilter = filter; }
static int convertToRegexp(idb_regex_t* regex, const p_DataValue* str);
inline static bool isEscapedChar(char c);
boost::shared_array<idb_regex_t> makeLikeFilter(const DictFilterElement* inputMsg, uint32_t count);
void setLikeFilter(boost::shared_array<idb_regex_t> filter)
{
parsedLikeFilter = filter;
}
private:
PrimitiveProcessor(const PrimitiveProcessor& rhs);
PrimitiveProcessor& operator=(const PrimitiveProcessor& rhs);
PrimitiveProcessor(const PrimitiveProcessor& rhs);
PrimitiveProcessor& operator=(const PrimitiveProcessor& rhs);
int *block;
int* block;
bool compare(int cmpResult, uint8_t COP, int len1, int len2) throw();
int compare(int val1, int val2, uint8_t COP, bool lastStage) throw();
void indexWalk_1(const IndexWalkHeader *in, std::vector<IndexWalkHeader *> *out) throw();
void indexWalk_2(const IndexWalkHeader *in, std::vector<IndexWalkHeader *> *out) throw();
void indexWalk_many(const IndexWalkHeader *in, std::vector<IndexWalkHeader *> *out) throw();
void grabSubTree(const IndexWalkHeader *in, std::vector<IndexWalkHeader *> *out) throw();
bool compare(int cmpResult, uint8_t COP, int len1, int len2) throw();
int compare(int val1, int val2, uint8_t COP, bool lastStage) throw();
void indexWalk_1(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
void indexWalk_2(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
void indexWalk_many(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
void grabSubTree(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
void nextSig(int NVALS, const PrimToken *tokens, p_DataValue *ret,
uint8_t outputFlags = 0, bool oldGetSigBehavior = false, bool skipNulls = false) throw();
bool isLike(const p_DataValue *dict, const idb_regex_t *arg) throw();
void nextSig(int NVALS, const PrimToken* tokens, p_DataValue* ret,
uint8_t outputFlags = 0, bool oldGetSigBehavior = false, bool skipNulls = false) throw();
bool isLike(const p_DataValue* dict, const idb_regex_t* arg) throw();
// void do_sum8(NewColAggResultHeader *out, int64_t val);
// void do_unsignedsum8(NewColAggResultHeader *out, int64_t val);
uint64_t masks[11];
int dict_OffsetIndex, currentOffsetIndex; // used by p_dictionary
int fDebugLevel;
dbbc::Stats* fStatsPtr; // pointer for pmstats
bool logicalBlockMode;
uint64_t masks[11];
int dict_OffsetIndex, currentOffsetIndex; // used by p_dictionary
int fDebugLevel;
dbbc::Stats* fStatsPtr; // pointer for pmstats
bool logicalBlockMode;
boost::shared_ptr<ParsedColumnFilter> parsedColumnFilter;
boost::shared_array<idb_regex_t> parsedLikeFilter;
boost::shared_ptr<ParsedColumnFilter> parsedColumnFilter;
boost::shared_array<idb_regex_t> parsedLikeFilter;
friend class ::PrimTest;
friend class ::PrimTest;
};
} //namespace primitives

102
primitives/linux-port/print_dictblock.cpp
View File

@ -36,69 +36,77 @@
using namespace std;
void usage(char *name)
void usage(char* name)
{
cerr << "Usage: " << name << " dict_block_filename" << endl;
exit(0);
cerr << "Usage: " << name << " dict_block_filename" << endl;
exit(0);
}
void parseDictBlock(char *block)
void parseDictBlock(char* block)
{
uint16_t *offsets;
uint16_t *freeBytes;
u_int64_t *contPtr;
int offsetIndex, size;
char sig[BLOCK_SIZE+1];
uint16_t* offsets;
uint16_t* freeBytes;
u_int64_t* contPtr;
int offsetIndex, size;
char sig[BLOCK_SIZE + 1];
freeBytes = reinterpret_cast<uint16_t *>(&block[0]);
contPtr = reinterpret_cast<u_int64_t *>(&block[2]);
offsets = reinterpret_cast<uint16_t *>(&block[10]);
freeBytes = reinterpret_cast<uint16_t*>(&block[0]);
contPtr = reinterpret_cast<u_int64_t*>(&block[2]);
offsets = reinterpret_cast<uint16_t*>(&block[10]);
cout << "Free Bytes: " << *freeBytes << endl;
cout << "Continuation Pointer: 0x" << hex << *contPtr << dec << endl;
for (offsetIndex = 0; offsets[offsetIndex+1] != 0xffff; offsetIndex++) {
size = offsets[offsetIndex] - offsets[offsetIndex+1];
memcpy(sig, &block[offsets[offsetIndex+1]], size);
sig[size] = '\0';
cout << "Offset #" << offsetIndex + 1 << ": size=" << size << " offset="
<< offsets[offsetIndex+1] << endl;
cout << "Free Bytes: " << *freeBytes << endl;
cout << "Continuation Pointer: 0x" << hex << *contPtr << dec << endl;
for (offsetIndex = 0; offsets[offsetIndex + 1] != 0xffff; offsetIndex++)
{
size = offsets[offsetIndex] - offsets[offsetIndex + 1];
memcpy(sig, &block[offsets[offsetIndex + 1]], size);
sig[size] = '\0';
cout << "Offset #" << offsetIndex + 1 << ": size=" << size << " offset="
<< offsets[offsetIndex + 1] << endl;
// Use these lines instead for non-ascii data.
// cout << " Signature: 0x";
// for (i = 0; i < size; i++)
// cout << hex << (((int) sig[i]) & 0xff);
cout << " Signature: " << sig;
cout << dec << endl;
}
cout << " Signature: " << sig;
cout << dec << endl;
}
}
int main(int argc, char **argv)
int main(int argc, char** argv)
{
int fd, err;
char buf[BLOCK_SIZE];
int fd, err;
char buf[BLOCK_SIZE];
if (argc != 2)
usage(argv[0]);
if (argc != 2)
usage(argv[0]);
fd = open(argv[1], O_RDONLY);
if (fd < 0) {
perror("open");
exit(1);
}
err = read(fd, buf, BLOCK_SIZE);
if (err < 0) {
perror("read");
exit(1);
}
if (err != BLOCK_SIZE) {
cerr << "Failed to read the file in one op, check the filelength and try again."
<< endl;
exit(1);
}
parseDictBlock(buf);
exit(0);
fd = open(argv[1], O_RDONLY);
if (fd < 0)
{
perror("open");
exit(1);
}
err = read(fd, buf, BLOCK_SIZE);
if (err < 0)
{
perror("read");
exit(1);
}
if (err != BLOCK_SIZE)
{
cerr << "Failed to read the file in one op, check the filelength and try again."
<< endl;
exit(1);
}
parseDictBlock(buf);
exit(0);
}

221
primitives/linux-port/print_indexlist.cpp
View File

@ -41,109 +41,134 @@ using namespace std;
void usage()
{
cout << "Usage: print_indexlist filename <S> block_offset subblock sbentry" << endl;
cout << " Where S=0 means print the whole block, S=1 means print the subblock" << endl;
cout << " Where subblock_number indicates which subblock to print." << endl;
cout << " Where sbentry indicates the first entry to print" << endl;
exit(1);
cout << "Usage: print_indexlist filename <S> block_offset subblock sbentry" << endl;
cout << " Where S=0 means print the whole block, S=1 means print the subblock" << endl;
cout << " Where subblock_number indicates which subblock to print." << endl;
cout << " Where sbentry indicates the first entry to print" << endl;
exit(1);
}
int main(int argc, char **argv)
int main(int argc, char** argv)
{
char buf[8192];
int s, fd, subblock, byteoffset, i, entries, sbentry;
string filename;
off_t offset;
off_t err;
IndexListEntry *entry;
IndexListParam *ptr;
uint32_t fbo;
char buf[8192];
int s, fd, subblock, byteoffset, i, entries, sbentry;
string filename;
off_t offset;
off_t err;
IndexListEntry* entry;
IndexListParam* ptr;
uint32_t fbo;
if (argc == 1)
usage();
filename = argv[1];
s = atoi(argv[2]);
fbo = strtoul(argv[3], 0, 0);
subblock = atoi(argv[4]);
sbentry = atoi(argv[5]);
if (argc == 1)
usage();
fd = open(filename.c_str(), O_RDONLY);
if (fd < 0) {
perror("open");
exit(1);
}
offset = ((off_t)fbo * BLOCK_SIZE);
filename = argv[1];
s = atoi(argv[2]);
fbo = strtoul(argv[3], 0, 0);
subblock = atoi(argv[4]);
sbentry = atoi(argv[5]);
fd = open(filename.c_str(), O_RDONLY);
if (fd < 0)
{
perror("open");
exit(1);
}
offset = ((off_t)fbo * BLOCK_SIZE);
// cout << "BLOCK_SIZE = " << BLOCK_SIZE << " fbo=" << fbo << ", seeking to offset " << offset << endl;
err = lseek(fd, offset, SEEK_SET);
if (err < 0) {
perror("lseek");
exit(1);
}
err = read(fd, buf, BLOCK_SIZE);
if (err != BLOCK_SIZE) {
if (err < 0)
perror("read");
cerr << "read error." << endl;
exit(1);
}
close(fd);
byteoffset = subblock * 256;
err = lseek(fd, offset, SEEK_SET);
if (err < 0)
{
perror("lseek");
exit(1);
}
err = read(fd, buf, BLOCK_SIZE);
if (err != BLOCK_SIZE)
{
if (err < 0)
perror("read");
cerr << "read error." << endl;
exit(1);
}
close(fd);
byteoffset = subblock * 256;
// cout << "subblock=" << subblock << " byte offset=" << byteoffset << endl;
entry = reinterpret_cast<IndexListEntry *>(&buf[byteoffset]);
if (s == 0 && subblock == 0)
entries = 1024;
else
entries = 32;
for (i = sbentry; i < entries; i++) {
cout << i << ": ";
switch (entry[i].type) {
case LLP_SUBBLK:
ptr = reinterpret_cast<IndexListParam *>(&entry[i]);
cout << "Subblock pointer. Rid count=" << entry[i].ridCt <<
" LBID=" << ptr->fbo << " subblock=" << ptr->sbid <<
" SBentry=" << ptr->entry << endl;
break;
case LLP_BLK:
ptr = reinterpret_cast<IndexListParam *>(&entry[i]);
cout << "Block pointer. Rid count=" << entry[i].ridCt <<
" LBID=" << ptr->fbo << " subblock=" <<
ptr->sbid << " SBentry=" << ptr->entry << endl;
break;
case RID:
cout << "RID: " << entry[i].value << endl;
break;
case LIST_SIZE:
if (i == sbentry) {
u_int64_t *val = reinterpret_cast<u_int64_t *>(&entry[i+1]);
cout << "List Header. Rid count=" << entry[i].value;
cout << " key value=0x" << hex << *val << dec << endl;
i++;
}
else if (i + 1 < entries) {
u_int64_t *val = reinterpret_cast<u_int64_t *>(&entry[i+1]);
cout << "List Size entry. Rid count=" << entry[i].value
<< " (if a header) value=0x" << hex << *val << dec << endl;
}
else
cout << "List Size entry. Rid count=" << entry[i].value << endl;
break;
case NOT_IN_USE:
cout << "Not in use (ignored by p_idxlist)" << endl;
break;
case EMPTY_LIST_PTR:
cout << "Empty List Pointer (?) (ignored by p_idxlist)" << endl;
break;
case EMPTY_PTR:
cout << "Empty Pointer entry (?) (ignored by p_idxlist)" << endl;
break;
default:
cout << "Unknown entry type (" << entry[i].type << ")" << endl;
break;
}
}
return 0;
entry = reinterpret_cast<IndexListEntry*>(&buf[byteoffset]);
if (s == 0 && subblock == 0)
entries = 1024;
else
entries = 32;
for (i = sbentry; i < entries; i++)
{
cout << i << ": ";
switch (entry[i].type)
{
case LLP_SUBBLK:
ptr = reinterpret_cast<IndexListParam*>(&entry[i]);
cout << "Subblock pointer. Rid count=" << entry[i].ridCt <<
" LBID=" << ptr->fbo << " subblock=" << ptr->sbid <<
" SBentry=" << ptr->entry << endl;
break;
case LLP_BLK:
ptr = reinterpret_cast<IndexListParam*>(&entry[i]);
cout << "Block pointer. Rid count=" << entry[i].ridCt <<
" LBID=" << ptr->fbo << " subblock=" <<
ptr->sbid << " SBentry=" << ptr->entry << endl;
break;
case RID:
cout << "RID: " << entry[i].value << endl;
break;
case LIST_SIZE:
if (i == sbentry)
{
u_int64_t* val = reinterpret_cast<u_int64_t*>(&entry[i + 1]);
cout << "List Header. Rid count=" << entry[i].value;
cout << " key value=0x" << hex << *val << dec << endl;
i++;
}
else if (i + 1 < entries)
{
u_int64_t* val = reinterpret_cast<u_int64_t*>(&entry[i + 1]);
cout << "List Size entry. Rid count=" << entry[i].value
<< " (if a header) value=0x" << hex << *val << dec << endl;
}
else
cout << "List Size entry. Rid count=" << entry[i].value << endl;
break;
case NOT_IN_USE:
cout << "Not in use (ignored by p_idxlist)" << endl;
break;
case EMPTY_LIST_PTR:
cout << "Empty List Pointer (?) (ignored by p_idxlist)" << endl;
break;
case EMPTY_PTR:
cout << "Empty Pointer entry (?) (ignored by p_idxlist)" << endl;
break;
default:
cout << "Unknown entry type (" << entry[i].type << ")" << endl;
break;
}
}
return 0;
}

78
primitives/linux-port/print_indextree_subblock.cpp
View File

@ -38,48 +38,54 @@ using namespace std;
void usage()
{
cout << "Usage: print_indextree_subblock filename block_offset subblock_number" << endl;
exit(1);
cout << "Usage: print_indextree_subblock filename block_offset subblock_number" << endl;
exit(1);
}
int main(int argc, char **argv)
int main(int argc, char** argv)
{
char buf[256];
int fd, err, subblock, fbo, byteoffset, i;
string filename;
WriteEngine::IdxBitTestEntry *entry;
char buf[256];
int fd, err, subblock, fbo, byteoffset, i;
string filename;
WriteEngine::IdxBitTestEntry* entry;
if (argc != 4)
usage();
filename = argv[1];
fbo = atoi(argv[2]);
subblock = atoi(argv[3]);
if (argc != 4)
usage();
cout << "FBO: " << fbo << " Subblock: " << subblock << endl;
fd = open(filename.c_str(), O_RDONLY);
if (fd < 0) {
perror("open");
exit(1);
}
filename = argv[1];
fbo = atoi(argv[2]);
subblock = atoi(argv[3]);
byteoffset = fbo * BLOCK_SIZE + subblock * 256;
lseek(fd, byteoffset, SEEK_SET);
err = read(fd, buf, 256);
if (err != 256) {
perror("read");
exit(1);
}
close(fd);
cout << "FBO: " << fbo << " Subblock: " << subblock << endl;
fd = open(filename.c_str(), O_RDONLY);
for (i = 0, byteoffset = 0; byteoffset < 256;
byteoffset += sizeof(WriteEngine::IdxBitTestEntry), i++) {
entry = (WriteEngine::IdxBitTestEntry *) &buf[byteoffset];
cout << "Entry " << i << ": fbo=" << (int)entry->fbo <<
" sbid=" << entry->sbid << " sbentry=" << entry->entry <<
" group=" << entry->group << " bittest=" << entry->bitTest <<
" type=" << entry->type << endl;
}
if (fd < 0)
{
perror("open");
exit(1);
}
exit(0);
byteoffset = fbo * BLOCK_SIZE + subblock * 256;
lseek(fd, byteoffset, SEEK_SET);
err = read(fd, buf, 256);
if (err != 256)
{
perror("read");
exit(1);
}
close(fd);
for (i = 0, byteoffset = 0; byteoffset < 256;
byteoffset += sizeof(WriteEngine::IdxBitTestEntry), i++)
{
entry = (WriteEngine::IdxBitTestEntry*) &buf[byteoffset];
cout << "Entry " << i << ": fbo=" << (int)entry->fbo <<
" sbid=" << entry->sbid << " sbentry=" << entry->entry <<
" group=" << entry->group << " bittest=" << entry->bitTest <<
" type=" << entry->type << endl;
}
exit(0);
}

8167
primitives/linux-port/tdriver.cpp
View File

File diff suppressed because it is too large Load Diff

3893
primitives/primproc/batchprimitiveprocessor.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

510
primitives/primproc/batchprimitiveprocessor.h Executable file → Normal file
View File

@ -65,300 +65,340 @@ namespace primitiveprocessor
typedef boost::shared_ptr<BatchPrimitiveProcessor> SBPP;
class scalar_exception : public std::exception {
const char * what() const throw() { return "Not a scalar subquery."; }
class scalar_exception : public std::exception
{
const char* what() const throw()
{
return "Not a scalar subquery.";
}
};
class NeedToRestartJob : public std::runtime_error
{
public:
NeedToRestartJob() : std::runtime_error("NeedToRestartJob") { }
NeedToRestartJob(const std::string &s) :
std::runtime_error(s) { }
public:
NeedToRestartJob() : std::runtime_error("NeedToRestartJob") { }
NeedToRestartJob(const std::string& s) :
std::runtime_error(s) { }
};
class BatchPrimitiveProcessor
{
public:
BatchPrimitiveProcessor(messageqcpp::ByteStream &, double prefetchThresh,
boost::shared_ptr<BPPSendThread>);
public:
BatchPrimitiveProcessor(messageqcpp::ByteStream&, double prefetchThresh,
boost::shared_ptr<BPPSendThread>);
~BatchPrimitiveProcessor();
~BatchPrimitiveProcessor();
/* Interface used by primproc */
void initBPP(messageqcpp::ByteStream &);
void resetBPP(messageqcpp::ByteStream &, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& outputSock);
void addToJoiner(messageqcpp::ByteStream &);
int endOfJoiner();
int operator()();
void setLBIDForScan(uint64_t rid);
/* Interface used by primproc */
void initBPP(messageqcpp::ByteStream&);
void resetBPP(messageqcpp::ByteStream&, const SP_UM_MUTEX& wLock, const SP_UM_IOSOCK& outputSock);
void addToJoiner(messageqcpp::ByteStream&);
int endOfJoiner();
int operator()();
void setLBIDForScan(uint64_t rid);
/* Duplicate() returns a deep copy of this object as it was init'd by initBPP.
It's thread-safe wrt resetBPP. */
SBPP duplicate();
/* Duplicate() returns a deep copy of this object as it was init'd by initBPP.
It's thread-safe wrt resetBPP. */
SBPP duplicate();
/* These need to be updated */
//bool operator==(const BatchPrimitiveProcessor&) const;
//inline bool operator!=(const BatchPrimitiveProcessor& bpp) const
//{
// return !(*this == bpp);
//}
/* These need to be updated */
//bool operator==(const BatchPrimitiveProcessor&) const;
//inline bool operator!=(const BatchPrimitiveProcessor& bpp) const
//{
// return !(*this == bpp);
//}
inline uint32_t getSessionID() { return sessionID; }
inline uint32_t getStepID() { return stepID; }
inline uint32_t getUniqueID() { return uniqueID; }
inline bool busy() { return fBusy; }
inline void busy(bool b) { fBusy = b; }
inline uint32_t getSessionID()
{
return sessionID;
}
inline uint32_t getStepID()
{
return stepID;
}
inline uint32_t getUniqueID()
{
return uniqueID;
}
inline bool busy()
{
return fBusy;
}
inline void busy(bool b)
{
fBusy = b;
}
uint16_t FilterCount() const {return filterCount;}
uint16_t ProjectCount() const {return projectCount;}
uint32_t PhysIOCount() const { return physIO;}
uint32_t CachedIOCount() const { return cachedIO;}
uint32_t BlocksTouchedCount() const { return touchedBlocks;}
uint16_t FilterCount() const
{
return filterCount;
}
uint16_t ProjectCount() const
{
return projectCount;
}
uint32_t PhysIOCount() const
{
return physIO;
}
uint32_t CachedIOCount() const
{
return cachedIO;
}
uint32_t BlocksTouchedCount() const
{
return touchedBlocks;
}
void setError(const std::string& error, logging::ErrorCodeValues errorCode) {}
void setError(const std::string& error, logging::ErrorCodeValues errorCode) {}
// these two functions are used by BPPV to create BPP instances
// on demand. TRY not to use unlock() for anything else.
void unlock() { pthread_mutex_unlock(&objLock); }
bool hasJoin() { return doJoin; }
private:
BatchPrimitiveProcessor();
BatchPrimitiveProcessor(const BatchPrimitiveProcessor &);
BatchPrimitiveProcessor& operator=(const BatchPrimitiveProcessor &);
// these two functions are used by BPPV to create BPP instances
// on demand. TRY not to use unlock() for anything else.
void unlock()
{
pthread_mutex_unlock(&objLock);
}
bool hasJoin()
{
return doJoin;
}
private:
BatchPrimitiveProcessor();
BatchPrimitiveProcessor(const BatchPrimitiveProcessor&);
BatchPrimitiveProcessor& operator=(const BatchPrimitiveProcessor&);
void initProcessor();
void initProcessor();
#ifdef PRIMPROC_STOPWATCH
void execute(logging::StopWatch *stopwatch);
void execute(logging::StopWatch* stopwatch);
#else
void execute();
void execute();
#endif
void writeProjectionPreamble();
void makeResponse();
void sendResponse();
void writeProjectionPreamble();
void makeResponse();
void sendResponse();
/* Used by scan operations to increment the LBIDs in successive steps */
void nextLBID();
/* Used by scan operations to increment the LBIDs in successive steps */
void nextLBID();
/* these send relative rids, should this be abs rids? */
void serializeElementTypes();
void serializeStrings();
/* these send relative rids, should this be abs rids? */
void serializeElementTypes();
void serializeStrings();
void asyncLoadProjectColumns();
void writeErrorMsg(const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true);
void asyncLoadProjectColumns();
void writeErrorMsg(const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true);
BPSOutputType ot;
BPSOutputType ot;
BRM::QueryContext versionInfo;
uint32_t txnID;
uint32_t sessionID;
uint32_t stepID;
uint32_t uniqueID;
BRM::QueryContext versionInfo;
uint32_t txnID;
uint32_t sessionID;
uint32_t stepID;
uint32_t uniqueID;
// # of times to loop over the command arrays
// ... This is 1, except when the first command is a scan, in which case
// this single BPP object produces count responses.
uint16_t count;
uint64_t baseRid; // first rid of the logical block
// # of times to loop over the command arrays
// ... This is 1, except when the first command is a scan, in which case
// this single BPP object produces count responses.
uint16_t count;
uint64_t baseRid; // first rid of the logical block
uint16_t relRids[LOGICAL_BLOCK_RIDS];
int64_t values[LOGICAL_BLOCK_RIDS];
boost::scoped_array<uint64_t> absRids;
boost::scoped_array<std::string> strValues;
uint16_t ridCount;
bool needStrValues;
uint16_t relRids[LOGICAL_BLOCK_RIDS];
int64_t values[LOGICAL_BLOCK_RIDS];
boost::scoped_array<uint64_t> absRids;
boost::scoped_array<std::string> strValues;
uint16_t ridCount;
bool needStrValues;
/* Common space for primitive data */
static const uint32_t BUFFER_SIZE = 65536;
uint8_t blockData[BLOCK_SIZE * 8];
boost::scoped_array<uint8_t> outputMsg;
uint32_t outMsgSize;
/* Common space for primitive data */
static const uint32_t BUFFER_SIZE = 65536;
uint8_t blockData[BLOCK_SIZE * 8];
boost::scoped_array<uint8_t> outputMsg;
uint32_t outMsgSize;
std::vector<SCommand> filterSteps;
std::vector<SCommand> projectSteps;
//@bug 1136
uint16_t filterCount;
uint16_t projectCount;
bool sendRidsAtDelivery;
uint8_t ridMap;
bool gotAbsRids;
bool gotValues;
std::vector<SCommand> filterSteps;
std::vector<SCommand> projectSteps;
//@bug 1136
uint16_t filterCount;
uint16_t projectCount;
bool sendRidsAtDelivery;
uint8_t ridMap;
bool gotAbsRids;
bool gotValues;
bool hasScan;
bool validCPData;
int64_t minVal, maxVal; // CP data from a scanned column
uint64_t lbidForCP;
bool hasScan;
bool validCPData;
int64_t minVal, maxVal; // CP data from a scanned column
uint64_t lbidForCP;
// IO counters
boost::mutex counterLock;
uint32_t busyLoaderCount;
// IO counters
boost::mutex counterLock;
uint32_t busyLoaderCount;
uint32_t physIO, cachedIO, touchedBlocks;
uint32_t physIO, cachedIO, touchedBlocks;
SP_UM_IOSOCK sock;
messageqcpp::SBS serialized;
SP_UM_MUTEX writelock;
SP_UM_IOSOCK sock;
messageqcpp::SBS serialized;
SP_UM_MUTEX writelock;
// MCOL-744 using pthread mutex instead of Boost mutex because
// in it is possible that this lock could be unlocked when it is
// already unlocked. In Ubuntu 16.04's Boost this triggers a
// crash. Whilst it is very hard to hit this it is still bad.
// Longer term TODO: fix/remove objLock and/or refactor BPP
pthread_mutex_t objLock;
bool LBIDTrace;
bool fBusy;
// MCOL-744 using pthread mutex instead of Boost mutex because
// in it is possible that this lock could be unlocked when it is
// already unlocked. In Ubuntu 16.04's Boost this triggers a
// crash. Whilst it is very hard to hit this it is still bad.
// Longer term TODO: fix/remove objLock and/or refactor BPP
pthread_mutex_t objLock;
bool LBIDTrace;
bool fBusy;
/* Join support TODO: Make join ops a seperate Command class. */
boost::shared_ptr<joiner::Joiner> joiner;
std::vector<joblist::ElementType> smallSideMatches;
bool doJoin;
uint32_t joinerSize;
uint16_t preJoinRidCount;
boost::mutex addToJoinerLock;
void executeJoin();
/* Join support TODO: Make join ops a seperate Command class. */
boost::shared_ptr<joiner::Joiner> joiner;
std::vector<joblist::ElementType> smallSideMatches;
bool doJoin;
uint32_t joinerSize;
uint16_t preJoinRidCount;
boost::mutex addToJoinerLock;
void executeJoin();
// uint32_t ridsIn, ridsOut;
//@bug 1051 FilterStep on PM
bool hasFilterStep;
bool filtOnString;
boost::scoped_array<uint16_t> fFiltCmdRids[2];
boost::scoped_array<int64_t> fFiltCmdValues[2];
boost::scoped_array<std::string> fFiltStrValues[2];
uint64_t fFiltRidCount[2];
//@bug 1051 FilterStep on PM
bool hasFilterStep;
bool filtOnString;
boost::scoped_array<uint16_t> fFiltCmdRids[2];
boost::scoped_array<int64_t> fFiltCmdValues[2];
boost::scoped_array<std::string> fFiltStrValues[2];
uint64_t fFiltRidCount[2];
// query density threshold for prefetch & async loading
double prefetchThreshold;
// query density threshold for prefetch & async loading
double prefetchThreshold;
/* RowGroup support */
rowgroup::RowGroup outputRG;
boost::scoped_ptr<rowgroup::RGData> outRowGroupData;
boost::shared_array<int> rgMap; // maps input cols to output cols
boost::shared_array<int> projectionMap; // maps the projection steps to the output RG
bool hasRowGroup;
/* RowGroup support */
rowgroup::RowGroup outputRG;
boost::scoped_ptr<rowgroup::RGData> outRowGroupData;
boost::shared_array<int> rgMap; // maps input cols to output cols
boost::shared_array<int> projectionMap; // maps the projection steps to the output RG
bool hasRowGroup;
/* Rowgroups + join */
typedef std::tr1::unordered_multimap<uint64_t, uint32_t,
joiner::TupleJoiner::hasher, std::equal_to<uint64_t>,
utils::SimpleAllocator<std::pair<const uint64_t, uint32_t> > > TJoiner;
/* Rowgroups + join */
typedef std::tr1::unordered_multimap<uint64_t, uint32_t,
joiner::TupleJoiner::hasher, std::equal_to<uint64_t>,
utils::SimpleAllocator<std::pair<const uint64_t, uint32_t> > > TJoiner;
typedef std::tr1::unordered_multimap<joiner::TypelessData,
uint32_t, joiner::TupleJoiner::hasher> TLJoiner;
typedef std::tr1::unordered_multimap<joiner::TypelessData,
uint32_t, joiner::TupleJoiner::hasher> TLJoiner;
bool generateJoinedRowGroup(rowgroup::Row &baseRow, const uint32_t depth = 0);
/* generateJoinedRowGroup helper fcns & vars */
void initGJRG(); // called once after joining
void resetGJRG(); // called after every rowgroup returned by generateJRG
boost::scoped_array<uint32_t> gjrgPlaceHolders;
uint32_t gjrgRowNumber;
bool gjrgFull;
rowgroup::Row largeRow, joinedRow, baseJRow;
boost::scoped_array<uint8_t> baseJRowMem;
boost::scoped_ptr<rowgroup::RGData> joinedRGMem;
boost::scoped_array<rowgroup::Row> smallRows;
boost::shared_array<boost::shared_array<int> > gjrgMappings;
bool generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth = 0);
/* generateJoinedRowGroup helper fcns & vars */
void initGJRG(); // called once after joining
void resetGJRG(); // called after every rowgroup returned by generateJRG
boost::scoped_array<uint32_t> gjrgPlaceHolders;
uint32_t gjrgRowNumber;
bool gjrgFull;
rowgroup::Row largeRow, joinedRow, baseJRow;
boost::scoped_array<uint8_t> baseJRowMem;
boost::scoped_ptr<rowgroup::RGData> joinedRGMem;
boost::scoped_array<rowgroup::Row> smallRows;
boost::shared_array<boost::shared_array<int> > gjrgMappings;
boost::shared_array<boost::shared_ptr<TJoiner> > tJoiners;
typedef std::vector<uint32_t> MatchedData[LOGICAL_BLOCK_RIDS];
boost::shared_array<MatchedData> tSmallSideMatches;
void executeTupleJoin();
bool getTupleJoinRowGroupData;
std::vector<rowgroup::RowGroup> smallSideRGs;
rowgroup::RowGroup largeSideRG;
boost::shared_array<rowgroup::RGData> smallSideRowData;
boost::shared_array<rowgroup::RGData> smallNullRowData;
boost::shared_array<rowgroup::Row::Pointer> smallNullPointers;
boost::shared_array<uint64_t> ssrdPos; // this keeps track of position when building smallSideRowData
boost::shared_array<uint32_t> smallSideRowLengths;
boost::shared_array<joblist::JoinType> joinTypes;
uint32_t joinerCount;
boost::shared_array<uint32_t> tJoinerSizes;
// LSKC[i] = the column in outputRG joiner i uses as its key column
boost::shared_array<uint32_t> largeSideKeyColumns;
// KCPP[i] = true means a joiner uses projection step i as a key column
boost::shared_array<bool> keyColumnProj;
rowgroup::Row oldRow, newRow; // used by executeTupleJoin()
boost::shared_array<uint64_t> joinNullValues;
boost::shared_array<bool> doMatchNulls;
boost::scoped_array<boost::scoped_ptr<funcexp::FuncExpWrapper> > joinFEFilters;
bool hasJoinFEFilters;
bool hasSmallOuterJoin;
boost::shared_array<boost::shared_ptr<TJoiner> > tJoiners;
typedef std::vector<uint32_t> MatchedData[LOGICAL_BLOCK_RIDS];
boost::shared_array<MatchedData> tSmallSideMatches;
void executeTupleJoin();
bool getTupleJoinRowGroupData;
std::vector<rowgroup::RowGroup> smallSideRGs;
rowgroup::RowGroup largeSideRG;
boost::shared_array<rowgroup::RGData> smallSideRowData;
boost::shared_array<rowgroup::RGData> smallNullRowData;
boost::shared_array<rowgroup::Row::Pointer> smallNullPointers;
boost::shared_array<uint64_t> ssrdPos; // this keeps track of position when building smallSideRowData
boost::shared_array<uint32_t> smallSideRowLengths;
boost::shared_array<joblist::JoinType> joinTypes;
uint32_t joinerCount;
boost::shared_array<uint32_t> tJoinerSizes;
// LSKC[i] = the column in outputRG joiner i uses as its key column
boost::shared_array<uint32_t> largeSideKeyColumns;
// KCPP[i] = true means a joiner uses projection step i as a key column
boost::shared_array<bool> keyColumnProj;
rowgroup::Row oldRow, newRow; // used by executeTupleJoin()
boost::shared_array<uint64_t> joinNullValues;
boost::shared_array<bool> doMatchNulls;
boost::scoped_array<boost::scoped_ptr<funcexp::FuncExpWrapper> > joinFEFilters;
bool hasJoinFEFilters;
bool hasSmallOuterJoin;
/* extra typeless join vars & fcns*/
boost::shared_array<bool> typelessJoin;
boost::shared_array<std::vector<uint32_t> > tlLargeSideKeyColumns;
boost::shared_array<boost::shared_ptr<TLJoiner> > tlJoiners;
boost::shared_array<uint32_t> tlKeyLengths;
inline void getJoinResults(const rowgroup::Row &r, uint32_t jIndex, std::vector<uint32_t>& v);
// these allocators hold the memory for the keys stored in tlJoiners
boost::shared_array<utils::PoolAllocator> storedKeyAllocators;
// these allocators hold the memory for the large side keys which are short-lived
boost::scoped_array<utils::FixedAllocator> tmpKeyAllocators;
/* extra typeless join vars & fcns*/
boost::shared_array<bool> typelessJoin;
boost::shared_array<std::vector<uint32_t> > tlLargeSideKeyColumns;
boost::shared_array<boost::shared_ptr<TLJoiner> > tlJoiners;
boost::shared_array<uint32_t> tlKeyLengths;
inline void getJoinResults(const rowgroup::Row& r, uint32_t jIndex, std::vector<uint32_t>& v);
// these allocators hold the memory for the keys stored in tlJoiners
boost::shared_array<utils::PoolAllocator> storedKeyAllocators;
// these allocators hold the memory for the large side keys which are short-lived
boost::scoped_array<utils::FixedAllocator> tmpKeyAllocators;
/* PM Aggregation */
rowgroup::RowGroup joinedRG; // if there's a join, the rows are formatted with this
rowgroup::SP_ROWAGG_PM_t fAggregator;
rowgroup::RowGroup fAggregateRG;
rowgroup::RGData fAggRowGroupData;
//boost::scoped_array<uint8_t> fAggRowGroupData;
boost::shared_array<boost::shared_ptr<utils::SimplePool> > _pools;
/* PM Aggregation */
rowgroup::RowGroup joinedRG; // if there's a join, the rows are formatted with this
rowgroup::SP_ROWAGG_PM_t fAggregator;
rowgroup::RowGroup fAggregateRG;
rowgroup::RGData fAggRowGroupData;
//boost::scoped_array<uint8_t> fAggRowGroupData;
boost::shared_array<boost::shared_ptr<utils::SimplePool> > _pools;
/* OR hacks */
uint8_t bop; // BOP_AND or BOP_OR
bool hasPassThru;
uint8_t forHJ;
/* OR hacks */
uint8_t bop; // BOP_AND or BOP_OR
bool hasPassThru;
uint8_t forHJ;
boost::scoped_ptr<funcexp::FuncExpWrapper> fe1, fe2;
rowgroup::RowGroup fe1Input, fe2Output, *fe2Input;
// note, joinFERG is only for metadata, and is shared between BPPs
boost::shared_ptr<rowgroup::RowGroup> joinFERG;
boost::scoped_array<uint8_t> joinFERowData;
boost::scoped_ptr<rowgroup::RGData> fe1Data, fe2Data; // can probably make these RGDatas not pointers to RGDatas
boost::shared_array<int> projectForFE1;
boost::shared_array<int> fe1ToProjection, fe2Mapping; // RG mappings
boost::scoped_array<boost::shared_array<int> > joinFEMappings;
rowgroup::Row fe1In, fe1Out, fe2In, fe2Out, joinFERow;
boost::scoped_ptr<funcexp::FuncExpWrapper> fe1, fe2;
rowgroup::RowGroup fe1Input, fe2Output, *fe2Input;
// note, joinFERG is only for metadata, and is shared between BPPs
boost::shared_ptr<rowgroup::RowGroup> joinFERG;
boost::scoped_array<uint8_t> joinFERowData;
boost::scoped_ptr<rowgroup::RGData> fe1Data, fe2Data; // can probably make these RGDatas not pointers to RGDatas
boost::shared_array<int> projectForFE1;
boost::shared_array<int> fe1ToProjection, fe2Mapping; // RG mappings
boost::scoped_array<boost::shared_array<int> > joinFEMappings;
rowgroup::Row fe1In, fe1Out, fe2In, fe2Out, joinFERow;
bool hasDictStep;
bool hasDictStep;
primitives::PrimitiveProcessor pp;
primitives::PrimitiveProcessor pp;
/* VSS cache members */
VSSCache vssCache;
void buildVSSCache(uint32_t loopCount);
/* VSS cache members */
VSSCache vssCache;
void buildVSSCache(uint32_t loopCount);
/* To support limited DEC queues on the PM */
boost::shared_ptr<BPPSendThread> sendThread;
bool newConnection; // to support the load balancing code in sendThread
/* To support limited DEC queues on the PM */
boost::shared_ptr<BPPSendThread> sendThread;
bool newConnection; // to support the load balancing code in sendThread
/* To support reentrancy */
uint32_t currentBlockOffset;
boost::scoped_array<uint64_t> relLBID;
boost::scoped_array<bool> asyncLoaded;
/* To support reentrancy */
uint32_t currentBlockOffset;
boost::scoped_array<uint64_t> relLBID;
boost::scoped_array<bool> asyncLoaded;
/* To support a smaller memory footprint when idle */
static const uint64_t maxIdleBufferSize = 16*1024*1024; // arbitrary
void allocLargeBuffers();
void freeLargeBuffers();
/* To support a smaller memory footprint when idle */
static const uint64_t maxIdleBufferSize = 16 * 1024 * 1024; // arbitrary
void allocLargeBuffers();
void freeLargeBuffers();
/* To ensure all packets of an LBID go out the same socket */
int sockIndex;
/* To ensure all packets of an LBID go out the same socket */
int sockIndex;
/* Shared nothing vars */
uint32_t dbRoot;
/* Shared nothing vars */
uint32_t dbRoot;
bool endOfJoinerRan;
bool endOfJoinerRan;
friend class Command;
friend class ColumnCommand;
friend class DictStep;
friend class PassThruCommand;
friend class RTSCommand;
friend class FilterCommand;
friend class ScaledFilterCmd;
friend class StrFilterCmd;
friend class PseudoCC;
friend class Command;
friend class ColumnCommand;
friend class DictStep;
friend class PassThruCommand;
friend class RTSCommand;
friend class FilterCommand;
friend class ScaledFilterCmd;
friend class StrFilterCmd;
friend class PseudoCC;
};
}

510
primitives/primproc/bppseeder.cpp
View File

@ -54,13 +54,20 @@ using namespace std;
namespace primitiveprocessor
{
struct PTLogs{
PTLogs() {};
PTLogs(const int t, const char * fname):thdId(t) {logFD.open(fname, ios_base::app | ios_base::ate);}
~PTLogs() {logFD.close();}
struct PTLogs
{
PTLogs() {};
PTLogs(const int t, const char* fname): thdId(t)
{
logFD.open(fname, ios_base::app | ios_base::ate);
}
~PTLogs()
{
logFD.close();
}
int thdId;
ofstream logFD;
int thdId;
ofstream logFD;
};
typedef PTLogs PTLogs_t;
@ -70,45 +77,48 @@ typedef std::tr1::unordered_map<pthread_t, SPPTLogs_t> PTLogsMap_t;
PTLogsMap_t gFDList;
SPPTLogs_t gLogFD;
boost::mutex gFDMutex; //pthread_mutex_t gFDMutex=PTHREAD_MUTEX_INITIALIZER;
int gThdCnt=0;
int gThdCnt = 0;
extern dbbc::BlockRequestProcessor **BRPp;
extern dbbc::BlockRequestProcessor** BRPp;
extern int fCacheCount;
void timespec_sub(const struct timespec &tv1,
const struct timespec &tv2,
double &tm)
void timespec_sub(const struct timespec& tv1,
const struct timespec& tv2,
double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9*(tv2.tv_nsec - tv1.tv_nsec);
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
BPPSeeder::BPPSeeder(const SBS &b,
const SP_UM_MUTEX& w,
const SP_UM_IOSOCK& s,
const int pmThreads,
const bool trace) :
bs(b), writelock(w), sock(s), fPMThreads(pmThreads), fTrace(trace),
failCount(0),
firstRun(true)
BPPSeeder::BPPSeeder(const SBS& b,
const SP_UM_MUTEX& w,
const SP_UM_IOSOCK& s,
const int pmThreads,
const bool trace) :
bs(b), writelock(w), sock(s), fPMThreads(pmThreads), fTrace(trace),
failCount(0),
firstRun(true)
{
uint8_t *buf = b->buf();
uint32_t pos = sizeof(ISMPacketHeader);
uint8_t* buf = b->buf();
uint32_t pos = sizeof(ISMPacketHeader);
sessionID = *((uint32_t *) &buf[pos]); pos += 4;
stepID = *((uint32_t *) &buf[pos]); pos += 4;
uniqueID = *((uint32_t *) &buf[pos]); pos +=4;
_priority = *((uint32_t *) &buf[pos]);
sessionID = *((uint32_t*) &buf[pos]);
pos += 4;
stepID = *((uint32_t*) &buf[pos]);
pos += 4;
uniqueID = *((uint32_t*) &buf[pos]);
pos += 4;
_priority = *((uint32_t*) &buf[pos]);
dieTime = boost::posix_time::second_clock::universal_time() +
boost::posix_time::seconds(100);
dieTime = boost::posix_time::second_clock::universal_time() +
boost::posix_time::seconds(100);
}
BPPSeeder::BPPSeeder(const BPPSeeder &b)
: bs(b.bs), writelock(b.writelock), sock(b.sock),
fPMThreads(b.fPMThreads), fTrace(b.fTrace), uniqueID(b.uniqueID),
sessionID(b.sessionID), stepID(b.stepID), failCount(b.failCount), bpp(b.bpp),
firstRun(b.firstRun), _priority(b._priority)
BPPSeeder::BPPSeeder(const BPPSeeder& b)
: bs(b.bs), writelock(b.writelock), sock(b.sock),
fPMThreads(b.fPMThreads), fTrace(b.fTrace), uniqueID(b.uniqueID),
sessionID(b.sessionID), stepID(b.stepID), failCount(b.failCount), bpp(b.bpp),
firstRun(b.firstRun), _priority(b._priority)
{
}
@ -118,250 +128,293 @@ BPPSeeder::~BPPSeeder()
int BPPSeeder::operator()()
{
uint32_t pos;
const uint8_t *buf = bs->buf();
BPPMap::iterator it;
ostringstream logData;
struct timespec tm;
struct timespec tm2;
double tm3=0;
bool ptLock=false;
bool gotBPP = false;
PTLogs_t* logFD=NULL;
int ret = 0;
pthread_t tid=0;
boost::mutex::scoped_lock scoped(bppLock, boost::defer_lock_t());
uint32_t pos;
const uint8_t* buf = bs->buf();
BPPMap::iterator it;
ostringstream logData;
struct timespec tm;
struct timespec tm2;
double tm3 = 0;
bool ptLock = false;
bool gotBPP = false;
PTLogs_t* logFD = NULL;
int ret = 0;
pthread_t tid = 0;
boost::mutex::scoped_lock scoped(bppLock, boost::defer_lock_t());
try {
if (firstRun) {
pos = sizeof(ISMPacketHeader) - 2;
uint16_t status = *((uint16_t *) &buf[pos]); pos += 2;
try
{
if (firstRun)
{
pos = sizeof(ISMPacketHeader) - 2;
uint16_t status = *((uint16_t*) &buf[pos]);
pos += 2;
sessionID = *((uint32_t *) &buf[pos]); pos += 4;
stepID = *((uint32_t *) &buf[pos]); pos += 4;
uniqueID = *((uint32_t *) &buf[pos]); pos += 4;
_priority = *((uint32_t *) &buf[pos]);
sessionID = *((uint32_t*) &buf[pos]);
pos += 4;
stepID = *((uint32_t*) &buf[pos]);
pos += 4;
uniqueID = *((uint32_t*) &buf[pos]);
pos += 4;
_priority = *((uint32_t*) &buf[pos]);
if (0 < status)
{
sendErrorMsg(uniqueID, status, stepID);
return ret;
}
if (0 < status)
{
sendErrorMsg(uniqueID, status, stepID);
return ret;
}
//if (!(sessionID & 0x80000000))
//cout << "got request for <" << sessionID <<", " << stepID << ">\n";
scoped.lock();
if (!bppv) {
it = bppMap.find(uniqueID);
if (it == bppMap.end()) {
/* mitigate a small race between creation and use */
scoped.unlock();
if (boost::posix_time::second_clock::universal_time() > dieTime) {
//if (!(sessionID & 0x80000000))
//cout << "got request for <" << sessionID <<", " << stepID << ">\n";
scoped.lock();
if (!bppv)
{
it = bppMap.find(uniqueID);
if (it == bppMap.end())
{
/* mitigate a small race between creation and use */
scoped.unlock();
if (boost::posix_time::second_clock::universal_time() > dieTime)
{
#if 0 // for debugging
#ifndef _MSC_VER
boost::posix_time::ptime pt = boost::posix_time::microsec_clock::local_time();
if (sessionID & 0x80000000)
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< (int) (sessionID ^ 0x80000000) << " stepID=" << stepID << " (syscat)" << pt << endl;
else
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< sessionID << " stepID=" << stepID << pt << endl;
boost::posix_time::ptime pt = boost::posix_time::microsec_clock::local_time();
if (sessionID & 0x80000000)
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< (int) (sessionID ^ 0x80000000) << " stepID=" << stepID << " (syscat)" << pt << endl;
else
cout << "BPPSeeder couldn't find the sessionID/stepID pair. sessionID="
<< sessionID << " stepID=" << stepID << pt << endl;
#endif
throw logic_error("BPPSeeder couldn't find the sessionID/stepID pair");
throw logic_error("BPPSeeder couldn't find the sessionID/stepID pair");
#endif
return 0;
}
return 0;
}
// if (!isSysCat())
return -1;
return -1;
// else { // syscat queries aren't run by a threadpool, can't reschedule those jobs
// usleep(1000);
// goto retry;
// }
}
bppv = it->second;
}
if (bppv->aborted())
return 0;
bpp = bppv->next();
scoped.unlock();
if (!bpp) {
}
bppv = it->second;
}
if (bppv->aborted())
return 0;
bpp = bppv->next();
scoped.unlock();
if (!bpp)
{
// if (isSysCat()) {
// usleep(1000);
// goto retry;
// }
return -1; // all BPP instances are busy, make threadpool reschedule
}
gotBPP = true;
bpp->resetBPP(*bs, writelock, sock);
firstRun = false;
} // firstRun
return -1; // all BPP instances are busy, make threadpool reschedule
}
gotBPP = true;
bpp->resetBPP(*bs, writelock, sock);
firstRun = false;
} // firstRun
if (fTrace)
{
PTLogsMap_t::iterator it;
if (fTrace)
{
PTLogsMap_t::iterator it;
#ifdef _MSC_VER
tid = GetCurrentThreadId();
tid = GetCurrentThreadId();
#else
tid = pthread_self();
tid = pthread_self();
#endif
// only lock map while inserted objects
// once there is an object for each thread
// there is not need to lock
if (gFDList.size()<(uint32_t)fPMThreads) {
gFDMutex.lock();
ptLock=true;
}
// only lock map while inserted objects
// once there is an object for each thread
// there is not need to lock
if (gFDList.size() < (uint32_t)fPMThreads)
{
gFDMutex.lock();
ptLock = true;
}
it = gFDList.find(tid);
if (it==gFDList.end())
{
ostringstream LogFileName;
SPPTLogs_t spof;
it = gFDList.find(tid);
if (it == gFDList.end())
{
ostringstream LogFileName;
SPPTLogs_t spof;
#ifdef _MSC_VER
LogFileName << "C:/Calpont/log/trace/pt." << tid;
LogFileName << "C:/Calpont/log/trace/pt." << tid;
#else
LogFileName << "/var/log/mariadb/columnstore/trace/pt." << tid;
LogFileName << "/var/log/mariadb/columnstore/trace/pt." << tid;
#endif
spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str()));
gThdCnt++;
// TODO: add error checking
if (spof->logFD.is_open()) {
gFDList[tid] = spof;
logFD = spof.get();
}
} else
logFD =(*it).second.get();
spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str()));
gThdCnt++;
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
clock_gettime(CLOCK_MONOTONIC, &tm);
} // if (fTrace)
// TODO: add error checking
if (spof->logFD.is_open())
{
gFDList[tid] = spof;
logFD = spof.get();
}
}
else
logFD = (*it).second.get();
uint32_t retries = 0;
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
clock_gettime(CLOCK_MONOTONIC, &tm);
} // if (fTrace)
uint32_t retries = 0;
restart:
try {
ret = (*bpp)();
}
catch (NeedToRestartJob &e) {
ostringstream os;
// experimentally the race can exist longer than 10s. "No way" should
// it take 10 minutes. If it does, the user will have to resubmit their
// query.
// 9/27/12 - changed the timeout to 2 mins b/c people report the system
// is hung if it does nothing for 10 mins. 2 mins should still be more
// than enough
if (++retries == 120) {
os << e.what() << ": Restarted a syscat job " << retries << " times, bailing\n";
throw NeedToRestartJob(os.str());
}
flushSyscatOIDs();
bs->rewind();
bpp->resetBPP(*bs, writelock, sock);
sleep(1);
goto restart;
}
try
{
ret = (*bpp)();
}
catch (NeedToRestartJob& e)
{
ostringstream os;
// experimentally the race can exist longer than 10s. "No way" should
// it take 10 minutes. If it does, the user will have to resubmit their
// query.
if (ret)
return ret;
// 9/27/12 - changed the timeout to 2 mins b/c people report the system
// is hung if it does nothing for 10 mins. 2 mins should still be more
// than enough
if (++retries == 120)
{
os << e.what() << ": Restarted a syscat job " << retries << " times, bailing\n";
throw NeedToRestartJob(os.str());
}
if (fTrace)
if (logFD && logFD->logFD.is_open()) {
clock_gettime(CLOCK_MONOTONIC, &tm2);
timespec_sub(tm, tm2, tm3);
logFD->logFD
<< left << setw(3) << logFD->thdId
<< right << fixed << ((double)(tm.tv_sec+(1.e-9*tm.tv_nsec))) << " "
<< right << fixed << tm3 << " "
<< right << setw(6) << bpp->getSessionID() << " "
<< right << setw(4) << bpp->getStepID() << " "
<< right << setw(2) << bpp->FilterCount() << " "
<< right << setw(2) << bpp->ProjectCount() << " "
<< right << setw(9) << bpp->PhysIOCount() << " "
<< right << setw(9) << bpp->CachedIOCount() << " "
<< right << setw(9) << bpp->BlocksTouchedCount()
<< endl;
} // if (logFD...
flushSyscatOIDs();
bs->rewind();
bpp->resetBPP(*bs, writelock, sock);
sleep(1);
goto restart;
}
}
catch (scalar_exception &se)
{
if (gotBPP)
bpp->busy(false);
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
}
catch(exception& ex)
{
if (gotBPP)
bpp->busy(false);
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
catchHandler(ex.what(), uniqueID, stepID);
cout << "BPPSeeder step " << stepID << " caught an exception: " << ex.what() << endl;
}
catch(...)
{
if (gotBPP)
bpp->busy(false);
if (ptLock) {
gFDMutex.unlock();
ptLock=false;
}
string msg("BPPSeeder caught an unknown exception");
catchHandler(msg, uniqueID, stepID);
cout << msg << endl;
}
return ret;
if (ret)
return ret;
if (fTrace)
if (logFD && logFD->logFD.is_open())
{
clock_gettime(CLOCK_MONOTONIC, &tm2);
timespec_sub(tm, tm2, tm3);
logFD->logFD
<< left << setw(3) << logFD->thdId
<< right << fixed << ((double)(tm.tv_sec + (1.e-9 * tm.tv_nsec))) << " "
<< right << fixed << tm3 << " "
<< right << setw(6) << bpp->getSessionID() << " "
<< right << setw(4) << bpp->getStepID() << " "
<< right << setw(2) << bpp->FilterCount() << " "
<< right << setw(2) << bpp->ProjectCount() << " "
<< right << setw(9) << bpp->PhysIOCount() << " "
<< right << setw(9) << bpp->CachedIOCount() << " "
<< right << setw(9) << bpp->BlocksTouchedCount()
<< endl;
} // if (logFD...
}
catch (scalar_exception& se)
{
if (gotBPP)
bpp->busy(false);
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
}
catch (exception& ex)
{
if (gotBPP)
bpp->busy(false);
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
catchHandler(ex.what(), uniqueID, stepID);
cout << "BPPSeeder step " << stepID << " caught an exception: " << ex.what() << endl;
}
catch (...)
{
if (gotBPP)
bpp->busy(false);
if (ptLock)
{
gFDMutex.unlock();
ptLock = false;
}
string msg("BPPSeeder caught an unknown exception");
catchHandler(msg, uniqueID, stepID);
cout << msg << endl;
}
return ret;
}
void BPPSeeder::catchHandler(const string& ex, uint32_t id, uint32_t step)
{
Logger log;
log.logMessage(ex);
sendErrorMsg(id, logging::bppSeederErr, step);
Logger log;
log.logMessage(ex);
sendErrorMsg(id, logging::bppSeederErr, step);
}
void BPPSeeder::sendErrorMsg(uint32_t id, uint16_t status, uint32_t step)
{
ISMPacketHeader ism;
PrimitiveHeader ph = {0};
ISMPacketHeader ism;
PrimitiveHeader ph = {0};
ism.Status = status;
ph.UniqueID = id;
ph.StepID = step;
ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader));
msg.append((uint8_t *) &ism, sizeof(ism));
msg.append((uint8_t *) &ph, sizeof(ph));
ism.Status = status;
ph.UniqueID = id;
ph.StepID = step;
ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader));
msg.append((uint8_t*) &ism, sizeof(ism));
msg.append((uint8_t*) &ph, sizeof(ph));
boost::mutex::scoped_lock lk(*writelock);
sock->write(msg);
boost::mutex::scoped_lock lk(*writelock);
sock->write(msg);
}
bool BPPSeeder::isSysCat()
{
const uint8_t *buf;
uint32_t sessionIDOffset = sizeof(ISMPacketHeader);
uint32_t sessionID;
const uint8_t* buf;
uint32_t sessionIDOffset = sizeof(ISMPacketHeader);
uint32_t sessionID;
buf = bs->buf();
sessionID = *((uint32_t *) &buf[sessionIDOffset]);
return (sessionID & 0x80000000);
buf = bs->buf();
sessionID = *((uint32_t*) &buf[sessionIDOffset]);
return (sessionID & 0x80000000);
}
uint32_t BPPSeeder::getID()
{
return uniqueID;
return uniqueID;
}
/* This is part of the syscat-retry hack. We should get rid of it once we
@ -369,14 +422,15 @@ uint32_t BPPSeeder::getID()
*/
void BPPSeeder::flushSyscatOIDs()
{
vector<BRM::OID_t> syscatOIDs;
vector<BRM::OID_t> syscatOIDs;
syscatOIDs = execplan::getAllSysCatOIDs();
syscatOIDs = execplan::getAllSysCatOIDs();
for (int i = 0; i < fCacheCount; i++) {
dbbc::blockCacheClient bc(*BRPp[i]);
bc.flushOIDs((const uint32_t *) &syscatOIDs[0], syscatOIDs.size());
}
for (int i = 0; i < fCacheCount; i++)
{
dbbc::blockCacheClient bc(*BRPp[i]);
bc.flushOIDs((const uint32_t*) &syscatOIDs[0], syscatOIDs.size());
}
}
};

68
primitives/primproc/bppseeder.h
View File

@ -50,46 +50,52 @@ namespace primitiveprocessor
{
class BPPSeeder : public threadpool::PriorityThreadPool::Functor
{
public:
BPPSeeder(const messageqcpp::SBS &,
const SP_UM_MUTEX& wLock,
const SP_UM_IOSOCK& ios,
const int pmThreads,
const bool trace=false);
BPPSeeder(const BPPSeeder &b);
public:
BPPSeeder(const messageqcpp::SBS&,
const SP_UM_MUTEX& wLock,
const SP_UM_IOSOCK& ios,
const int pmThreads,
const bool trace = false);
BPPSeeder(const BPPSeeder& b);
virtual ~BPPSeeder();
virtual ~BPPSeeder();
int operator()();
int operator()();
bool isSysCat();
boost::shared_ptr<std::ofstream> spof;
bool isSysCat();
boost::shared_ptr<std::ofstream> spof;
uint32_t getID();
uint32_t getID();
void priority(uint32_t p) { _priority = p; }
uint32_t priority() { return _priority; }
void priority(uint32_t p)
{
_priority = p;
}
uint32_t priority()
{
return _priority;
}
private:
BPPSeeder();
void catchHandler(const std::string& s, uint32_t uniqueID, uint32_t step);
void sendErrorMsg(uint32_t id, uint16_t status, uint32_t step);
void flushSyscatOIDs();
private:
BPPSeeder();
void catchHandler(const std::string& s, uint32_t uniqueID, uint32_t step);
void sendErrorMsg(uint32_t id, uint16_t status, uint32_t step);
void flushSyscatOIDs();
messageqcpp::SBS bs;
SP_UM_MUTEX writelock;
SP_UM_IOSOCK sock;
int fPMThreads;
bool fTrace;
messageqcpp::SBS bs;
SP_UM_MUTEX writelock;
SP_UM_IOSOCK sock;
int fPMThreads;
bool fTrace;
/* To support reentrancy */
uint32_t uniqueID, sessionID, stepID, failCount;
boost::shared_ptr<BatchPrimitiveProcessor> bpp;
SBPPV bppv;
bool firstRun;
boost::posix_time::ptime dieTime;
/* To support reentrancy */
uint32_t uniqueID, sessionID, stepID, failCount;
boost::shared_ptr<BatchPrimitiveProcessor> bpp;
SBPPV bppv;
bool firstRun;
boost::posix_time::ptime dieTime;
uint32_t _priority;
uint32_t _priority;
};
};

341
primitives/primproc/bppsendthread.cpp
View File

@ -32,196 +32,241 @@ using namespace boost;
namespace primitiveprocessor
{
extern uint32_t connectionsPerUM;
BPPSendThread::BPPSendThread() : die(false), gotException(false), mainThreadWaiting(false),
sizeThreshold(100), msgsLeft(-1), waiting(false), sawAllConnections(false),
fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
sizeThreshold(100), msgsLeft(-1), waiting(false), sawAllConnections(false),
fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
{
runner = boost::thread(Runner_t(this));
runner = boost::thread(Runner_t(this));
}
BPPSendThread::BPPSendThread(uint32_t initMsgsLeft) : die(false), gotException(false),
mainThreadWaiting(false), sizeThreshold(100), msgsLeft(initMsgsLeft), waiting(false),
sawAllConnections(false), fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
mainThreadWaiting(false), sizeThreshold(100), msgsLeft(initMsgsLeft), waiting(false),
sawAllConnections(false), fcEnabled(false), currentByteSize(0), maxByteSize(25000000)
{
runner = boost::thread(Runner_t(this));
runner = boost::thread(Runner_t(this));
}
BPPSendThread::~BPPSendThread()
{
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
runner.join();
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
runner.join();
}
bool BPPSendThread::okToProceed()
{
// keep the queue size below the 100 msg threshold & below the 25MB mark,
// but at least 2 msgs so there is always 1 ready to be sent.
return ((msgQueue.size() < sizeThreshold && currentByteSize < maxByteSize)
|| msgQueue.size() < 3) && !die;
// keep the queue size below the 100 msg threshold & below the 25MB mark,
// but at least 2 msgs so there is always 1 ready to be sent.
return ((msgQueue.size() < sizeThreshold && currentByteSize < maxByteSize)
|| msgQueue.size() < 3) && !die;
}
void BPPSendThread::sendResult(const Msg_t &msg, bool newConnection)
void BPPSendThread::sendResult(const Msg_t& msg, bool newConnection)
{
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msg.msg->lengthWithHdrOverhead());
msgQueue.push(msg);
if (!sawAllConnections && newConnection) {
Connection_t ins(msg.sockLock, msg.sock);
bool inserted = connections_s.insert(ins).second;
if (inserted) {
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM) {
connections_s.clear();
sawAllConnections = true;
}
}
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msg.msg->lengthWithHdrOverhead());
msgQueue.push(msg);
if (!sawAllConnections && newConnection)
{
Connection_t ins(msg.sockLock, msg.sock);
bool inserted = connections_s.insert(ins).second;
if (inserted)
{
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM)
{
connections_s.clear();
sawAllConnections = true;
}
}
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
}
void BPPSendThread::sendResults(const vector<Msg_t> &msgs, bool newConnection)
void BPPSendThread::sendResults(const vector<Msg_t>& msgs, bool newConnection)
{
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
if (!sawAllConnections && newConnection) {
idbassert(msgs.size() > 0);
Connection_t ins(msgs[0].sockLock, msgs[0].sock);
bool inserted = connections_s.insert(ins).second;
if (inserted) {
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM) {
connections_s.clear();
sawAllConnections = true;
}
}
}
for (uint32_t i = 0; i < msgs.size(); i++) {
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msgs[i].msg->lengthWithHdrOverhead());
msgQueue.push(msgs[i]);
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
if (die)
return;
mutex::scoped_lock sl(msgQueueLock);
if (gotException)
throw runtime_error(exceptionString);
if (!sawAllConnections && newConnection)
{
idbassert(msgs.size() > 0);
Connection_t ins(msgs[0].sockLock, msgs[0].sock);
bool inserted = connections_s.insert(ins).second;
if (inserted)
{
connections_v.push_back(ins);
if (connections_v.size() == connectionsPerUM)
{
connections_s.clear();
sawAllConnections = true;
}
}
}
for (uint32_t i = 0; i < msgs.size(); i++)
{
(void)atomicops::atomicAdd<uint64_t>(&currentByteSize, msgs[i].msg->lengthWithHdrOverhead());
msgQueue.push(msgs[i]);
}
if (mainThreadWaiting)
queueNotEmpty.notify_one();
}
void BPPSendThread::sendMore(int num)
{
mutex::scoped_lock sl(ackLock);
mutex::scoped_lock sl(ackLock);
// cout << "got an ACK for " << num << " msgsLeft=" << msgsLeft << endl;
if (num == -1)
fcEnabled = false;
else if (num == 0) {
fcEnabled = true;
msgsLeft = 0;
}
else
(void)atomicops::atomicAdd(&msgsLeft, num);
if (waiting)
okToSend.notify_one();
if (num == -1)
fcEnabled = false;
else if (num == 0)
{
fcEnabled = true;
msgsLeft = 0;
}
else
(void)atomicops::atomicAdd(&msgsLeft, num);
if (waiting)
okToSend.notify_one();
}
bool BPPSendThread::flowControlEnabled()
{
return fcEnabled;
return fcEnabled;
}
void BPPSendThread::mainLoop()
{
const uint32_t msgCap = 20;
boost::scoped_array<Msg_t> msg;
uint32_t msgCount = 0, i, msgsSent;
SP_UM_MUTEX lock;
SP_UM_IOSOCK sock;
bool doLoadBalancing = false;
const uint32_t msgCap = 20;
boost::scoped_array<Msg_t> msg;
uint32_t msgCount = 0, i, msgsSent;
SP_UM_MUTEX lock;
SP_UM_IOSOCK sock;
bool doLoadBalancing = false;
msg.reset(new Msg_t[msgCap]);
msg.reset(new Msg_t[msgCap]);
while (!die) {
mutex::scoped_lock sl(msgQueueLock);
if (msgQueue.empty() && !die) {
mainThreadWaiting = true;
queueNotEmpty.wait(sl);
mainThreadWaiting = false;
continue;
}
while (!die)
{
mutex::scoped_lock sl(msgQueueLock);
msgCount = (msgQueue.size() > msgCap ? msgCap : msgQueue.size());
for (i = 0; i < msgCount; i++) {
msg[i] = msgQueue.front();
msgQueue.pop();
}
doLoadBalancing = sawAllConnections;
sl.unlock();
if (msgQueue.empty() && !die)
{
mainThreadWaiting = true;
queueNotEmpty.wait(sl);
mainThreadWaiting = false;
continue;
}
/* In the send loop below, msgsSent tracks progress on sending the msg array,
* i how many msgs are sent by 1 run of the loop, limited by msgCount or msgsLeft. */
msgsSent = 0;
while (msgsSent < msgCount && !die) {
uint64_t bsSize;
if (msgsLeft <= 0 && fcEnabled && !die) {
mutex::scoped_lock sl2(ackLock);
while (msgsLeft <= 0 && fcEnabled && !die) {
waiting = true;
okToSend.wait(sl2);
waiting = false;
}
}
for (i = 0; msgsSent < msgCount && ((fcEnabled && msgsLeft > 0) || !fcEnabled) && !die;
msgsSent++, i++) {
if (doLoadBalancing) {
// Bug 4475 move control of sockIndex to batchPrimitiveProcessor
lock = connections_v[msg[msgsSent].sockIndex].sockLock;
sock = connections_v[msg[msgsSent].sockIndex].sock;
}
else {
lock = msg[msgsSent].sockLock;
sock = msg[msgsSent].sock;
}
bsSize = msg[msgsSent].msg->lengthWithHdrOverhead();
try {
mutex::scoped_lock sl2(*lock);
sock->write(*msg[msgsSent].msg);
//cout << "sent 1 msg\n";
}
catch (std::exception &e) {
sl.lock();
exceptionString = e.what();
gotException = true;
return;
}
(void)atomicops::atomicDec(&msgsLeft);
(void)atomicops::atomicSub(&currentByteSize, bsSize);
msg[msgsSent].msg.reset();
}
}
}
msgCount = (msgQueue.size() > msgCap ? msgCap : msgQueue.size());
for (i = 0; i < msgCount; i++)
{
msg[i] = msgQueue.front();
msgQueue.pop();
}
doLoadBalancing = sawAllConnections;
sl.unlock();
/* In the send loop below, msgsSent tracks progress on sending the msg array,
* i how many msgs are sent by 1 run of the loop, limited by msgCount or msgsLeft. */
msgsSent = 0;
while (msgsSent < msgCount && !die)
{
uint64_t bsSize;
if (msgsLeft <= 0 && fcEnabled && !die)
{
mutex::scoped_lock sl2(ackLock);
while (msgsLeft <= 0 && fcEnabled && !die)
{
waiting = true;
okToSend.wait(sl2);
waiting = false;
}
}
for (i = 0; msgsSent < msgCount && ((fcEnabled && msgsLeft > 0) || !fcEnabled) && !die;
msgsSent++, i++)
{
if (doLoadBalancing)
{
// Bug 4475 move control of sockIndex to batchPrimitiveProcessor
lock = connections_v[msg[msgsSent].sockIndex].sockLock;
sock = connections_v[msg[msgsSent].sockIndex].sock;
}
else
{
lock = msg[msgsSent].sockLock;
sock = msg[msgsSent].sock;
}
bsSize = msg[msgsSent].msg->lengthWithHdrOverhead();
try
{
mutex::scoped_lock sl2(*lock);
sock->write(*msg[msgsSent].msg);
//cout << "sent 1 msg\n";
}
catch (std::exception& e)
{
sl.lock();
exceptionString = e.what();
gotException = true;
return;
}
(void)atomicops::atomicDec(&msgsLeft);
(void)atomicops::atomicSub(&currentByteSize, bsSize);
msg[msgsSent].msg.reset();
}
}
}
}
void BPPSendThread::abort()
{
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
mutex::scoped_lock sl(msgQueueLock);
mutex::scoped_lock sl2(ackLock);
die = true;
queueNotEmpty.notify_one();
okToSend.notify_one();
sl.unlock();
sl2.unlock();
}
}

155
primitives/primproc/bppsendthread.h
View File

@ -35,79 +35,96 @@
namespace primitiveprocessor
{
class BPPSendThread {
class BPPSendThread
{
public:
BPPSendThread(); // starts unthrottled
BPPSendThread(uint32_t initMsgsLeft); // starts throttled
virtual ~BPPSendThread();
public:
BPPSendThread(); // starts unthrottled
BPPSendThread(uint32_t initMsgsLeft); // starts throttled
virtual ~BPPSendThread();
struct Msg_t {
messageqcpp::SBS msg;
SP_UM_IOSOCK sock;
SP_UM_MUTEX sockLock;
int sockIndex; // Socket index for round robin control. Bug 4475
Msg_t() : sockIndex(0) { }
Msg_t(const Msg_t &m) : msg(m.msg), sock(m.sock), sockLock(m.sockLock), sockIndex(m.sockIndex) { }
Msg_t & operator=(const Msg_t &m)
{ msg = m.msg; sock = m.sock; sockLock = m.sockLock; sockIndex = m.sockIndex; return *this; }
Msg_t(const messageqcpp::SBS &m, const SP_UM_IOSOCK &so, const SP_UM_MUTEX &sl, int si) :
msg(m), sock(so), sockLock(sl), sockIndex(si) { }
};
struct Msg_t
{
messageqcpp::SBS msg;
SP_UM_IOSOCK sock;
SP_UM_MUTEX sockLock;
int sockIndex; // Socket index for round robin control. Bug 4475
Msg_t() : sockIndex(0) { }
Msg_t(const Msg_t& m) : msg(m.msg), sock(m.sock), sockLock(m.sockLock), sockIndex(m.sockIndex) { }
Msg_t& operator=(const Msg_t& m)
{
msg = m.msg;
sock = m.sock;
sockLock = m.sockLock;
sockIndex = m.sockIndex;
return *this;
}
Msg_t(const messageqcpp::SBS& m, const SP_UM_IOSOCK& so, const SP_UM_MUTEX& sl, int si) :
msg(m), sock(so), sockLock(sl), sockIndex(si) { }
};
bool okToProceed();
void sendMore(int num);
void sendResults(const std::vector<Msg_t> &msgs, bool newConnection);
void sendResult(const Msg_t &msg, bool newConnection);
void mainLoop();
bool flowControlEnabled();
void abort();
inline bool aborted() const
{
return die;
}
private:
BPPSendThread(const BPPSendThread&);
BPPSendThread& operator=(const BPPSendThread&);
bool okToProceed();
void sendMore(int num);
void sendResults(const std::vector<Msg_t>& msgs, bool newConnection);
void sendResult(const Msg_t& msg, bool newConnection);
void mainLoop();
bool flowControlEnabled();
void abort();
inline bool aborted() const
{
return die;
}
struct Runner_t {
BPPSendThread *bppst;
Runner_t(BPPSendThread *b) : bppst(b) { }
void operator()() { bppst->mainLoop(); }
};
boost::thread runner;
std::queue<Msg_t> msgQueue;
boost::mutex msgQueueLock;
boost::condition queueNotEmpty;
volatile bool die, gotException, mainThreadWaiting;
std::string exceptionString;
uint32_t sizeThreshold;
volatile int32_t msgsLeft;
bool waiting;
boost::mutex ackLock;
boost::condition okToSend;
/* Load balancing structures */
struct Connection_t {
Connection_t(const SP_UM_MUTEX &lock, const SP_UM_IOSOCK &so) :
sockLock(lock), sock(so) { }
SP_UM_MUTEX sockLock;
SP_UM_IOSOCK sock;
bool operator<(const Connection_t &t) const
{ return sockLock.get() < t.sockLock.get(); }
bool operator==(const Connection_t &t) const
{ return sockLock.get() == t.sockLock.get(); }
};
std::set<Connection_t> connections_s;
std::vector<Connection_t> connections_v;
bool sawAllConnections;
volatile bool fcEnabled;
/* secondary queue size restriction based on byte size */
volatile uint64_t currentByteSize;
uint64_t maxByteSize;
private:
BPPSendThread(const BPPSendThread&);
BPPSendThread& operator=(const BPPSendThread&);
struct Runner_t
{
BPPSendThread* bppst;
Runner_t(BPPSendThread* b) : bppst(b) { }
void operator()()
{
bppst->mainLoop();
}
};
boost::thread runner;
std::queue<Msg_t> msgQueue;
boost::mutex msgQueueLock;
boost::condition queueNotEmpty;
volatile bool die, gotException, mainThreadWaiting;
std::string exceptionString;
uint32_t sizeThreshold;
volatile int32_t msgsLeft;
bool waiting;
boost::mutex ackLock;
boost::condition okToSend;
/* Load balancing structures */
struct Connection_t
{
Connection_t(const SP_UM_MUTEX& lock, const SP_UM_IOSOCK& so) :
sockLock(lock), sock(so) { }
SP_UM_MUTEX sockLock;
SP_UM_IOSOCK sock;
bool operator<(const Connection_t& t) const
{
return sockLock.get() < t.sockLock.get();
}
bool operator==(const Connection_t& t) const
{
return sockLock.get() == t.sockLock.get();
}
};
std::set<Connection_t> connections_s;
std::vector<Connection_t> connections_v;
bool sawAllConnections;
volatile bool fcEnabled;
/* secondary queue size restriction based on byte size */
volatile uint64_t currentByteSize;
uint64_t maxByteSize;
};
}

1362
primitives/primproc/columncommand.cpp
View File

File diff suppressed because it is too large Load Diff

179
primitives/primproc/columncommand.h
View File

@ -41,105 +41,132 @@ namespace primitiveprocessor
class ColumnCommand : public Command
{
public:
ColumnCommand();
virtual ~ColumnCommand();
ColumnCommand();
virtual ~ColumnCommand();
inline uint64_t getLBID() { return lbid; }
inline uint8_t getWidth() { return colType.colWidth; }
inline uint8_t getScale() { return colType.scale; }
uint16_t getFilterCount() { return filterCount; }
const execplan::CalpontSystemCatalog::ColType& getColType() { return colType; }
inline uint64_t getLBID()
{
return lbid;
}
inline uint8_t getWidth()
{
return colType.colWidth;
}
inline uint8_t getScale()
{
return colType.scale;
}
uint16_t getFilterCount()
{
return filterCount;
}
const execplan::CalpontSystemCatalog::ColType& getColType()
{
return colType;
}
void execute();
void execute(int64_t *vals); //used by RTSCommand to redirect values
void prep(int8_t outputType, bool makeAbsRids);
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t pos);
void nextLBID();
bool isScan() { return _isScan; }
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
void setMakeAbsRids(bool m) { makeAbsRids = m; }
bool willPrefetch();
const uint64_t getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const;
const int64_t getLastLbid();
void getLBIDList(uint32_t loopCount, std::vector<int64_t> *lbids);
void execute();
void execute(int64_t* vals); //used by RTSCommand to redirect values
void prep(int8_t outputType, bool makeAbsRids);
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t pos);
void nextLBID();
bool isScan()
{
return _isScan;
}
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
void setMakeAbsRids(bool m)
{
makeAbsRids = m;
}
bool willPrefetch();
const uint64_t getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const;
const int64_t getLastLbid();
void getLBIDList(uint32_t loopCount, std::vector<int64_t>* lbids);
virtual SCommand duplicate();
bool operator==(const ColumnCommand &) const;
bool operator!=(const ColumnCommand &) const;
virtual SCommand duplicate();
bool operator==(const ColumnCommand&) const;
bool operator!=(const ColumnCommand&) const;
/* OR hacks */
void setScan(bool b) { _isScan = b; }
void disableFilters();
void enableFilters();
/* OR hacks */
void setScan(bool b)
{
_isScan = b;
}
void disableFilters();
void enableFilters();
int getCompType() const { return colType.compressionType; }
int getCompType() const
{
return colType.compressionType;
}
protected:
virtual void loadData();
void duplicate(ColumnCommand *);
virtual void loadData();
void duplicate(ColumnCommand*);
// we only care about the width and type fields.
//On the PM the rest is uninitialized
execplan::CalpontSystemCatalog::ColType colType;
// we only care about the width and type fields.
//On the PM the rest is uninitialized
execplan::CalpontSystemCatalog::ColType colType;
private:
ColumnCommand(const ColumnCommand &);
ColumnCommand& operator=(const ColumnCommand &);
ColumnCommand(const ColumnCommand&);
ColumnCommand& operator=(const ColumnCommand&);
void _execute();
void issuePrimitive();
void processResult();
void process_OT_BOTH();
void process_OT_RID();
void process_OT_DATAVALUE();
void process_OT_ROWGROUP();
void projectResult();
void projectResultRG(rowgroup::RowGroup &rg, uint32_t pos);
void removeRowsFromRowGroup(rowgroup::RowGroup &);
void makeScanMsg();
void makeStepMsg();
void setLBID(uint64_t rid);
void _execute();
void issuePrimitive();
void processResult();
void process_OT_BOTH();
void process_OT_RID();
void process_OT_DATAVALUE();
void process_OT_ROWGROUP();
void projectResult();
void projectResultRG(rowgroup::RowGroup& rg, uint32_t pos);
void removeRowsFromRowGroup(rowgroup::RowGroup&);
void makeScanMsg();
void makeStepMsg();
void setLBID(uint64_t rid);
bool _isScan;
bool _isScan;
boost::scoped_array<uint8_t> inputMsg;
NewColRequestHeader *primMsg;
NewColResultHeader *outMsg;
boost::scoped_array<uint8_t> inputMsg;
NewColRequestHeader* primMsg;
NewColResultHeader* outMsg;
// the length of base prim msg, which is everything up to the
// rid array for the pCol message
uint32_t baseMsgLength;
// the length of base prim msg, which is everything up to the
// rid array for the pCol message
uint32_t baseMsgLength;
uint64_t lbid;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
messageqcpp::ByteStream filterString;
uint16_t filterCount;
bool makeAbsRids;
int64_t *values; // this is usually bpp->values; RTSCommand needs to use a different container
uint64_t lbid;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
messageqcpp::ByteStream filterString;
uint16_t filterCount;
bool makeAbsRids;
int64_t* values; // this is usually bpp->values; RTSCommand needs to use a different container
uint8_t mask, shift; // vars for the selective block loader
uint8_t mask, shift; // vars for the selective block loader
// counters to decide whether to prefetch or not
uint32_t blockCount, loadCount;
// counters to decide whether to prefetch or not
uint32_t blockCount, loadCount;
boost::shared_ptr<primitives::ParsedColumnFilter> parsedColumnFilter;
boost::shared_ptr<primitives::ParsedColumnFilter> parsedColumnFilter;
/* OR hacks */
boost::shared_ptr<primitives::ParsedColumnFilter> emptyFilter;
bool suppressFilter;
/* OR hacks */
boost::shared_ptr<primitives::ParsedColumnFilter> emptyFilter;
bool suppressFilter;
std::vector<uint64_t> lastLbid;
std::vector<uint64_t> lastLbid;
/* speculative optimizations for projectintorowgroup() */
rowgroup::Row r;
uint32_t rowSize;
/* speculative optimizations for projectintorowgroup() */
rowgroup::Row r;
uint32_t rowSize;
bool wasVersioned;
bool wasVersioned;
friend class RTSCommand;
friend class RTSCommand;
};
}

204
primitives/primproc/command.cpp
View File

@ -30,124 +30,146 @@ Command::Command(CommandType c) : cmdType(c), fFilterFeeder(NOT_FEEDER) { }
Command::~Command() { };
void Command::createCommand(ByteStream &bs)
void Command::createCommand(ByteStream& bs)
{
bs >> OID;
bs >> tupleKey;
bs >> queryUuid;
bs >> stepUuid;
bs >> OID;
bs >> tupleKey;
bs >> queryUuid;
bs >> stepUuid;
}
void Command::resetCommand(ByteStream &bs) { };
void Command::resetCommand(ByteStream& bs) { };
void Command::setMakeAbsRids(bool) { }
Command* Command::makeCommand(ByteStream &bs, CommandType *type, vector<SCommand>& cmds)
Command* Command::makeCommand(ByteStream& bs, CommandType* type, vector<SCommand>& cmds)
{
Command* ret;
uint8_t tmp8;
Command* ret;
uint8_t tmp8;
bs.peek(tmp8);
*type = (CommandType) tmp8;
switch (*type)
{
case COLUMN_COMMAND:
ret = new ColumnCommand();
break;
case DICT_STEP:
ret = new DictStep();
break;
case PASS_THRU:
ret = new PassThruCommand();
break;
case RID_TO_STRING:
ret = new RTSCommand();
break;
case FILTER_COMMAND:
ret = FilterCommand::makeFilterCommand(bs, cmds);
break;
bs.peek(tmp8);
*type = (CommandType) tmp8;
switch (*type) {
case COLUMN_COMMAND:
ret = new ColumnCommand();
break;
case DICT_STEP:
ret = new DictStep();
break;
case PASS_THRU:
ret = new PassThruCommand();
break;
case RID_TO_STRING:
ret = new RTSCommand();
break;
case FILTER_COMMAND:
ret = FilterCommand::makeFilterCommand(bs, cmds);
break;
case PSEUDOCOLUMN:
ret = new PseudoCC();
break;
default:
throw logic_error("Command::makeCommand(): can't deserialize this bytestream");
};
ret->createCommand(bs);
return ret;
default:
throw logic_error("Command::makeCommand(): can't deserialize this bytestream");
};
ret->createCommand(bs);
return ret;
}
void Command::setBatchPrimitiveProcessor(BatchPrimitiveProcessor *b)
void Command::setBatchPrimitiveProcessor(BatchPrimitiveProcessor* b)
{
bpp = b;
bpp = b;
}
void Command::copyRidsForFilterCmd()
{
if (fFilterFeeder == LEFT_FEEDER)
{
bpp->fFiltRidCount[0] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[0][i] = bpp->relRids[i];
}
else // if (fFilterFeeder == RIGHT_FEEDER)
{
bpp->fFiltRidCount[1] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[1][i] = bpp->relRids[i];
}
if (fFilterFeeder == LEFT_FEEDER)
{
bpp->fFiltRidCount[0] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[0][i] = bpp->relRids[i];
}
else // if (fFilterFeeder == RIGHT_FEEDER)
{
bpp->fFiltRidCount[1] = bpp->ridCount;
for (uint64_t i = 0; i < bpp->ridCount; i++)
bpp->fFiltCmdRids[1][i] = bpp->relRids[i];
}
}
bool Command::operator==(const Command &c) const
bool Command::operator==(const Command& c) const
{
const type_info &cType = typeid(c);
const type_info& cType = typeid(c);
if (cType != typeid(*this))
return false;
if (cType != typeid(*this))
return false;
if (cType == typeid(ColumnCommand)) {
const ColumnCommand *cc = dynamic_cast<const ColumnCommand *>(&c);
const ColumnCommand *t = dynamic_cast<const ColumnCommand *>(this);
if (*cc != *t)
return false;
}
else if (cType == typeid(DictStep)) {
const DictStep *ds = dynamic_cast<const DictStep *>(&c);
const DictStep *t = dynamic_cast<const DictStep *>(this);
if (*ds != *t)
return false;
}
else if (cType == typeid(PassThruCommand)) {
const PassThruCommand *pt = dynamic_cast<const PassThruCommand *>(&c);
const PassThruCommand *t = dynamic_cast<const PassThruCommand *>(this);
if (*pt != *t)
return false;
}
else if (cType == typeid(RTSCommand)) {
const RTSCommand *rts = dynamic_cast<const RTSCommand *>(&c);
const RTSCommand *t = dynamic_cast<const RTSCommand *>(this);
if (*rts != *t)
return false;
}
else if (cType == typeid(FilterCommand)) {
const FilterCommand *fc = dynamic_cast<const FilterCommand *>(&c);
const FilterCommand *t = dynamic_cast<const FilterCommand *>(this);
if (*fc != *t)
return false;
}
else
cerr << "unknown Command type\n";
if (cType == typeid(ColumnCommand))
{
const ColumnCommand* cc = dynamic_cast<const ColumnCommand*>(&c);
const ColumnCommand* t = dynamic_cast<const ColumnCommand*>(this);
return true;
if (*cc != *t)
return false;
}
else if (cType == typeid(DictStep))
{
const DictStep* ds = dynamic_cast<const DictStep*>(&c);
const DictStep* t = dynamic_cast<const DictStep*>(this);
if (*ds != *t)
return false;
}
else if (cType == typeid(PassThruCommand))
{
const PassThruCommand* pt = dynamic_cast<const PassThruCommand*>(&c);
const PassThruCommand* t = dynamic_cast<const PassThruCommand*>(this);
if (*pt != *t)
return false;
}
else if (cType == typeid(RTSCommand))
{
const RTSCommand* rts = dynamic_cast<const RTSCommand*>(&c);
const RTSCommand* t = dynamic_cast<const RTSCommand*>(this);
if (*rts != *t)
return false;
}
else if (cType == typeid(FilterCommand))
{
const FilterCommand* fc = dynamic_cast<const FilterCommand*>(&c);
const FilterCommand* t = dynamic_cast<const FilterCommand*>(this);
if (*fc != *t)
return false;
}
else
cerr << "unknown Command type\n";
return true;
}
void Command::duplicate(Command *c)
void Command::duplicate(Command* c)
{
bpp = c->bpp;
cmdType = c->cmdType;
fFilterFeeder = c->fFilterFeeder;
OID = c->OID;
tupleKey = c->tupleKey;
queryUuid = c->queryUuid;
stepUuid = c->stepUuid;
bpp = c->bpp;
cmdType = c->cmdType;
fFilterFeeder = c->fFilterFeeder;
OID = c->OID;
tupleKey = c->tupleKey;
queryUuid = c->queryUuid;
stepUuid = c->stepUuid;
}
}

123
primitives/primproc/command.h
View File

@ -35,74 +35,93 @@ typedef boost::shared_ptr<Command> SCommand;
class Command
{
public:
enum CommandType {
NONE,
COLUMN_COMMAND,
DICT_STEP,
DICT_SCAN,
PASS_THRU,
RID_TO_STRING,
FILTER_COMMAND,
enum CommandType
{
NONE,
COLUMN_COMMAND,
DICT_STEP,
DICT_SCAN,
PASS_THRU,
RID_TO_STRING,
FILTER_COMMAND,
PSEUDOCOLUMN
};
};
static const uint8_t NOT_FEEDER = 0;
static const uint8_t FILT_FEEDER = 1;
static const uint8_t LEFT_FEEDER = 3;
static const uint8_t RIGHT_FEEDER = 5;
static const uint8_t NOT_FEEDER = 0;
static const uint8_t FILT_FEEDER = 1;
static const uint8_t LEFT_FEEDER = 3;
static const uint8_t RIGHT_FEEDER = 5;
Command(CommandType c);
virtual ~Command();
Command(CommandType c);
virtual ~Command();
virtual void execute() = 0;
virtual void project() = 0;
virtual void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t columnPosition) = 0;
virtual uint64_t getLBID() = 0;
virtual void getLBIDList(uint32_t loopCount, std::vector<int64_t> *out) {} // the default fcn returns 0 lbids
virtual void nextLBID() = 0;
virtual void createCommand(messageqcpp::ByteStream &);
virtual void resetCommand(messageqcpp::ByteStream &);
virtual void execute() = 0;
virtual void project() = 0;
virtual void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t columnPosition) = 0;
virtual uint64_t getLBID() = 0;
virtual void getLBIDList(uint32_t loopCount, std::vector<int64_t>* out) {} // the default fcn returns 0 lbids
virtual void nextLBID() = 0;
virtual void createCommand(messageqcpp::ByteStream&);
virtual void resetCommand(messageqcpp::ByteStream&);
/* Duplicate() makes a copy of this object as constructed by createCommand().
It's thread-safe */
virtual SCommand duplicate() = 0;
bool operator==(const Command &) const;
bool operator!=(const Command& c) const { return !(*this == c); }
/* Duplicate() makes a copy of this object as constructed by createCommand().
It's thread-safe */
virtual SCommand duplicate() = 0;
bool operator==(const Command&) const;
bool operator!=(const Command& c) const
{
return !(*this == c);
}
/* Put bootstrap code here (ie, build the template primitive msg) */
virtual void prep(int8_t outputType, bool makeAbsRids) = 0;
virtual void setBatchPrimitiveProcessor(BatchPrimitiveProcessor *);
virtual void setMakeAbsRids(bool);
/* Put bootstrap code here (ie, build the template primitive msg) */
virtual void prep(int8_t outputType, bool makeAbsRids) = 0;
virtual void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*);
virtual void setMakeAbsRids(bool);
CommandType getCommandType() const { return cmdType; }
CommandType getCommandType() const
{
return cmdType;
}
// if feeding a filtercommand
// note: a filtercommand itself can feed another filtercommand
uint8_t filterFeeder() { return fFilterFeeder; }
void filterFeeder(uint8_t f) { fFilterFeeder = f; }
virtual void copyRidsForFilterCmd();
// if feeding a filtercommand
// note: a filtercommand itself can feed another filtercommand
uint8_t filterFeeder()
{
return fFilterFeeder;
}
void filterFeeder(uint8_t f)
{
fFilterFeeder = f;
}
virtual void copyRidsForFilterCmd();
static Command* makeCommand(messageqcpp::ByteStream&, CommandType*, std::vector<SCommand>&);
static Command* makeCommand(messageqcpp::ByteStream&, CommandType*, std::vector<SCommand>&);
uint32_t getOID() const { return OID; }
uint32_t getTupleKey() const { return tupleKey; }
uint32_t getOID() const
{
return OID;
}
uint32_t getTupleKey() const
{
return tupleKey;
}
virtual int getCompType() const=0;
virtual int getCompType() const = 0;
protected:
BatchPrimitiveProcessor *bpp;
CommandType cmdType;
uint8_t fFilterFeeder;
uint32_t OID;
uint32_t tupleKey;
boost::uuids::uuid queryUuid;
boost::uuids::uuid stepUuid;
BatchPrimitiveProcessor* bpp;
CommandType cmdType;
uint8_t fFilterFeeder;
uint32_t OID;
uint32_t tupleKey;
boost::uuids::uuid queryUuid;
boost::uuids::uuid stepUuid;
void duplicate(Command *);
void duplicate(Command*);
private:
Command();
Command(const Command &);
Command();
Command(const Command&);
};

910
primitives/primproc/dictstep.cpp
View File

File diff suppressed because it is too large Load Diff

179
primitives/primproc/dictstep.h
View File

@ -19,7 +19,7 @@
// $Id: dictstep.h 2110 2013-06-19 15:51:38Z bwilkinson $
// C++ Interface: dictstep
//
// Description:
// Description:
//
//
// Author: Patrick LeBlanc <pleblanc@calpont.com>, (C) 2008
@ -34,106 +34,121 @@
#include "command.h"
#include "primitivemsg.h"
namespace primitiveprocessor {
namespace primitiveprocessor
{
class DictStep : public Command
{
public:
DictStep();
virtual ~DictStep();
public:
DictStep();
virtual ~DictStep();
void execute();
void project();
void project(int64_t *vals); //used by RTSCommand to redirect input
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t row);
void projectIntoRowGroup(rowgroup::RowGroup &rg, int64_t *vals, uint32_t col);
uint64_t getLBID();
void execute();
void project();
void project(int64_t* vals); //used by RTSCommand to redirect input
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t row);
void projectIntoRowGroup(rowgroup::RowGroup& rg, int64_t* vals, uint32_t col);
uint64_t getLBID();
/* This doesn't do anything for this class... make it column-specific or not? */
void nextLBID();
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
/* This doesn't do anything for this class... make it column-specific or not? */
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
SCommand duplicate();
bool operator==(const DictStep &) const;
bool operator!=(const DictStep &) const;
SCommand duplicate();
bool operator==(const DictStep&) const;
bool operator!=(const DictStep&) const;
int getCompType() const { return compressionType; }
void setCompType(int ct) { compressionType = ct; }
int getCompType() const
{
return compressionType;
}
void setCompType(int ct)
{
compressionType = ct;
}
private:
DictStep(const DictStep &);
DictStep& operator=(const DictStep &);
private:
DictStep(const DictStep&);
DictStep& operator=(const DictStep&);
struct StringPtr {
const uint8_t* ptr;
unsigned len;
struct StringPtr
{
const uint8_t* ptr;
unsigned len;
StringPtr() : ptr(NULL), len(0) {;}
StringPtr(const uint8_t* p, unsigned l) : ptr(p), len(l) {;}
};
StringPtr() : ptr(NULL), len(0) {;}
StringPtr(const uint8_t* p, unsigned l) : ptr(p), len(l) {;}
};
void _execute();
void issuePrimitive(bool isProjection);
void processResult();
void projectResult(std::string* tmpStrings);
void projectResult(StringPtr *tmpStrings);
void _project();
void _projectToRG(rowgroup::RowGroup &rg, uint32_t col);
void _execute();
void issuePrimitive(bool isProjection);
void processResult();
void projectResult(std::string* tmpStrings);
void projectResult(StringPtr* tmpStrings);
void _project();
void _projectToRG(rowgroup::RowGroup& rg, uint32_t col);
// struct used for scratch space
struct OrderedToken {
uint64_t rid;
uint64_t token;
uint16_t pos;
std::string str;
bool inResult;
OrderedToken() : inResult(false) { }
~OrderedToken() { }
};
struct TokenSorter {
inline bool operator()(const OrderedToken &c1, const OrderedToken &c2) const
{ return (c1.token < c2.token); }
};
struct PosSorter {
inline bool operator()(const OrderedToken &c1, const OrderedToken &c2) const
{ return (c1.pos < c2.pos); }
};
// struct used for scratch space
struct OrderedToken
{
uint64_t rid;
uint64_t token;
uint16_t pos;
std::string str;
bool inResult;
OrderedToken() : inResult(false) { }
~OrderedToken() { }
};
struct TokenSorter
{
inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const
{
return (c1.token < c2.token);
}
};
struct PosSorter
{
inline bool operator()(const OrderedToken& c1, const OrderedToken& c2) const
{
return (c1.pos < c2.pos);
}
};
//bug 3679. FilterCommand depends on the result being in the same relative
//order as the input. These fcns help restore the original order.
void copyResultToTmpSpace(OrderedToken *ot);
void copyResultToFinalPosition(OrderedToken *ot);
//bug 3679. FilterCommand depends on the result being in the same relative
//order as the input. These fcns help restore the original order.
void copyResultToTmpSpace(OrderedToken* ot);
void copyResultToFinalPosition(OrderedToken* ot);
// Worst case, 8192 tokens in the msg. Each is 10 bytes. */
boost::scoped_array<uint8_t> inputMsg;
uint32_t tmpResultCounter;
uint32_t totalResultLength;
DictInput *primMsg;
std::vector<uint8_t> result;
// Worst case, 8192 tokens in the msg. Each is 10 bytes. */
boost::scoped_array<uint8_t> inputMsg;
uint32_t tmpResultCounter;
uint32_t totalResultLength;
DictInput* primMsg;
std::vector<uint8_t> result;
uint64_t lbid;
uint32_t fbo;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
int64_t *values;
boost::scoped_array<std::string>* strValues;
int compressionType;
ByteStream filterString;
uint32_t filterCount;
uint32_t bufferSize;
uint16_t inputRidCount;
uint64_t lbid;
uint32_t fbo;
uint32_t traceFlags; // probably move this to Command
uint8_t BOP;
int64_t* values;
boost::scoped_array<std::string>* strValues;
int compressionType;
ByteStream filterString;
uint32_t filterCount;
uint32_t bufferSize;
uint16_t inputRidCount;
bool hasEqFilter;
boost::shared_ptr<primitives::DictEqualityFilter> eqFilter;
boost::shared_array<primitives::idb_regex_t> likeFilter;
uint8_t eqOp; // COMPARE_EQ or COMPARE_NE
bool hasEqFilter;
boost::shared_ptr<primitives::DictEqualityFilter> eqFilter;
boost::shared_array<primitives::idb_regex_t> likeFilter;
uint8_t eqOp; // COMPARE_EQ or COMPARE_NE
friend class RTSCommand;
friend class RTSCommand;
};
} // namespace

770
primitives/primproc/filtercommand.cpp
View File

@ -49,121 +49,128 @@ namespace primitiveprocessor
Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector<SCommand>& cmds)
{
bs.advance(1);
// find out the # of commands in the cmds vector,
// vector::size() will not work, because cmds is resize() to filterCount
uint64_t nc = 0;
while (cmds[nc].get() != NULL) nc++;
// find out the # of commands in the cmds vector,
// vector::size() will not work, because cmds is resize() to filterCount
uint64_t nc = 0;
// figure out the feeding commands
// must have 2 columncommands, may have 1 or 2 dictsteps.
uint64_t cols = 0; // # of columncommands
uint32_t columns = 0;
uint64_t i = nc;
while (i > 0 && cols < 2)
{
Command::CommandType cmdType = cmds[i-1]->getCommandType();
if (cmdType != Command::COLUMN_COMMAND && cmdType != Command::DICT_STEP)
{
stringstream msg;
msg << "FilterCommand: feeded by " << cmdType << " is not supported.";
throw logic_error(msg.str());
}
while (cmds[nc].get() != NULL) nc++;
columns = (columns<<8) + cmdType;
if (cmdType == Command::COLUMN_COMMAND)
cols++;
i--;
}
// figure out the feeding commands
// must have 2 columncommands, may have 1 or 2 dictsteps.
uint64_t cols = 0; // # of columncommands
uint32_t columns = 0;
uint64_t i = nc;
// should not happen
if (cols < 2) throw logic_error("FilterCommand: not enough feeding ColumnCommands.");
while (i > 0 && cols < 2)
{
Command::CommandType cmdType = cmds[i - 1]->getCommandType();
if (cmdType != Command::COLUMN_COMMAND && cmdType != Command::DICT_STEP)
{
stringstream msg;
msg << "FilterCommand: feeded by " << cmdType << " is not supported.";
throw logic_error(msg.str());
}
columns = (columns << 8) + cmdType;
if (cmdType == Command::COLUMN_COMMAND)
cols++;
i--;
}
// should not happen
if (cols < 2) throw logic_error("FilterCommand: not enough feeding ColumnCommands.");
FilterCommand* fc = NULL;
// the order setting left/right feeder is important, left is the smaller index.
// because the doFilter relies on the rids of right-feeder is a subset of left.
if (columns == CC)
{
cmds[nc-2]->filterFeeder(LEFT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-2].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc-1].get());
int scale0 = cmd0->getScale();
int scale1 = cmd1->getScale();
// char[] is stored as int, but cannot directly compare if length is different
// due to endian issue
if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::TEXT)
{
StrFilterCmd* sc = new StrFilterCmd();
sc->setCompareFunc(CC);
fc = sc;
}
else if (scale0 == scale1)
{
fc = new FilterCommand();
}
else
{
ScaledFilterCmd* sc = new ScaledFilterCmd();
sc->setFactor(pow(10.0, scale1) / pow(10.0, scale0));
fc = sc;
}
// the order setting left/right feeder is important, left is the smaller index.
// because the doFilter relies on the rids of right-feeder is a subset of left.
if (columns == CC)
{
cmds[nc - 2]->filterFeeder(LEFT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == DCDC) // both string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc-4]->filterFeeder(FILT_FEEDER);
cmds[nc-3]->filterFeeder(LEFT_FEEDER);
cmds[nc-2]->filterFeeder(FILT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
sc->setCompareFunc(DCDC);
fc = sc;
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 2].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc - 1].get());
int scale0 = cmd0->getScale();
int scale1 = cmd1->getScale();
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-4].get());
ColumnCommand* cmd2 = dynamic_cast<ColumnCommand*>(cmds[nc-2].get());
fc->setColTypes(cmd0->getColType(), cmd2->getColType());
}
else if (columns == DCC) // lhs: char[]; rhs: string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc-3]->filterFeeder(LEFT_FEEDER);
cmds[nc-2]->filterFeeder(FILT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc-2].get());
size_t cl = cmd0->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(DCC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == CDC) // lhs: string; rhs: char[]
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc-3]->filterFeeder(FILT_FEEDER);
cmds[nc-2]->filterFeeder(LEFT_FEEDER);
cmds[nc-1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc-3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc-1].get());
size_t cl = cmd1->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(CDC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else
{
stringstream msg;
msg << "FilterCommand does not handle this column code: " << hex << columns << dec;
throw logic_error(msg.str());
}
// char[] is stored as int, but cannot directly compare if length is different
// due to endian issue
if (cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::CHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::VARCHAR ||
cmd0->getColType().colDataType == execplan::CalpontSystemCatalog::TEXT)
{
StrFilterCmd* sc = new StrFilterCmd();
sc->setCompareFunc(CC);
fc = sc;
}
else if (scale0 == scale1)
{
fc = new FilterCommand();
}
else
{
ScaledFilterCmd* sc = new ScaledFilterCmd();
sc->setFactor(pow(10.0, scale1) / pow(10.0, scale0));
fc = sc;
}
return fc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == DCDC) // both string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc - 4]->filterFeeder(FILT_FEEDER);
cmds[nc - 3]->filterFeeder(LEFT_FEEDER);
cmds[nc - 2]->filterFeeder(FILT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
sc->setCompareFunc(DCDC);
fc = sc;
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 4].get());
ColumnCommand* cmd2 = dynamic_cast<ColumnCommand*>(cmds[nc - 2].get());
fc->setColTypes(cmd0->getColType(), cmd2->getColType());
}
else if (columns == DCC) // lhs: char[]; rhs: string
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc - 3]->filterFeeder(LEFT_FEEDER);
cmds[nc - 2]->filterFeeder(FILT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc - 2].get());
size_t cl = cmd0->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(DCC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else if (columns == CDC) // lhs: string; rhs: char[]
{
StrFilterCmd* sc = new StrFilterCmd();
cmds[nc - 3]->filterFeeder(FILT_FEEDER);
cmds[nc - 2]->filterFeeder(LEFT_FEEDER);
cmds[nc - 1]->filterFeeder(RIGHT_FEEDER);
ColumnCommand* cmd0 = dynamic_cast<ColumnCommand*>(cmds[nc - 3].get());
ColumnCommand* cmd1 = dynamic_cast<ColumnCommand*>(cmds[nc - 1].get());
size_t cl = cmd1->getWidth(); // char[] column
sc->setCharLength(cl);
sc->setCompareFunc(CDC);
fc = sc;
fc->setColTypes(cmd0->getColType(), cmd1->getColType());
}
else
{
stringstream msg;
msg << "FilterCommand does not handle this column code: " << hex << columns << dec;
throw logic_error(msg.str());
}
return fc;
}
@ -179,18 +186,18 @@ FilterCommand::~FilterCommand()
void FilterCommand::execute()
{
doFilter();
doFilter();
}
void FilterCommand::createCommand(ByteStream& bs)
{
bs >> fBOP;
Command::createCommand(bs);
bs >> fBOP;
Command::createCommand(bs);
}
void FilterCommand::resetCommand(ByteStream &bs)
void FilterCommand::resetCommand(ByteStream& bs)
{
}
@ -204,14 +211,14 @@ void FilterCommand::project()
{
}
void FilterCommand::projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col)
void FilterCommand::projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col)
{
}
uint64_t FilterCommand::getLBID()
{
return 0;
return 0;
}
@ -222,102 +229,108 @@ void FilterCommand::nextLBID()
SCommand FilterCommand::duplicate()
{
SCommand ret;
FilterCommand* filterCmd;
SCommand ret;
FilterCommand* filterCmd;
ret.reset(new FilterCommand());
filterCmd = (FilterCommand *) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->leftColType = leftColType;
filterCmd->rightColType = rightColType;
filterCmd->Command::duplicate(this);
return ret;
ret.reset(new FilterCommand());
filterCmd = (FilterCommand*) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->leftColType = leftColType;
filterCmd->rightColType = rightColType;
filterCmd->Command::duplicate(this);
return ret;
}
void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& left,
const execplan::CalpontSystemCatalog::ColType& right)
const execplan::CalpontSystemCatalog::ColType& right)
{
leftColType = left;
rightColType = right;
leftColType = left;
rightColType = right;
}
void FilterCommand::doFilter()
{
bpp->ridMap = 0;
bpp->ridCount = 0;
bpp->ridMap = 0;
bpp->ridCount = 0;
// rids in [0] is used for scan [1], so [1] is a subset of [0], and same order.
// -- see makeFilterCommand() above.
for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1]; )
{
if (bpp->fFiltCmdRids[0][i] != bpp->fFiltCmdRids[1][j])
{
i++;
}
else
{
if (compare(i,j) == true)
{
bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i];
bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i];
bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 10);
bpp->ridCount++;
}
// rids in [0] is used for scan [1], so [1] is a subset of [0], and same order.
// -- see makeFilterCommand() above.
for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1]; )
{
if (bpp->fFiltCmdRids[0][i] != bpp->fFiltCmdRids[1][j])
{
i++;
}
else
{
if (compare(i, j) == true)
{
bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i];
bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i];
bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 10);
bpp->ridCount++;
}
i++;
j++;
}
}
i++;
j++;
}
}
// bug 1247 -- reset the rid count
bpp->fFiltRidCount[0] = bpp->fFiltRidCount[1] = 0;
// bug 1247 -- reset the rid count
bpp->fFiltRidCount[0] = bpp->fFiltRidCount[1] = 0;
}
bool FilterCommand::compare(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
switch(fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i] > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i] < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i] == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i] >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i] <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i] != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i] > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i] < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i] == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i] >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i] <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i] != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
}
bool FilterCommand::operator==(const FilterCommand& c) const
{
return (fBOP == c.fBOP);
return (fBOP == c.fBOP);
}
bool FilterCommand::operator!=(const FilterCommand& c) const
{
return !(*this == c);
return !(*this == c);
}
@ -334,72 +347,78 @@ ScaledFilterCmd::~ScaledFilterCmd()
SCommand ScaledFilterCmd::duplicate()
{
SCommand ret;
ScaledFilterCmd* filterCmd;
SCommand ret;
ScaledFilterCmd* filterCmd;
ret.reset(new ScaledFilterCmd());
filterCmd = (ScaledFilterCmd *) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fFactor = fFactor;
ret.reset(new ScaledFilterCmd());
filterCmd = (ScaledFilterCmd*) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fFactor = fFactor;
return ret;
return ret;
}
bool ScaledFilterCmd::compare(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
switch(fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i]*fFactor > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i]*fFactor < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i]*fFactor == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i]*fFactor >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i]*fFactor <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i]*fFactor != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return bpp->fFiltCmdValues[0][i] * fFactor > bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltCmdValues[0][i] * fFactor < bpp->fFiltCmdValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltCmdValues[0][i] * fFactor == bpp->fFiltCmdValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltCmdValues[0][i] * fFactor >= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltCmdValues[0][i] * fFactor <= bpp->fFiltCmdValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltCmdValues[0][i] * fFactor != bpp->fFiltCmdValues[1][j];
break;
default:
return false;
break;
}
}
void ScaledFilterCmd::setFactor(double f)
{
fFactor = f;
fFactor = f;
}
double ScaledFilterCmd::factor()
{
return fFactor;
return fFactor;
}
bool ScaledFilterCmd::operator==(const ScaledFilterCmd& c) const
{
return ((fBOP == c.fBOP) && (fFactor == c.fFactor));
return ((fBOP == c.fBOP) && (fFactor == c.fFactor));
}
bool ScaledFilterCmd::operator!=(const ScaledFilterCmd& c) const
{
return !(*this == c);
return !(*this == c);
}
@ -416,216 +435,241 @@ StrFilterCmd::~StrFilterCmd()
SCommand StrFilterCmd::duplicate()
{
SCommand ret;
StrFilterCmd* filterCmd;
SCommand ret;
StrFilterCmd* filterCmd;
ret.reset(new StrFilterCmd());
filterCmd = (StrFilterCmd *) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fCompare = fCompare;
filterCmd->fCharLength = fCharLength;
ret.reset(new StrFilterCmd());
filterCmd = (StrFilterCmd*) ret.get();
filterCmd->fBOP = fBOP;
filterCmd->fCompare = fCompare;
filterCmd->fCharLength = fCharLength;
return ret;
return ret;
}
void StrFilterCmd::execute()
{
doFilter();
doFilter();
}
bool StrFilterCmd::compare(uint64_t i, uint64_t j)
{
return (this->*fCompare)(i, j);
return (this->*fCompare)(i, j);
}
void StrFilterCmd::setCompareFunc(uint32_t columns)
{
if (columns == CC) // char[] : char
{
fCompare = &StrFilterCmd::compare_cc;
}
else if (columns == DCDC) // string : string
{
fCompare = &StrFilterCmd::compare_ss;
}
else if (columns == DCC) // char[] : string
{
fCompare = &StrFilterCmd::compare_cs;
}
else if (columns == CDC) // string : char[]
{
fCompare = &StrFilterCmd::compare_sc;
}
else
{
stringstream msg;
msg << "StrFilterCmd: unhandled column combination " << hex << columns << dec;
throw logic_error(msg.str());
}
if (columns == CC) // char[] : char
{
fCompare = &StrFilterCmd::compare_cc;
}
else if (columns == DCDC) // string : string
{
fCompare = &StrFilterCmd::compare_ss;
}
else if (columns == DCC) // char[] : string
{
fCompare = &StrFilterCmd::compare_cs;
}
else if (columns == CDC) // string : char[]
{
fCompare = &StrFilterCmd::compare_sc;
}
else
{
stringstream msg;
msg << "StrFilterCmd: unhandled column combination " << hex << columns << dec;
throw logic_error(msg.str());
}
}
bool StrFilterCmd::compare_cc(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
switch(fBOP)
{
case COMPARE_GT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) > uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) < uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_EQ:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) == uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_GE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) >= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) <= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_NE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) != uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) > uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LT:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) < uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_EQ:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) == uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_GE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) >= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_LE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) <= uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
case COMPARE_NE:
return uint64ToStr(bpp->fFiltCmdValues[0][i]) != uint64ToStr(bpp->fFiltCmdValues[1][j]);
break;
default:
return false;
break;
}
}
bool StrFilterCmd::compare_ss(uint64_t i, uint64_t j)
{
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[1][j] == "" ||
bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[1][j] == "" ||
bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
switch(fBOP)
{
case COMPARE_GT:
return bpp->fFiltStrValues[0][i] > bpp->fFiltStrValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltStrValues[0][i] < bpp->fFiltStrValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltStrValues[0][i] == bpp->fFiltStrValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltStrValues[0][i] >= bpp->fFiltStrValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltStrValues[0][i] <= bpp->fFiltStrValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltStrValues[0][i] != bpp->fFiltStrValues[1][j];
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return bpp->fFiltStrValues[0][i] > bpp->fFiltStrValues[1][j];
break;
case COMPARE_LT:
return bpp->fFiltStrValues[0][i] < bpp->fFiltStrValues[1][j];
break;
case COMPARE_EQ:
return bpp->fFiltStrValues[0][i] == bpp->fFiltStrValues[1][j];
break;
case COMPARE_GE:
return bpp->fFiltStrValues[0][i] >= bpp->fFiltStrValues[1][j];
break;
case COMPARE_LE:
return bpp->fFiltStrValues[0][i] <= bpp->fFiltStrValues[1][j];
break;
case COMPARE_NE:
return bpp->fFiltStrValues[0][i] != bpp->fFiltStrValues[1][j];
break;
default:
return false;
break;
}
}
bool StrFilterCmd::compare_cs(uint64_t i, uint64_t j)
{
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
bpp->fFiltStrValues[1][j] == "" || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType) ||
bpp->fFiltStrValues[1][j] == "" || bpp->fFiltStrValues[1][j] == joblist::CPNULLSTRMARK)
return false;
int cmp = strncmp(reinterpret_cast<const char*>(&bpp->fFiltCmdValues[0][i]),
bpp->fFiltStrValues[1][j].c_str(), fCharLength);
switch(fBOP)
{
case COMPARE_GT:
return (cmp > 0);
break;
case COMPARE_LT:
return (cmp < 0 || (cmp == 0 && fCharLength < bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_EQ:
return (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length());
break;
case COMPARE_GE:
return (cmp > 0 || (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_LE:
return (cmp <= 0);
break;
case COMPARE_NE:
return (cmp != 0 || fCharLength < bpp->fFiltStrValues[1][j].length());
break;
default:
return false;
break;
}
int cmp = strncmp(reinterpret_cast<const char*>(&bpp->fFiltCmdValues[0][i]),
bpp->fFiltStrValues[1][j].c_str(), fCharLength);
switch (fBOP)
{
case COMPARE_GT:
return (cmp > 0);
break;
case COMPARE_LT:
return (cmp < 0 || (cmp == 0 && fCharLength < bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_EQ:
return (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length());
break;
case COMPARE_GE:
return (cmp > 0 || (cmp == 0 && fCharLength >= bpp->fFiltStrValues[1][j].length()));
break;
case COMPARE_LE:
return (cmp <= 0);
break;
case COMPARE_NE:
return (cmp != 0 || fCharLength < bpp->fFiltStrValues[1][j].length());
break;
default:
return false;
break;
}
}
bool StrFilterCmd::compare_sc(uint64_t i, uint64_t j)
{
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
if (bpp->fFiltStrValues[0][i] == "" || bpp->fFiltStrValues[0][i] == joblist::CPNULLSTRMARK ||
execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType))
return false;
int cmp = strncmp(bpp->fFiltStrValues[0][i].c_str(),
reinterpret_cast<const char*>(&bpp->fFiltCmdValues[1][j]), fCharLength);
int cmp = strncmp(bpp->fFiltStrValues[0][i].c_str(),
reinterpret_cast<const char*>(&bpp->fFiltCmdValues[1][j]), fCharLength);
switch(fBOP)
{
case COMPARE_GT:
return (cmp > 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() > fCharLength));
break;
case COMPARE_LT:
return (cmp < 0);
break;
case COMPARE_EQ:
return (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength);
break;
case COMPARE_GE:
return (cmp >= 0);
break;
case COMPARE_LE:
return (cmp < 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength));
break;
case COMPARE_NE:
return (cmp != 0 || bpp->fFiltStrValues[0][i].length() > fCharLength);
break;
default:
return false;
break;
}
switch (fBOP)
{
case COMPARE_GT:
return (cmp > 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() > fCharLength));
break;
case COMPARE_LT:
return (cmp < 0);
break;
case COMPARE_EQ:
return (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength);
break;
case COMPARE_GE:
return (cmp >= 0);
break;
case COMPARE_LE:
return (cmp < 0 || (cmp == 0 && bpp->fFiltStrValues[0][i].length() <= fCharLength));
break;
case COMPARE_NE:
return (cmp != 0 || bpp->fFiltStrValues[0][i].length() > fCharLength);
break;
default:
return false;
break;
}
}
void StrFilterCmd::setCharLength(size_t l)
{
fCharLength = l;
fCharLength = l;
}
size_t StrFilterCmd::charLength()
{
return fCharLength;
return fCharLength;
}
bool StrFilterCmd::operator==(const StrFilterCmd& c) const
{
return ((fBOP == c.fBOP) && fCharLength == c.fCharLength);
return ((fBOP == c.fBOP) && fCharLength == c.fCharLength);
}
bool StrFilterCmd::operator!=(const StrFilterCmd& c) const
{
return !(*this == c);
return !(*this == c);
}

165
primitives/primproc/filtercommand.h
View File

@ -39,119 +39,122 @@ namespace primitiveprocessor
class FilterCommand : public Command
{
public:
FilterCommand();
virtual ~FilterCommand();
public:
FilterCommand();
virtual ~FilterCommand();
// returns a FilterCommand based on column types
static Command* makeFilterCommand(messageqcpp::ByteStream&, std::vector<SCommand>& cmds);
// returns a FilterCommand based on column types
static Command* makeFilterCommand(messageqcpp::ByteStream&, std::vector<SCommand>& cmds);
// virtuals from base class -- Command
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
void prep(int8_t outputType, bool makeAbsRids);
// virtuals from base class -- Command
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
void prep(int8_t outputType, bool makeAbsRids);
void setColTypes(const execplan::CalpontSystemCatalog::ColType& left,
const execplan::CalpontSystemCatalog::ColType& right);
void setColTypes(const execplan::CalpontSystemCatalog::ColType& left,
const execplan::CalpontSystemCatalog::ColType& right);
// operator override
bool operator==(const FilterCommand&) const;
bool operator!=(const FilterCommand&) const;
// operator override
bool operator==(const FilterCommand&) const;
bool operator!=(const FilterCommand&) const;
int getCompType() const { return 0; }
int getCompType() const
{
return 0;
}
protected:
// filter operation
virtual void doFilter();
protected:
// filter operation
virtual void doFilter();
// compare method, take the indices to the values array
virtual bool compare(uint64_t, uint64_t);
// compare method, take the indices to the values array
virtual bool compare(uint64_t, uint64_t);
// binary operator
uint8_t fBOP;
// binary operator
uint8_t fBOP;
// column type for null check
execplan::CalpontSystemCatalog::ColType leftColType;
execplan::CalpontSystemCatalog::ColType rightColType;
// column type for null check
execplan::CalpontSystemCatalog::ColType leftColType;
execplan::CalpontSystemCatalog::ColType rightColType;
private:
// disabled copy constructor and operator
FilterCommand(const FilterCommand&);
FilterCommand& operator=(const FilterCommand&);
private:
// disabled copy constructor and operator
FilterCommand(const FilterCommand&);
FilterCommand& operator=(const FilterCommand&);
};
class ScaledFilterCmd : public FilterCommand
{
public:
ScaledFilterCmd();
virtual ~ScaledFilterCmd();
SCommand duplicate();
public:
ScaledFilterCmd();
virtual ~ScaledFilterCmd();
SCommand duplicate();
void setFactor(double);
double factor();
void setFactor(double);
double factor();
// operator override
bool operator==(const ScaledFilterCmd&) const;
bool operator!=(const ScaledFilterCmd&) const;
// operator override
bool operator==(const ScaledFilterCmd&) const;
bool operator!=(const ScaledFilterCmd&) const;
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
// value used in comparison;
double fFactor;
// value used in comparison;
double fFactor;
private:
// disabled copy constructor and operator
ScaledFilterCmd(const ScaledFilterCmd &);
ScaledFilterCmd& operator=(const ScaledFilterCmd &);
private:
// disabled copy constructor and operator
ScaledFilterCmd(const ScaledFilterCmd&);
ScaledFilterCmd& operator=(const ScaledFilterCmd&);
};
class StrFilterCmd : public FilterCommand
{
public:
StrFilterCmd();
virtual ~StrFilterCmd();
public:
StrFilterCmd();
virtual ~StrFilterCmd();
// override FilterCommand methods
void execute();
SCommand duplicate();
// override FilterCommand methods
void execute();
SCommand duplicate();
void setCompareFunc(uint32_t);
void setCharLength(size_t);
size_t charLength();
void setCompareFunc(uint32_t);
void setCharLength(size_t);
size_t charLength();
// operator override
bool operator==(const StrFilterCmd&) const;
bool operator!=(const StrFilterCmd&) const;
// operator override
bool operator==(const StrFilterCmd&) const;
bool operator!=(const StrFilterCmd&) const;
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
protected:
// compare method, take the indices to the values array
bool compare(uint64_t, uint64_t);
// compare method for differernt column combination, c--char[], s--string
// compare char[]-char[] is not the same as int-int due to endian issue.
bool compare_cc(uint64_t, uint64_t);
bool compare_ss(uint64_t, uint64_t);
bool compare_cs(uint64_t, uint64_t);
bool compare_sc(uint64_t, uint64_t);
bool (StrFilterCmd::*fCompare)(uint64_t, uint64_t);
// compare method for differernt column combination, c--char[], s--string
// compare char[]-char[] is not the same as int-int due to endian issue.
bool compare_cc(uint64_t, uint64_t);
bool compare_ss(uint64_t, uint64_t);
bool compare_cs(uint64_t, uint64_t);
bool compare_sc(uint64_t, uint64_t);
bool (StrFilterCmd::*fCompare)(uint64_t, uint64_t);
// colWidth of columns the don't need a dictionary
size_t fCharLength;
// colWidth of columns the don't need a dictionary
size_t fCharLength;
private:
// disabled copy constructor and operator
StrFilterCmd(const StrFilterCmd &);
StrFilterCmd& operator=(const StrFilterCmd &);
private:
// disabled copy constructor and operator
StrFilterCmd(const StrFilterCmd&);
StrFilterCmd& operator=(const StrFilterCmd&);
};

66
primitives/primproc/logger.cpp
View File

@ -33,53 +33,55 @@ namespace primitiveprocessor
{
Logger::Logger() :
fMl1(LoggingID(28))
fMl1(LoggingID(28))
{
fMsgMap[logging::M0000] = Message(logging::M0000);
fMsgMap[logging::M0016] = Message(logging::M0016);
fMsgMap[logging::M0045] = Message(logging::M0045);
fMsgMap[logging::M0053] = Message(logging::M0053);
fMsgMap[logging::M0058] = Message(logging::M0058);
fMsgMap[logging::M0061] = Message(logging::M0061);
fMsgMap[logging::M0069] = Message(logging::M0069);
fMsgMap[logging::M0070] = Message(logging::M0070);
fMsgMap[logging::M0077] = Message(logging::M0077);
fMsgMap[logging::M0000] = Message(logging::M0000);
fMsgMap[logging::M0016] = Message(logging::M0016);
fMsgMap[logging::M0045] = Message(logging::M0045);
fMsgMap[logging::M0053] = Message(logging::M0053);
fMsgMap[logging::M0058] = Message(logging::M0058);
fMsgMap[logging::M0061] = Message(logging::M0061);
fMsgMap[logging::M0069] = Message(logging::M0069);
fMsgMap[logging::M0070] = Message(logging::M0070);
fMsgMap[logging::M0077] = Message(logging::M0077);
}
void Logger::logMessage(const Message::MessageID mid,
const Message::Args& args,
bool critical)
{
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
msgIter->second.reset();
msgIter->second.format(args);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
if (critical)
{
fMl1.logCriticalMessage(msgIter->second);
}
else
{
fMl1.logWarningMessage(msgIter->second);
}
msgIter->second.reset();
msgIter->second.format(args);
if (critical)
{
fMl1.logCriticalMessage(msgIter->second);
}
else
{
fMl1.logWarningMessage(msgIter->second);
}
}
void Logger::logInfoMessage(const Message::MessageID mid,
const Message::Args& args)
const Message::Args& args)
{
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
msgIter->second.reset();
msgIter->second.format(args);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
fMl1.logInfoMessage(msgIter->second);
msgIter->second.reset();
msgIter->second.format(args);
fMl1.logInfoMessage(msgIter->second);
}
}

183
primitives/primproc/passthrucommand.cpp
View File

@ -49,10 +49,11 @@ PassThruCommand::~PassThruCommand()
void PassThruCommand::prep(int8_t outputType, bool makeAbsRids)
{
if (bpp->ot == ROW_GROUP) {
bpp->outputRG.initRow(&r);
rowSize = r.getSize();
}
if (bpp->ot == ROW_GROUP)
{
bpp->outputRG.initRow(&r);
rowSize = r.getSize();
}
}
void PassThruCommand::execute()
@ -62,118 +63,142 @@ void PassThruCommand::execute()
void PassThruCommand::project()
{
uint32_t i;
uint32_t i;
*bpp->serialized << (uint32_t) (bpp->ridCount * colWidth);
*bpp->serialized << (uint32_t) (bpp->ridCount * colWidth);
#if 0
cout << "pass thru serializing " << (uint32_t) (bpp->ridCount * colWidth) << " bytes:\n";
cout << "at relative position " << bpp->serialized->length() - sizeof(ISMPacketHeader) - sizeof(PrimitiveHeader) - 4 << endl;
for (i = 0; i < bpp->ridCount; i++)
cout << " " << i << ": " << bpp->values[i] << endl;
cout << "pass thru serializing " << (uint32_t) (bpp->ridCount * colWidth) << " bytes:\n";
cout << "at relative position " << bpp->serialized->length() - sizeof(ISMPacketHeader) - sizeof(PrimitiveHeader) - 4 << endl;
for (i = 0; i < bpp->ridCount; i++)
cout << " " << i << ": " << bpp->values[i] << endl;
#endif
switch (colWidth) {
case 8:
bpp->serialized->append((uint8_t *) bpp->values, bpp->ridCount << 3);
break;
case 4:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint32_t) bpp->values[i];
break;
case 2:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint16_t) bpp->values[i];
break;
case 1:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint8_t) bpp->values[i];
break;
default:
throw logic_error("PassThruCommand has a bad column width");
}
switch (colWidth)
{
case 8:
bpp->serialized->append((uint8_t*) bpp->values, bpp->ridCount << 3);
break;
case 4:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint32_t) bpp->values[i];
break;
case 2:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint16_t) bpp->values[i];
break;
case 1:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint8_t) bpp->values[i];
break;
default:
throw logic_error("PassThruCommand has a bad column width");
}
}
void PassThruCommand::projectIntoRowGroup(RowGroup &rg, uint32_t col)
void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col)
{
uint32_t i;
uint32_t i;
rg.initRow(&r);
rg.getRow(0, &r);
uint32_t offset = r.getOffset(col);
rowSize = r.getSize();
rg.initRow(&r);
rg.getRow(0, &r);
uint32_t offset = r.getOffset(col);
rowSize = r.getSize();
switch (colWidth) {
case 1:
for (i = 0; i < bpp->ridCount; i++) {
switch (colWidth)
{
case 1:
for (i = 0; i < bpp->ridCount; i++)
{
// cout << "PTC: " << bpp->values[i] << endl;
r.setUintField_offset<1>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 2:
for (i = 0; i < bpp->ridCount; i++) {
r.setUintField_offset<1>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 2:
for (i = 0; i < bpp->ridCount; i++)
{
// cout << "PTC: " << bpp->values[i] << endl;
r.setUintField_offset<2>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 4:
for (i = 0; i < bpp->ridCount; i++) {
r.setUintField_offset<4>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 8:
for (i = 0; i < bpp->ridCount; i++) {
r.setUintField_offset<2>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 4:
for (i = 0; i < bpp->ridCount; i++)
{
r.setUintField_offset<4>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
case 8:
for (i = 0; i < bpp->ridCount; i++)
{
// cout << "PTC: " << bpp->values[i] << endl;
r.setUintField_offset<8>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
}
r.setUintField_offset<8>(bpp->values[i], offset);
r.nextRow(rowSize);
}
break;
}
}
uint64_t PassThruCommand::getLBID()
{
return 0;
return 0;
}
void PassThruCommand::nextLBID()
{
}
void PassThruCommand::createCommand(ByteStream &bs)
void PassThruCommand::createCommand(ByteStream& bs)
{
bs.advance(1);
bs >> colWidth;
Command::createCommand(bs);
bs >> colWidth;
Command::createCommand(bs);
}
void PassThruCommand::resetCommand(ByteStream &bs)
void PassThruCommand::resetCommand(ByteStream& bs)
{
}
SCommand PassThruCommand::duplicate()
{
SCommand ret;
PassThruCommand *p;
SCommand ret;
PassThruCommand* p;
ret.reset(new PassThruCommand());
p = (PassThruCommand *) ret.get();
p->colWidth = colWidth;
p->Command::duplicate(this);
return ret;
ret.reset(new PassThruCommand());
p = (PassThruCommand*) ret.get();
p->colWidth = colWidth;
p->Command::duplicate(this);
return ret;
}
bool PassThruCommand::operator==(const PassThruCommand &p) const
bool PassThruCommand::operator==(const PassThruCommand& p) const
{
if (colWidth != p.colWidth)
return false;
return true;
if (colWidth != p.colWidth)
return false;
return true;
}
bool PassThruCommand::operator!=(const PassThruCommand &p) const
bool PassThruCommand::operator!=(const PassThruCommand& p) const
{
return !(*this == p);
return !(*this == p);
}
};

47
primitives/primproc/passthrucommand.h
View File

@ -19,7 +19,7 @@
// $Id: passthrucommand.h 2035 2013-01-21 14:12:19Z rdempsey $
// C++ Interface: passthrucommand
//
// Description:
// Description:
//
//
// Author: Patrick <pleblanc@localhost.localdomain>, (C) 2008
@ -38,31 +38,34 @@ namespace primitiveprocessor
class PassThruCommand : public Command
{
public:
PassThruCommand();
virtual ~PassThruCommand();
public:
PassThruCommand();
virtual ~PassThruCommand();
void prep(int8_t outputType, bool makeAbsRids);
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
SCommand duplicate();
bool operator==(const PassThruCommand &) const;
bool operator!=(const PassThruCommand &) const;
void prep(int8_t outputType, bool makeAbsRids);
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
bool operator==(const PassThruCommand&) const;
bool operator!=(const PassThruCommand&) const;
int getCompType() const { return 0; }
private:
PassThruCommand(const PassThruCommand &);
int getCompType() const
{
return 0;
}
private:
PassThruCommand(const PassThruCommand&);
uint8_t colWidth;
uint8_t colWidth;
/* Minor optimization for projectIntoRowGroup() */
rowgroup::Row r;
uint32_t rowSize;
/* Minor optimization for projectIntoRowGroup() */
rowgroup::Row r;
uint32_t rowSize;
};
}

38
primitives/primproc/pp_logger.h
View File

@ -37,32 +37,32 @@ namespace primitiveprocessor
class Logger
{
public:
Logger();
Logger();
// critical=true logs CRITICAL; critical=false logs WARNING
void logMessage(const logging::Message::MessageID mid,
// critical=true logs CRITICAL; critical=false logs WARNING
void logMessage(const logging::Message::MessageID mid,
const logging::Message::Args& args,
bool critical=false);
void logInfoMessage(const logging::Message::MessageID mid,
const logging::Message::Args& args);
bool critical = false);
void logInfoMessage(const logging::Message::MessageID mid,
const logging::Message::Args& args);
void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 )
{
logging::Message::Args args;
args.add(msg);
logMessage(mid, args, critical);
}
void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 )
{
logging::Message::Args args;
args.add(msg);
logMessage(mid, args, critical);
}
private:
// defaults okay
//Logger(const Logger& rhs);
//Logger& operator=(const Logger& rhs);
// defaults okay
//Logger(const Logger& rhs);
//Logger& operator=(const Logger& rhs);
typedef std::map<logging::Message::MessageID, logging::Message> MsgMap;
typedef std::map<logging::Message::MessageID, logging::Message> MsgMap;
MsgMap fMsgMap;
boost::mutex fLogLock;
logging::MessageLog fMl1;
MsgMap fMsgMap;
boost::mutex fLogLock;
logging::MessageLog fMl1;
};

3681
primitives/primproc/primitiveserver.cpp Executable file → Normal file
View File

File diff suppressed because it is too large Load Diff

230
primitives/primproc/primitiveserver.h
View File

@ -47,131 +47,153 @@
#endif
#include "oamcache.h"
extern oam::OamCache *oamCache;
extern oam::OamCache* oamCache;
namespace primitiveprocessor
{
extern boost::shared_ptr<threadpool::PriorityThreadPool> OOBPool;
extern dbbc::BlockRequestProcessor **BRPp;
extern BRM::DBRM *brm;
extern boost::mutex bppLock;
extern uint32_t highPriorityThreads, medPriorityThreads, lowPriorityThreads;
extern boost::shared_ptr<threadpool::PriorityThreadPool> OOBPool;
extern dbbc::BlockRequestProcessor** BRPp;
extern BRM::DBRM* brm;
extern boost::mutex bppLock;
extern uint32_t highPriorityThreads, medPriorityThreads, lowPriorityThreads;
#ifdef PRIMPROC_STOPWATCH
extern map<pthread_t, logging::StopWatch*> stopwatchMap;
extern pthread_mutex_t stopwatchMapMutex;
extern bool stopwatchThreadCreated;
extern map<pthread_t, logging::StopWatch*> stopwatchMap;
extern pthread_mutex_t stopwatchMapMutex;
extern bool stopwatchThreadCreated;
extern void pause_(int seconds);
extern void *autoFinishStopwatchThread(void *arg);
extern void pause_(int seconds);
extern void* autoFinishStopwatchThread(void* arg);
#endif
class BPPV {
public:
BPPV();
~BPPV();
boost::shared_ptr<BatchPrimitiveProcessor> next();
void add(boost::shared_ptr<BatchPrimitiveProcessor> a);
const std::vector<boost::shared_ptr<BatchPrimitiveProcessor> > & get();
inline boost::shared_ptr<BPPSendThread> getSendThread() { return sendThread; }
void abort();
bool aborted();
volatile bool joinDataReceived;
private:
std::vector<boost::shared_ptr<BatchPrimitiveProcessor> > v;
boost::shared_ptr<BPPSendThread> sendThread;
class BPPV
{
public:
BPPV();
~BPPV();
boost::shared_ptr<BatchPrimitiveProcessor> next();
void add(boost::shared_ptr<BatchPrimitiveProcessor> a);
const std::vector<boost::shared_ptr<BatchPrimitiveProcessor> >& get();
inline boost::shared_ptr<BPPSendThread> getSendThread()
{
return sendThread;
}
void abort();
bool aborted();
volatile bool joinDataReceived;
private:
std::vector<boost::shared_ptr<BatchPrimitiveProcessor> > v;
boost::shared_ptr<BPPSendThread> sendThread;
// the instance other instances are created from
boost::shared_ptr<BatchPrimitiveProcessor> unusedInstance;
// the instance other instances are created from
boost::shared_ptr<BatchPrimitiveProcessor> unusedInstance;
// pos keeps the position of the last BPP returned by next(),
// next() will start searching at this pos on the next call.
uint32_t pos;
};
// pos keeps the position of the last BPP returned by next(),
// next() will start searching at this pos on the next call.
uint32_t pos;
};
typedef boost::shared_ptr<BPPV> SBPPV;
typedef std::map<uint32_t, SBPPV> BPPMap;
extern BPPMap bppMap;
typedef boost::shared_ptr<BPPV> SBPPV;
typedef std::map<uint32_t, SBPPV> BPPMap;
extern BPPMap bppMap;
void prefetchBlocks(uint64_t lbid, const int compType, uint32_t* rCount);
void prefetchExtent(uint64_t lbid, uint32_t ver, uint32_t txn, uint32_t* rCount);
void loadBlock(uint64_t lbid, BRM::QueryContext q, uint32_t txn, int compType, void* bufferPtr,
bool* pWasBlockInCache, uint32_t* rCount=NULL, bool LBIDTrace = false,
uint32_t sessionID = 0, bool doPrefetch=true, VSSCache *vssCache = NULL);
void loadBlockAsync(uint64_t lbid, const BRM::QueryContext &q, uint32_t txn, int CompType,
uint32_t *cCount, uint32_t *rCount, bool LBIDTrace, uint32_t sessionID,
boost::mutex *m, uint32_t *busyLoaders, boost::shared_ptr<BPPSendThread> sendThread, VSSCache* vssCache=0);
uint32_t loadBlocks(BRM::LBID_t *lbids, BRM::QueryContext q, BRM::VER_t txn, int compType,
uint8_t **bufferPtrs, uint32_t *rCount, bool LBIDTrace, uint32_t sessionID,
uint32_t blockCount, bool *wasVersioned, bool doPrefetch = true, VSSCache *vssCache = NULL);
uint32_t cacheNum(uint64_t lbid);
void buildFileName(BRM::OID_t oid, char* fileName);
void prefetchBlocks(uint64_t lbid, const int compType, uint32_t* rCount);
void prefetchExtent(uint64_t lbid, uint32_t ver, uint32_t txn, uint32_t* rCount);
void loadBlock(uint64_t lbid, BRM::QueryContext q, uint32_t txn, int compType, void* bufferPtr,
bool* pWasBlockInCache, uint32_t* rCount = NULL, bool LBIDTrace = false,
uint32_t sessionID = 0, bool doPrefetch = true, VSSCache* vssCache = NULL);
void loadBlockAsync(uint64_t lbid, const BRM::QueryContext& q, uint32_t txn, int CompType,
uint32_t* cCount, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID,
boost::mutex* m, uint32_t* busyLoaders, boost::shared_ptr<BPPSendThread> sendThread, VSSCache* vssCache = 0);
uint32_t loadBlocks(BRM::LBID_t* lbids, BRM::QueryContext q, BRM::VER_t txn, int compType,
uint8_t** bufferPtrs, uint32_t* rCount, bool LBIDTrace, uint32_t sessionID,
uint32_t blockCount, bool* wasVersioned, bool doPrefetch = true, VSSCache* vssCache = NULL);
uint32_t cacheNum(uint64_t lbid);
void buildFileName(BRM::OID_t oid, char* fileName);
/** @brief process primitives as they arrive
*/
class PrimitiveServer
{
public:
/** @brief ctor
*/
PrimitiveServer(int serverThreads,
int serverQueueSize,
int processorWeight,
int processorQueueSize,
bool rotatingDestination,
uint32_t BRPBlocks=(1024 * 1024 * 2),
int BRPThreads=64,
int cacheCount = 8,
int maxBlocksPerRead = 128,
int readAheadBlocks=256,
uint32_t deleteBlocks = 0,
bool ptTrace=false,
double prefetchThreshold = 0,
uint64_t pmSmallSide = 0);
/** @brief process primitives as they arrive
*/
class PrimitiveServer
{
public:
/** @brief ctor
*/
PrimitiveServer(int serverThreads,
int serverQueueSize,
int processorWeight,
int processorQueueSize,
bool rotatingDestination,
uint32_t BRPBlocks = (1024 * 1024 * 2),
int BRPThreads = 64,
int cacheCount = 8,
int maxBlocksPerRead = 128,
int readAheadBlocks = 256,
uint32_t deleteBlocks = 0,
bool ptTrace = false,
double prefetchThreshold = 0,
uint64_t pmSmallSide = 0);
/** @brief dtor
*/
~PrimitiveServer();
/** @brief dtor
*/
~PrimitiveServer();
/** @brief start the primitive server
*
*/
void start();
/** @brief start the primitive server
*
*/
void start();
/** @brief get a pointer the shared processor thread pool
*/
inline boost::shared_ptr<threadpool::PriorityThreadPool> getProcessorThreadPool() const { return fProcessorPool; }
/** @brief get a pointer the shared processor thread pool
*/
inline boost::shared_ptr<threadpool::PriorityThreadPool> getProcessorThreadPool() const
{
return fProcessorPool;
}
// int fCacheCount;
const int ReadAheadBlocks() const {return fReadAheadBlocks;}
bool rotatingDestination() const {return fRotatingDestination;}
bool PTTrace() const {return fPTTrace;}
double prefetchThreshold() const { return fPrefetchThreshold; }
uint32_t ProcessorThreads() const { return highPriorityThreads + medPriorityThreads + lowPriorityThreads; }
protected:
const int ReadAheadBlocks() const
{
return fReadAheadBlocks;
}
bool rotatingDestination() const
{
return fRotatingDestination;
}
bool PTTrace() const
{
return fPTTrace;
}
double prefetchThreshold() const
{
return fPrefetchThreshold;
}
uint32_t ProcessorThreads() const
{
return highPriorityThreads + medPriorityThreads + lowPriorityThreads;
}
protected:
private:
/** @brief the thread pool used to listen for
* incoming primitive commands
*/
threadpool::ThreadPool fServerpool;
private:
/** @brief the thread pool used to listen for
* incoming primitive commands
*/
threadpool::ThreadPool fServerpool;
/** @brief the thread pool used to process
* primitive commands
*/
boost::shared_ptr<threadpool::PriorityThreadPool> fProcessorPool;
/** @brief the thread pool used to process
* primitive commands
*/
boost::shared_ptr<threadpool::PriorityThreadPool> fProcessorPool;
int fServerThreads;
int fServerQueueSize;
int fProcessorWeight;
int fProcessorQueueSize;
int fMaxBlocksPerRead;
int fReadAheadBlocks;
bool fRotatingDestination;
bool fPTTrace;
double fPrefetchThreshold;
uint64_t fPMSmallSide;
};
int fServerThreads;
int fServerQueueSize;
int fProcessorWeight;
int fProcessorQueueSize;
int fMaxBlocksPerRead;
int fReadAheadBlocks;
bool fRotatingDestination;
bool fPTTrace;
double fPrefetchThreshold;
uint64_t fPMSmallSide;
};
} // namespace primitiveprocessor

843
primitives/primproc/primproc.cpp
View File

File diff suppressed because it is too large Load Diff

32
primitives/primproc/primproc.h
View File

@ -94,25 +94,25 @@ namespace primitiveprocessor
std::cout << message1 << message2 << message3 << std::endl; \
}
enum DebugLevel /** @brief Debug level type enumeration */
{
NONE = 0, /** @brief No debug info */
STATS = 1, /** @brief stats info */
SUMMARY = 2, /** @brief Summary level debug info */
DETAIL = 3, /** @brief A little detail debug info */
VERBOSE = 4, /** @brief Detailed debug info */
};
enum DebugLevel /** @brief Debug level type enumeration */
{
NONE = 0, /** @brief No debug info */
STATS = 1, /** @brief stats info */
SUMMARY = 2, /** @brief Summary level debug info */
DETAIL = 3, /** @brief A little detail debug info */
VERBOSE = 4, /** @brief Detailed debug info */
};
bool isDebug( const DebugLevel level );
bool isDebug( const DebugLevel level );
const int MAX_BUFFER_SIZE = 32768 * 2;
const int MAX_BUFFER_SIZE = 32768 * 2;
// message log globals
//const logging::LoggingID lid1(28);
//extern logging::Message msg16;
//extern logging::MessageLog ml1;
//extern boost::mutex logLock;
extern Logger* mlp;
// message log globals
//const logging::LoggingID lid1(28);
//extern logging::Message msg16;
//extern logging::MessageLog ml1;
//extern boost::mutex logLock;
extern Logger* mlp;
}
#endif //PRIMPROC_H

368
primitives/primproc/pseudocc.cpp
View File

@ -22,7 +22,8 @@
using namespace std;
using namespace execplan;
namespace primitiveprocessor {
namespace primitiveprocessor
{
PseudoCC::PseudoCC() : ColumnCommand()
{
@ -34,29 +35,30 @@ PseudoCC::~PseudoCC()
SCommand PseudoCC::duplicate()
{
SCommand ret;
PseudoCC *pseudo;
SCommand ret;
PseudoCC* pseudo;
pseudo = new PseudoCC();
ret.reset(pseudo);
pseudo->function = function;
pseudo->valueFromUM = valueFromUM;
ColumnCommand::duplicate(pseudo);
return ret;
pseudo = new PseudoCC();
ret.reset(pseudo);
pseudo->function = function;
pseudo->valueFromUM = valueFromUM;
ColumnCommand::duplicate(pseudo);
return ret;
}
void PseudoCC::createCommand(messageqcpp::ByteStream &bs)
void PseudoCC::createCommand(messageqcpp::ByteStream& bs)
{
bs.advance(1);
bs >> function;
ColumnCommand::createCommand(bs);
bs.advance(1);
bs >> function;
ColumnCommand::createCommand(bs);
}
void PseudoCC::resetCommand(messageqcpp::ByteStream &bs)
void PseudoCC::resetCommand(messageqcpp::ByteStream& bs)
{
if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN || function == PSEUDO_EXTENTID)
bs >> valueFromUM;
ColumnCommand::resetCommand(bs);
if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN || function == PSEUDO_EXTENTID)
bs >> valueFromUM;
ColumnCommand::resetCommand(bs);
}
#ifdef __GNUC__
@ -69,147 +71,197 @@ __attribute__((optimize("no-tree-vectorize")))
void PseudoCC::loadData()
{
switch (function) {
case PSEUDO_PM:
switch(colType.colWidth) {
case 1:
loadPMNumber<uint8_t>();
break;
case 2:
loadPMNumber<uint16_t>();
break;
case 4:
loadPMNumber<uint32_t>();
break;
case 8:
loadPMNumber<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_EXTENTRELATIVERID:
switch(colType.colWidth) {
case 1:
loadRIDs<uint8_t>();
break;
case 2:
loadRIDs<uint16_t>();
break;
case 4:
loadRIDs<uint32_t>();
break;
case 8:
loadRIDs<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENT:
switch(colType.colWidth) {
case 1:
loadSegmentNum<uint8_t>();
break;
case 2:
loadSegmentNum<uint16_t>();
break;
case 4:
loadSegmentNum<uint32_t>();
break;
case 8:
loadSegmentNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENTDIR:
switch(colType.colWidth) {
case 1:
loadPartitionNum<uint8_t>();
break;
case 2:
loadPartitionNum<uint16_t>();
break;
case 4:
loadPartitionNum<uint32_t>();
break;
case 8:
loadPartitionNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_BLOCKID:
switch(colType.colWidth) {
case 1:
loadLBID<uint8_t>();
break;
case 2:
loadLBID<uint16_t>();
break;
case 4:
loadLBID<uint32_t>();
break;
case 8:
loadLBID<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_DBROOT:
switch(colType.colWidth) {
case 1:
loadDBRootNum<uint8_t>();
break;
case 2:
loadDBRootNum<uint16_t>();
break;
case 4:
loadDBRootNum<uint32_t>();
break;
case 8:
loadDBRootNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
/* cases where the value to use is sent from the UM */
case PSEUDO_EXTENTMAX:
case PSEUDO_EXTENTMIN:
case PSEUDO_EXTENTID:
switch(colType.colWidth) {
case 1:
loadSingleValue<int8_t>(valueFromUM);
break;
case 2:
loadSingleValue<int16_t>(valueFromUM);
break;
case 4:
loadSingleValue<int32_t>(valueFromUM);
break;
case 8:
loadSingleValue<int64_t>(valueFromUM);
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
default:
cout << "PC::loadData(): bad function" << endl;
break;
}
switch (function)
{
case PSEUDO_PM:
switch (colType.colWidth)
{
case 1:
loadPMNumber<uint8_t>();
break;
case 2:
loadPMNumber<uint16_t>();
break;
case 4:
loadPMNumber<uint32_t>();
break;
case 8:
loadPMNumber<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_EXTENTRELATIVERID:
switch (colType.colWidth)
{
case 1:
loadRIDs<uint8_t>();
break;
case 2:
loadRIDs<uint16_t>();
break;
case 4:
loadRIDs<uint32_t>();
break;
case 8:
loadRIDs<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENT:
switch (colType.colWidth)
{
case 1:
loadSegmentNum<uint8_t>();
break;
case 2:
loadSegmentNum<uint16_t>();
break;
case 4:
loadSegmentNum<uint32_t>();
break;
case 8:
loadSegmentNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_SEGMENTDIR:
switch (colType.colWidth)
{
case 1:
loadPartitionNum<uint8_t>();
break;
case 2:
loadPartitionNum<uint16_t>();
break;
case 4:
loadPartitionNum<uint32_t>();
break;
case 8:
loadPartitionNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_BLOCKID:
switch (colType.colWidth)
{
case 1:
loadLBID<uint8_t>();
break;
case 2:
loadLBID<uint16_t>();
break;
case 4:
loadLBID<uint32_t>();
break;
case 8:
loadLBID<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
case PSEUDO_DBROOT:
switch (colType.colWidth)
{
case 1:
loadDBRootNum<uint8_t>();
break;
case 2:
loadDBRootNum<uint16_t>();
break;
case 4:
loadDBRootNum<uint32_t>();
break;
case 8:
loadDBRootNum<uint64_t>();
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
/* cases where the value to use is sent from the UM */
case PSEUDO_EXTENTMAX:
case PSEUDO_EXTENTMIN:
case PSEUDO_EXTENTID:
switch (colType.colWidth)
{
case 1:
loadSingleValue<int8_t>(valueFromUM);
break;
case 2:
loadSingleValue<int16_t>(valueFromUM);
break;
case 4:
loadSingleValue<int32_t>(valueFromUM);
break;
case 8:
loadSingleValue<int64_t>(valueFromUM);
break;
default:
cout << "PC::loadData(): bad column width" << endl;
break;
}
break;
default:
cout << "PC::loadData(): bad function" << endl;
break;
}
}
}

95
primitives/primproc/pseudocc.h
View File

@ -22,32 +22,33 @@
#include "columncommand.h"
#include "blocksize.h"
namespace primitiveprocessor {
namespace primitiveprocessor
{
class PseudoCC : public ColumnCommand
{
public:
PseudoCC();
virtual ~PseudoCC();
PseudoCC();
virtual ~PseudoCC();
virtual SCommand duplicate();
virtual void createCommand(messageqcpp::ByteStream &);
virtual void resetCommand(messageqcpp::ByteStream &);
virtual SCommand duplicate();
virtual void createCommand(messageqcpp::ByteStream&);
virtual void resetCommand(messageqcpp::ByteStream&);
protected:
virtual void loadData();
virtual void loadData();
private:
template<typename W> void loadSingleValue(W val);
template<typename W> void loadPMNumber();
template<typename W> void loadRIDs();
template<typename W> void loadSegmentNum();
template<typename W> void loadDBRootNum();
template<typename W> void loadPartitionNum();
template<typename W> void loadLBID();
template<typename W> void loadSingleValue(W val);
template<typename W> void loadPMNumber();
template<typename W> void loadRIDs();
template<typename W> void loadSegmentNum();
template<typename W> void loadDBRootNum();
template<typename W> void loadPartitionNum();
template<typename W> void loadLBID();
uint32_t function;
uint64_t valueFromUM;
uint32_t function;
uint64_t valueFromUM;
};
@ -61,65 +62,71 @@ __attribute__((optimize("no-tree-vectorize")))
#endif
void PseudoCC::loadSingleValue(W val)
{
W *bData = (W *) bpp->blockData;
for (uint32_t i = 0; i < BLOCK_SIZE; i++)
bData[i] = val; // this breaks when using tree-vectorization
//memcpy(&bData[i], &val, sizeof(W)); // this always works
W* bData = (W*) bpp->blockData;
for (uint32_t i = 0; i < BLOCK_SIZE; i++)
bData[i] = val; // this breaks when using tree-vectorization
//memcpy(&bData[i], &val, sizeof(W)); // this always works
}
template<typename W>
void PseudoCC::loadPMNumber()
{
uint32_t pmNum = oamCache->getLocalPMId();
loadSingleValue<W>(pmNum);
uint32_t pmNum = oamCache->getLocalPMId();
loadSingleValue<W>(pmNum);
}
/* This returns extent-relative rids */
template<typename W>
void PseudoCC::loadRIDs()
{
uint32_t i;
uint64_t baseRid = rowgroup::getExtentRelativeRid(bpp->baseRid);
W *bData = (W *) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++) {
//W val = baseRid + i;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = baseRid + i; // breaks with tree-vectorization
}
uint32_t i;
uint64_t baseRid = rowgroup::getExtentRelativeRid(bpp->baseRid);
W* bData = (W*) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++)
{
//W val = baseRid + i;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = baseRid + i; // breaks with tree-vectorization
}
}
template<typename W>
void PseudoCC::loadSegmentNum()
{
uint16_t segNum;
rowgroup::getLocationFromRid(bpp->baseRid, NULL, &segNum, NULL, NULL);
loadSingleValue<W>(segNum);
uint16_t segNum;
rowgroup::getLocationFromRid(bpp->baseRid, NULL, &segNum, NULL, NULL);
loadSingleValue<W>(segNum);
}
template<typename W>
void PseudoCC::loadPartitionNum()
{
uint32_t partNum;
rowgroup::getLocationFromRid(bpp->baseRid, &partNum, NULL, NULL, NULL);
loadSingleValue<W>(partNum);
uint32_t partNum;
rowgroup::getLocationFromRid(bpp->baseRid, &partNum, NULL, NULL, NULL);
loadSingleValue<W>(partNum);
}
template<typename W>
void PseudoCC::loadDBRootNum()
{
loadSingleValue<W>(bpp->dbRoot);
loadSingleValue<W>(bpp->dbRoot);
}
template<typename W>
void PseudoCC::loadLBID()
{
uint32_t i;
W *bData = (W *) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++) {
//W val = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; // breaks with tree-vectorization
}
uint32_t i;
W* bData = (W*) bpp->blockData;
for (i = 0; i < BLOCK_SIZE; i++)
{
//W val = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE;
//memcpy(&bData[i], &val, sizeof(W));
bData[i] = ColumnCommand::getLBID() + (i * sizeof(W)) / BLOCK_SIZE; // breaks with tree-vectorization
}
}
}

28
primitives/primproc/resource.h
View File

@ -1,14 +1,14 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by PrimProc.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by PrimProc.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

254
primitives/primproc/rtscommand.cpp
View File

@ -51,174 +51,202 @@ RTSCommand::~RTSCommand()
void RTSCommand::execute()
{
throw logic_error("RTSCommand shouldn't be used for filter steps");
throw logic_error("RTSCommand shouldn't be used for filter steps");
}
void RTSCommand::project()
{
uint32_t i;
uint32_t i;
if (passThru) {
if (passThru)
{
// if (bpp->absRids.get() == NULL)
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
// need something in values
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
dict.project();
}
else {
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
col.execute(tmpValues);
// need something in values
if (old_rc != bpp->ridCount) {
ostringstream os;
dict.project();
}
else
{
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else {
throw PrimitiveColumnProjectResultExcept(os.str());
}
}
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
dict.project(tmpValues);
}
col.execute(tmpValues);
if (old_rc != bpp->ridCount)
{
ostringstream os;
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else
{
throw PrimitiveColumnProjectResultExcept(os.str());
}
}
dict.project(tmpValues);
}
}
void RTSCommand::projectIntoRowGroup(RowGroup &rg, uint32_t colNum)
void RTSCommand::projectIntoRowGroup(RowGroup& rg, uint32_t colNum)
{
uint32_t i;
uint32_t i;
if (passThru) {
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
dict.projectIntoRowGroup(rg, colNum);
}
else {
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
if (passThru)
{
if (absNull) //@bug 1003. Always need to remake absRids
{
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
col.execute(tmpValues);
for (i = 0; i < bpp->ridCount; i++)
bpp->absRids[i] = bpp->relRids[i] + bpp->baseRid;
}
if (old_rc != bpp->ridCount) {
dict.projectIntoRowGroup(rg, colNum);
}
else
{
int64_t tmpValues[LOGICAL_BLOCK_RIDS];
uint32_t old_rc = bpp->ridCount;
ostringstream os;
if (bpp->absRids.get() == NULL)
bpp->absRids.reset(new uint64_t[LOGICAL_BLOCK_RIDS]);
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else
throw PrimitiveColumnProjectResultExcept(os.str());
}
col.execute(tmpValues);
dict.projectIntoRowGroup(rg, tmpValues, colNum);
}
if (old_rc != bpp->ridCount)
{
ostringstream os;
os << __FILE__ << " (token column) error on projection for oid " << col.getOID() << " lbid " << col.getLBID();
os << ": input rids " << old_rc;
os << ", output rids " << bpp->ridCount << endl;
if (bpp->sessionID & 0x80000000)
throw NeedToRestartJob(os.str());
else
throw PrimitiveColumnProjectResultExcept(os.str());
}
dict.projectIntoRowGroup(rg, tmpValues, colNum);
}
}
uint64_t RTSCommand::getLBID()
{
if (!passThru)
return col.getLBID();
else
return 0;
if (!passThru)
return col.getLBID();
else
return 0;
}
void RTSCommand::nextLBID()
{
if (!passThru)
col.nextLBID();
if (!passThru)
col.nextLBID();
}
void RTSCommand::prep(int8_t outputType, bool makeAbsRids)
{
if (!passThru)
col.prep(OT_BOTH, true);
dict.prep(OT_DATAVALUE, true);
if (!passThru)
col.prep(OT_BOTH, true);
dict.prep(OT_DATAVALUE, true);
}
void RTSCommand::createCommand(ByteStream &bs)
void RTSCommand::createCommand(ByteStream& bs)
{
bs.advance(1);
bs >> passThru;
if (!passThru) {
col.createCommand(bs);
}
dict.createCommand(bs);
Command::createCommand(bs);
bs.advance(1);
bs >> passThru;
if (!passThru)
{
col.createCommand(bs);
}
dict.createCommand(bs);
Command::createCommand(bs);
}
void RTSCommand::resetCommand(ByteStream &bs)
void RTSCommand::resetCommand(ByteStream& bs)
{
if (!passThru)
col.resetCommand(bs);
dict.resetCommand(bs);
if (!passThru)
col.resetCommand(bs);
dict.resetCommand(bs);
}
SCommand RTSCommand::duplicate()
{
SCommand ret;
RTSCommand *rts;
SCommand ret;
RTSCommand* rts;
ret.reset(new RTSCommand());
rts = (RTSCommand *) ret.get();
rts->passThru = passThru;
if (!passThru)
rts->col = col;
rts->dict = dict;
rts->Command::duplicate(this);
return ret;
ret.reset(new RTSCommand());
rts = (RTSCommand*) ret.get();
rts->passThru = passThru;
if (!passThru)
rts->col = col;
rts->dict = dict;
rts->Command::duplicate(this);
return ret;
}
bool RTSCommand::operator==(const RTSCommand &r) const
bool RTSCommand::operator==(const RTSCommand& r) const
{
if (passThru != r.passThru)
return false;
if (!passThru)
if (col != r.col)
return false;
if (absNull != r.absNull)
return false;
if (dict != dict)
return false;
return true;
if (passThru != r.passThru)
return false;
if (!passThru)
if (col != r.col)
return false;
if (absNull != r.absNull)
return false;
if (dict != dict)
return false;
return true;
}
bool RTSCommand::operator!=(const RTSCommand &r) const
bool RTSCommand::operator!=(const RTSCommand& r) const
{
return !(*this == r);
return !(*this == r);
}
void RTSCommand::getLBIDList(uint32_t loopCount, vector<int64_t> *lbids)
void RTSCommand::getLBIDList(uint32_t loopCount, vector<int64_t>* lbids)
{
dict.getLBIDList(loopCount, lbids);
if (!passThru)
col.getLBIDList(loopCount, lbids);
dict.getLBIDList(loopCount, lbids);
if (!passThru)
col.getLBIDList(loopCount, lbids);
}
void RTSCommand::setBatchPrimitiveProcessor(BatchPrimitiveProcessor *b)
void RTSCommand::setBatchPrimitiveProcessor(BatchPrimitiveProcessor* b)
{
Command::setBatchPrimitiveProcessor(b);
dict.setBatchPrimitiveProcessor(b);
if (!passThru)
col.setBatchPrimitiveProcessor(b);
Command::setBatchPrimitiveProcessor(b);
dict.setBatchPrimitiveProcessor(b);
if (!passThru)
col.setBatchPrimitiveProcessor(b);
}
};

67
primitives/primproc/rtscommand.h
View File

@ -19,7 +19,7 @@
// $Id: rtscommand.h 2035 2013-01-21 14:12:19Z rdempsey $
// C++ Interface: rtscommand
//
// Description:
// Description:
//
//
// Author: Patrick <pleblanc@localhost.localdomain>, (C) 2008
@ -39,38 +39,47 @@ namespace primitiveprocessor
class RTSCommand : public Command
{
public:
RTSCommand();
virtual ~RTSCommand();
public:
RTSCommand();
virtual ~RTSCommand();
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup &rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream &);
void resetCommand(messageqcpp::ByteStream &);
SCommand duplicate();
bool operator==(const RTSCommand &) const;
bool operator!=(const RTSCommand &) const;
void setBatchPrimitiveProcessor(BatchPrimitiveProcessor *);
void execute();
void project();
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col);
uint64_t getLBID();
void nextLBID();
void createCommand(messageqcpp::ByteStream&);
void resetCommand(messageqcpp::ByteStream&);
SCommand duplicate();
bool operator==(const RTSCommand&) const;
bool operator!=(const RTSCommand&) const;
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
uint8_t isPassThru() { return passThru; }
void setAbsNull(bool a = true) { absNull = a; }
void getLBIDList(uint32_t loopCount, std::vector<int64_t> *lbids);
void setBatchPrimitiveProcessor(BatchPrimitiveProcessor*);
//TODO: do we need to reference either col or dict?
int getCompType() const { return dict.getCompType(); }
private:
RTSCommand(const RTSCommand &);
/* Put bootstrap code here (ie, build the template primitive msg) */
void prep(int8_t outputType, bool makeAbsRids);
uint8_t isPassThru()
{
return passThru;
}
void setAbsNull(bool a = true)
{
absNull = a;
}
void getLBIDList(uint32_t loopCount, std::vector<int64_t>* lbids);
ColumnCommand col;
DictStep dict;
uint8_t passThru;
bool absNull;
//TODO: do we need to reference either col or dict?
int getCompType() const
{
return dict.getCompType();
}
private:
RTSCommand(const RTSCommand&);
ColumnCommand col;
DictStep dict;
uint8_t passThru;
bool absNull;
};
}

287
primitives/primproc/tdriver-umsocksel.cpp
View File

@ -41,158 +41,171 @@ using namespace primitiveprocessor;
int main()
{
// Columnstore.xml file should be configured as follows:
// um1: 10.100.4.85 and 10.100.5.85
// um2: 10.101.4.85 and 10.101.5.85
sockaddr_in sa = { 1, 0, {0} , {' '} };
char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"};
// Columnstore.xml file should be configured as follows:
// um1: 10.100.4.85 and 10.100.5.85
// um2: 10.101.4.85 and 10.101.5.85
sockaddr_in sa = { 1, 0, {0}, {' '} };
char* ips[] = {"10.100.4.85", "10.100.5.85", "10.101.4.85", "10.101.5.85"};
// These are the IP addresses we use to test runtime connections
// "not" in the Columnstore.xml file.
sockaddr_in saUnknown = { 1, 0, {0} , {' '} };
char* ipsUnknown[]={"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"};
// These are the IP addresses we use to test runtime connections
// "not" in the Columnstore.xml file.
sockaddr_in saUnknown = { 1, 0, {0}, {' '} };
char* ipsUnknown[] = {"10.102.1.1", "10.102.2.1", "10.102.3.1", "10.102.4.1"};
//--------------------------------------------------------------------------
// Test initialization
//--------------------------------------------------------------------------
UmSocketSelector* sockSel = UmSocketSelector::instance();
//--------------------------------------------------------------------------
// Test initialization
//--------------------------------------------------------------------------
UmSocketSelector* sockSel = UmSocketSelector::instance();
std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl;
std::cout << std::endl << "----Dump1 after initialization..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump1 End...................." << std::endl << std::endl;
std::cout << "IPAddressCount: " << sockSel->ipAddressCount() << std::endl;
std::cout << std::endl << "----Dump1 after initialization..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump1 End...................." << std::endl << std::endl;
IOSocket sock[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ips[i], &sa.sin_addr);
for (int j=0; j<4; j++)
{
sock[i][j].setSocketImpl(new InetStreamSocket());
sa.sin_port = htons((i*4)+j);
sock[i][j].sa( sa );
sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
IOSocket sock[4][4];
std::cout << std::endl << "----Dump2 after adding 16 connections..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump2 End...................." << std::endl << std::endl;
for (int i = 0; i < 4; i++)
{
inet_aton(ips[i], &sa.sin_addr);
//--------------------------------------------------------------------------
// Test socket/port selection
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<17; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
for (int j = 0; j < 4; j++)
{
sock[i][j].setSocketImpl(new InetStreamSocket());
sa.sin_port = htons((i * 4) + j);
sock[i][j].sa( sa );
sockSel->addConnection( SP_UM_IOSOCK(new IOSocket(sock[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << std::endl << "----Dump2 after adding 16 connections..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump2 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k = 0; k < 17; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
#if 1
if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
if (sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
#else
if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
if (!sockSel->nextIOSocket( sock[0][0], outIos, writeLock ))
#endif
std::cout << "no nextIP found for " << sock[0][0] << std::endl;
}
for (unsigned k=0; k<7; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sock[2][0] << std::endl;
}
std::cout << std::endl;
std::cout << "----Dump3 after selecting 17 connections from IP " <<
ips[0] << "; and 7 connections from IP " << ips[2] << " ..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump3 End...................." << std::endl << std::endl;
std::cout << "no nextIP found for " << sock[0][0] << std::endl;
}
//--------------------------------------------------------------------------
// Test connection deletions
//--------------------------------------------------------------------------
for (unsigned k=0; k<4; k++)
{
sockSel->delConnection( sock[k][0] );
}
std::cout << "----Dump4 after deleting first connection for each IP..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump4 End...................." << std::endl << std::endl;
for (unsigned k = 0; k < 7; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
//--------------------------------------------------------------------------
// Test addition of unknown connections
//--------------------------------------------------------------------------
IOSocket sockUnknown[4][4];
for (int i=0; i<4; i++)
{
inet_aton(ipsUnknown[i], &saUnknown.sin_addr);
for (int j=0; j<4; j++)
{
sockUnknown[i][j].setSocketImpl(new InetStreamSocket());
saUnknown.sin_port = htons((i*4)+j);
sockUnknown[i][j].sa( saUnknown );
sockSel->addConnection(
SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
if (sockSel->nextIOSocket( sock[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sock[2][0] << std::endl;
}
std::cout << "----Dump5 after adding connections for unknown IP's..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump5 End...................." << std::endl << std::endl;
std::cout << std::endl;
std::cout << "----Dump3 after selecting 17 connections from IP " <<
ips[0] << "; and 7 connections from IP " << ips[2] << " ..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump3 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test resetting of "next" indexes after deleting all sockets from a
// specific IP address for which the "next" index is pointing.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[1][1] );
sockSel->delConnection( sock[1][2] );
sockSel->delConnection( sock[1][3] );
//--------------------------------------------------------------------------
// Test connection deletions
//--------------------------------------------------------------------------
for (unsigned k = 0; k < 4; k++)
{
sockSel->delConnection( sock[k][0] );
}
std::cout << "----Dump6 after deleting all connections for IP " << ips[1] <<
" ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump6 End...................." << std::endl << std::endl;
std::cout << "----Dump4 after deleting first connection for each IP..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump4 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection for an unknown module
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k=0; k<11; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sockUnknown[2][0]<<std::endl;
}
std::cout << std::endl;
std::cout << "----Dump7 after selecting 11 connections for IP " <<
ipsUnknown[2] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump7 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test addition of unknown connections
//--------------------------------------------------------------------------
IOSocket sockUnknown[4][4];
//--------------------------------------------------------------------------
// Test deletion of last socket/port connection and resetting if the
// "next" index that is pointing to it.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[3][3] );
std::cout << "----Dump8 after deleting last connection for IP " <<
ips[3] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump8 End...................." << std::endl << std::endl;
for (int i = 0; i < 4; i++)
{
inet_aton(ipsUnknown[i], &saUnknown.sin_addr);
return 0;
for (int j = 0; j < 4; j++)
{
sockUnknown[i][j].setSocketImpl(new InetStreamSocket());
saUnknown.sin_port = htons((i * 4) + j);
sockUnknown[i][j].sa( saUnknown );
sockSel->addConnection(
SP_UM_IOSOCK(new IOSocket(sockUnknown[i][j])),
SP_UM_MUTEX( new boost::mutex()) );
}
}
std::cout << "----Dump5 after adding connections for unknown IP's..." <<
std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump5 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test resetting of "next" indexes after deleting all sockets from a
// specific IP address for which the "next" index is pointing.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[1][1] );
sockSel->delConnection( sock[1][2] );
sockSel->delConnection( sock[1][3] );
std::cout << "----Dump6 after deleting all connections for IP " << ips[1] <<
" ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump6 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test socket/port selection for an unknown module
//--------------------------------------------------------------------------
std::cout << "Test socket/port selection logic..." << std::endl;
for (unsigned k = 0; k < 11; k++)
{
SP_UM_IOSOCK outIos;
SP_UM_MUTEX writeLock;
if (sockSel->nextIOSocket( sockUnknown[2][0], outIos, writeLock ))
std::cout << "next IP: " << inet_ntoa(outIos->sa().sin_addr) <<
"; port: " << ntohs(outIos->sa().sin_port) << std::endl;
else
std::cout << "no nextIP found for " << sockUnknown[2][0] << std::endl;
}
std::cout << std::endl;
std::cout << "----Dump7 after selecting 11 connections for IP " <<
ipsUnknown[2] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump7 End...................." << std::endl << std::endl;
//--------------------------------------------------------------------------
// Test deletion of last socket/port connection and resetting if the
// "next" index that is pointing to it.
//--------------------------------------------------------------------------
sockSel->delConnection( sock[3][3] );
std::cout << "----Dump8 after deleting last connection for IP " <<
ips[3] << " ..." << std::endl;
std::cout << sockSel->toString();
std::cout << "----Dump8 End...................." << std::endl << std::endl;
return 0;
}

48
primitives/primproc/tdriver.cpp
View File

@ -47,7 +47,7 @@ void testColByScan()
cout << "Sending COL_BY_SCAN primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ISMPacketHeader packetHeader;
packetHeader.Reserve = 0;
@ -91,6 +91,7 @@ void testColByScan()
ibs = proc.read();
int messageLen = ibs.length();
if (messageLen)
{
cout << "Received results" << endl << endl;
@ -131,12 +132,13 @@ void testColByScan()
char* data = new char[remaining];
memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining);
char *ptr = data;
char* ptr = data;
for(int i = 0; i < remaining; i++)
for (int i = 0; i < remaining; i++)
{
for (int j = 0; j < 10 && j < remaining; j++)
printf("%02x ", *ptr++);
printf("\n");
}
@ -154,7 +156,7 @@ void testColByRid()
cout << "Sending COL_BY_RID primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ISMPacketHeader packetHeader;
packetHeader.Reserve = 0;
@ -198,6 +200,7 @@ void testColByRid()
ibs = proc.read();
int messageLen = ibs.length();
if (messageLen)
{
@ -239,12 +242,13 @@ void testColByRid()
char* data = new char[remaining];
memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining);
char *ptr = data;
char* ptr = data;
for(int i = 0; i < remaining; i++)
for (int i = 0; i < remaining; i++)
{
for (int j = 0; j < 10 && j < remaining; j++)
printf("%02x ", *ptr++);
printf("\n");
}
@ -263,7 +267,7 @@ void testColAggByScan()
cout << "Sending COL_AGG_BY_SCAN primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ISMPacketHeader packetHeader;
packetHeader.Reserve = 0;
@ -298,7 +302,7 @@ void testColAggByScan()
memmove(message + sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader), &TempData, sizeof(Int64));
obs.append((messageqcpp::ByteStream::byte*)message, sizeof(ISMPacketHeader) + sizeof(ColAggByScanRequestHeader)
+ sizeof(Int64));
+ sizeof(Int64));
cout << "Sending to Primitive Server" << endl;
@ -308,6 +312,7 @@ void testColAggByScan()
ibs = proc.read();
int messageLen = ibs.length();
if (messageLen)
{
ISMPacketHeader pktHeader;
@ -351,12 +356,13 @@ void testColAggByScan()
char* data = new char[remaining];
memmove(data, bytePtr + sizeof(ISMPacketHeader) + sizeof(ColResultHeader), remaining);
char *ptr = data;
char* ptr = data;
for(int i = 0; i < remaining; i++)
for (int i = 0; i < remaining; i++)
{
for (int j = 0; j < 10 && j < remaining; j++)
printf("%02x ", *ptr++);
printf("\n");
}
@ -374,7 +380,7 @@ void testColAggByRid()
cout << "Sending COL_AGG_BY_RID primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ByteStream::octbyte value64;
ByteStream::quadbyte value32;
@ -496,6 +502,7 @@ void testColAggByRid()
cout << "Sent... awaiting results" << endl;
ibs = proc.read();
if (ibs.length() > 0)
{
ISMPacketHeader pktHeader;
@ -583,6 +590,7 @@ void testColAggByRid()
cout << "Data" << endl;
cout << "----" << endl << endl;
cout << "Data Size: " << ibs.length() << endl;
for (int i = 0; i < colAggResult.NVALS && ibs.length(); i++)
{
if (colAggByRID.OutputType == 1 || colAggByRID.OutputType == 3 )
@ -590,9 +598,11 @@ void testColAggByRid()
ibs >> value16;
cout << "RID: " << value16 << endl;
}
if (colAggByRID.OutputType == 2 || colAggByRID.OutputType == 3)
{
cout << "Token: ";
switch (colAggByRID.DataSize)
{
case 1:
@ -629,7 +639,7 @@ void testDictTokenByIndexCompare()
cout << "Sending DICT_TOKEN_BY_INDEX_COMPARE primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ByteStream::octbyte value64;
ByteStream::quadbyte value32;
@ -741,9 +751,11 @@ void testDictTokenByIndexCompare()
ibs = proc.read();
cout << "Data" << endl;
cout << "----" << endl << endl;
if (ibs.length() > 0)
{
cout << "Data Size: " << ibs.length() << endl;
while (ibs.length())
{
ibs >> value16;
@ -764,7 +776,7 @@ void testDictSignature()
cout << "DICT_SIGNATURE primitive" << endl;
MessageQueueClient proc("PMS1");
ByteStream obs,ibs;
ByteStream obs, ibs;
ByteStream::octbyte value64;
ByteStream::quadbyte value32;
@ -877,9 +889,11 @@ void testDictSignature()
cout << "Data" << endl;
cout << "----" << endl << endl;
if (ibs.length() > 0)
{
cout << "Data Size: " << ibs.length() << endl;
while (ibs.length())
{
ibs >> value16;
@ -914,9 +928,9 @@ int main(int argc, char* argv[])
po::options_description desc ("Allowed options");
desc.add_options ()
("help", "produce help message")
("all", "process all tests" )
("loop", "loop processing all tests");
("help", "produce help message")
("all", "process all tests" )
("loop", "loop processing all tests");
po::variables_map vm;
po::store (po::parse_command_line (argc, argv, desc), vm);
po::notify (vm);
@ -932,7 +946,7 @@ int main(int argc, char* argv[])
}
else if (vm.count ("loop"))
{
while(1)
while (1)
{
testColByScan();
testColByRid();

40
primitives/primproc/udf.cpp
View File

@ -39,29 +39,33 @@ namespace primitiveprocessor
void loadUDFs()
{
#ifndef _MSC_VER
int flags = RTLD_NOW;
int flags = RTLD_NOW;
void* libPtr = 0;
void* libPtr = 0;
UDFFcnPtr_t fcnPtr;
UDFFcnPtr_t fcnPtr;
libPtr = dlopen(UdfLibName.c_str(), flags);
libPtr = dlopen(UdfLibName.c_str(), flags);
if (libPtr == 0)
return;
if (libPtr == 0)
return;
unsigned i = 1;
for (;;)
{
ostringstream oss;
oss << "cpfunc" << i;
fcnPtr = (UDFFcnPtr_t)dlsym(libPtr, oss.str().c_str());
if (fcnPtr == 0)
break;
UDFFcnMap[i] = fcnPtr;
i++;
}
cout << "loaded " << UDFFcnMap.size() << " UDF's" << endl;
unsigned i = 1;
for (;;)
{
ostringstream oss;
oss << "cpfunc" << i;
fcnPtr = (UDFFcnPtr_t)dlsym(libPtr, oss.str().c_str());
if (fcnPtr == 0)
break;
UDFFcnMap[i] = fcnPtr;
i++;
}
cout << "loaded " << UDFFcnMap.size() << " UDF's" << endl;
#endif
}

637
primitives/primproc/umsocketselector.cpp
View File

@ -55,9 +55,9 @@ using namespace oam;
namespace primitiveprocessor
{
/*static*/ const int32_t UmIPSocketConns::NEXT_IOSOCKET_UNASSIGNED = -1;
/*static*/ const int32_t UmModuleIPs::NEXT_IP_SOCKET_UNASSIGNED = -1;
/*static*/ UmSocketSelector* UmSocketSelector::fpUmSocketSelector = 0;
/*static*/ const int32_t UmIPSocketConns::NEXT_IOSOCKET_UNASSIGNED = -1;
/*static*/ const int32_t UmModuleIPs::NEXT_IP_SOCKET_UNASSIGNED = -1;
/*static*/ UmSocketSelector* UmSocketSelector::fpUmSocketSelector = 0;
//------------------------------------------------------------------------------
// UmSocketSelector methods
@ -67,12 +67,12 @@ namespace primitiveprocessor
/* static */ UmSocketSelector*
UmSocketSelector::instance()
{
if (fpUmSocketSelector == 0)
{
fpUmSocketSelector = new UmSocketSelector();
}
if (fpUmSocketSelector == 0)
{
fpUmSocketSelector = new UmSocketSelector();
}
return fpUmSocketSelector;
return fpUmSocketSelector;
}
//------------------------------------------------------------------------------
@ -80,7 +80,7 @@ UmSocketSelector::instance()
//------------------------------------------------------------------------------
UmSocketSelector::UmSocketSelector()
{
loadUMModuleInfo();
loadUMModuleInfo();
}
//------------------------------------------------------------------------------
@ -90,14 +90,14 @@ UmSocketSelector::UmSocketSelector()
uint32_t
UmSocketSelector::ipAddressCount() const
{
uint32_t ipCount = 0;
uint32_t ipCount = 0;
for (unsigned int i=0; i<fUmModuleIPs.size(); ++i)
{
ipCount += fUmModuleIPs[i]->ipAddressCount();
}
for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i)
{
ipCount += fUmModuleIPs[i]->ipAddressCount();
}
return ipCount;
return ipCount;
}
//------------------------------------------------------------------------------
@ -109,67 +109,68 @@ UmSocketSelector::ipAddressCount() const
void
UmSocketSelector::loadUMModuleInfo()
{
Oam oam;
ModuleTypeConfig moduleTypeConfig;
const std::string UM_MODTYPE("um");
Oam oam;
ModuleTypeConfig moduleTypeConfig;
const std::string UM_MODTYPE("um");
oam.getSystemConfig(UM_MODTYPE, moduleTypeConfig);
oam.getSystemConfig(UM_MODTYPE, moduleTypeConfig);
int moduleCount= moduleTypeConfig.ModuleCount;
std::string moduleType = moduleTypeConfig.ModuleType;
int moduleCount = moduleTypeConfig.ModuleCount;
std::string moduleType = moduleTypeConfig.ModuleType;
#ifdef LOAD_MODULE_DEBUG
std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl;
std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl;
std::cout << "ModuleCount = " << moduleCount << std::endl;
std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl;
std::cout << "ModuleConfig for type: " << UM_MODTYPE << std::endl;
std::cout << "ModuleDesc = " << moduleTypeConfig.ModuleDesc << std::endl;
std::cout << "ModuleCount = " << moduleCount << std::endl;
std::cout << "RunType = " << moduleTypeConfig.RunType << std::endl;
#endif
if ( moduleCount > 0 )
{
//..Loop through the list of UM modules
for (DeviceNetworkList::iterator iter1 =
moduleTypeConfig.ModuleNetworkList.begin();
(iter1 != moduleTypeConfig.ModuleNetworkList.end());
++iter1)
{
std::string moduleName = iter1->DeviceName;
if ( moduleCount > 0 )
{
//..Loop through the list of UM modules
for (DeviceNetworkList::iterator iter1 =
moduleTypeConfig.ModuleNetworkList.begin();
(iter1 != moduleTypeConfig.ModuleNetworkList.end());
++iter1)
{
std::string moduleName = iter1->DeviceName;
#ifdef LOAD_MODULE_DEBUG
std::cout << "ModuleName-" << moduleName << std::endl;
std::cout << "ModuleName-" << moduleName << std::endl;
#endif
//..Assign the UM index based on whether it is a new UM or one
// we have seen before
unsigned int umIdx = findOrAddUm( moduleName );
//..Assign the UM index based on whether it is a new UM or one
// we have seen before
unsigned int umIdx = findOrAddUm( moduleName );
//..Get the list of IP addresses (NIC's) for this UM module
for (HostConfigList::iterator iter2 = iter1->hostConfigList.begin();
(iter2 != iter1->hostConfigList.end());
++iter2)
{
std::string ipAddr = iter2->IPAddr;
//..Get the list of IP addresses (NIC's) for this UM module
for (HostConfigList::iterator iter2 = iter1->hostConfigList.begin();
(iter2 != iter1->hostConfigList.end());
++iter2)
{
std::string ipAddr = iter2->IPAddr;
#ifdef LOAD_MODULE_DEBUG
std::cout << " NIC-" << iter2->NicID <<
"; host-" << iter2->HostName <<
"; IP-" << ipAddr << std::endl;
std::cout << " NIC-" << iter2->NicID <<
"; host-" << iter2->HostName <<
"; IP-" << ipAddr << std::endl;
#endif
struct in_addr ip;
if ( inet_aton(ipAddr.c_str(), &ip ) )
{
fIpAddressUmMap[ ip.s_addr ] = umIdx;
fUmModuleIPs[umIdx]->addIP( ip.s_addr );
}
else
{
std::cerr << "Invalid IP address in SystemModuleConfig "
"section: " << ipAddr << std::endl;
}
} // loop through the IP addresses for a UM module
} // loop through the list of UM modules
} // moduleCount > 0
struct in_addr ip;
if ( inet_aton(ipAddr.c_str(), &ip ) )
{
fIpAddressUmMap[ ip.s_addr ] = umIdx;
fUmModuleIPs[umIdx]->addIP( ip.s_addr );
}
else
{
std::cerr << "Invalid IP address in SystemModuleConfig "
"section: " << ipAddr << std::endl;
}
} // loop through the IP addresses for a UM module
} // loop through the list of UM modules
} // moduleCount > 0
}
//------------------------------------------------------------------------------
@ -180,21 +181,22 @@ UmSocketSelector::loadUMModuleInfo()
unsigned int
UmSocketSelector::findOrAddUm( const std::string& moduleName )
{
unsigned int umIdx = std::numeric_limits<unsigned int>::max();
for (unsigned int i=0; i<fUmModuleIPs.size(); ++i)
{
if (fUmModuleIPs[i]->moduleName() == moduleName)
{
umIdx = i;
return umIdx;
}
}
unsigned int umIdx = std::numeric_limits<unsigned int>::max();
//..We have encountered a new UM module we should add to the list
fUmModuleIPs.push_back( SP_UM_MODIPS(new UmModuleIPs(moduleName)) );
umIdx = fUmModuleIPs.size() - 1;
for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i)
{
if (fUmModuleIPs[i]->moduleName() == moduleName)
{
umIdx = i;
return umIdx;
}
}
return umIdx;
//..We have encountered a new UM module we should add to the list
fUmModuleIPs.push_back( SP_UM_MODIPS(new UmModuleIPs(moduleName)) );
umIdx = fUmModuleIPs.size() - 1;
return umIdx;
}
//------------------------------------------------------------------------------
@ -206,34 +208,34 @@ UmSocketSelector::findOrAddUm( const std::string& moduleName )
//------------------------------------------------------------------------------
bool
UmSocketSelector::addConnection(
const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock )
const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock )
{
bool bConnAdded = false;
bool bConnAdded = false;
sockaddr sa = ios->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
sockaddr sa = ios->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
// Add this socket/port connection to the UM connection list it belongs to.
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
bConnAdded = fUmModuleIPs[umIdx]->addSocketConn( ios, writeLock );
}
// Add this socket/port connection to the UM connection list it belongs to.
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
bConnAdded = fUmModuleIPs[umIdx]->addSocketConn( ios, writeLock );
}
if (!bConnAdded)
{
if (!bConnAdded)
{
#ifdef SEL_CONN_DEBUG
std::ostringstream oss;
oss << "No UM/IP match found to add connection " << ios->toString() <<
std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "No UM/IP match found to add connection " << ios->toString() <<
std::endl;
std::cout << oss.str();
#endif
}
}
return bConnAdded;
return bConnAdded;
}
//------------------------------------------------------------------------------
@ -243,16 +245,16 @@ UmSocketSelector::addConnection(
void
UmSocketSelector::delConnection( const IOSocket& ios )
{
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
fUmModuleIPs[umIdx]->delSocketConn( ios );
}
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
fUmModuleIPs[umIdx]->delSocketConn( ios );
}
}
//------------------------------------------------------------------------------
@ -269,36 +271,37 @@ UmSocketSelector::delConnection( const IOSocket& ios )
//------------------------------------------------------------------------------
bool
UmSocketSelector::nextIOSocket(
const IOSocket& ios,
SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock )
const IOSocket& ios,
SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock )
{
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
sockaddr sa = ios.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
IpAddressUmMap_t::iterator mapIter =
fIpAddressUmMap.find ( sinp->sin_addr.s_addr );
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
if (fUmModuleIPs[umIdx]->nextIOSocket( outIos, writeLock ))
{
if ( mapIter != fIpAddressUmMap.end() )
{
unsigned int umIdx = mapIter->second;
if (fUmModuleIPs[umIdx]->nextIOSocket( outIos, writeLock ))
{
#ifdef SEL_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleIPs[umIdx]->moduleName() <<
"; in: " << ios.toString() <<
"; selected out: " << outIos->toString() << std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "UM " << fUmModuleIPs[umIdx]->moduleName() <<
"; in: " << ios.toString() <<
"; selected out: " << outIos->toString() << std::endl;
std::cout << oss.str();
#endif
return true;
}
}
return true;
}
}
//..This should not happen. Application is asking for next socket/port for
// a connection not in our UM module list.
return false;
//..This should not happen. Application is asking for next socket/port for
// a connection not in our UM module list.
return false;
}
//------------------------------------------------------------------------------
@ -307,25 +310,26 @@ UmSocketSelector::nextIOSocket(
const std::string
UmSocketSelector::toString() const
{
std::ostringstream oss;
std::ostringstream oss;
oss << "IP Address to UM index map:" << std::endl;
for (IpAddressUmMap_t::const_iterator mapIter = fIpAddressUmMap.begin();
(mapIter != fIpAddressUmMap.end());
++mapIter)
{
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " <<
UmIPSocketConns::nwToString(mapIter->first, ipString) <<
" maps to UM: " << mapIter->second << std::endl;
}
oss << "IP Address to UM index map:" << std::endl;
for (unsigned int i=0; i<fUmModuleIPs.size(); ++i)
{
oss << std::endl << fUmModuleIPs[i]->toString();
}
for (IpAddressUmMap_t::const_iterator mapIter = fIpAddressUmMap.begin();
(mapIter != fIpAddressUmMap.end());
++mapIter)
{
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " <<
UmIPSocketConns::nwToString(mapIter->first, ipString) <<
" maps to UM: " << mapIter->second << std::endl;
}
return oss.str();
for (unsigned int i = 0; i < fUmModuleIPs.size(); ++i)
{
oss << std::endl << fUmModuleIPs[i]->toString();
}
return oss.str();
}
//------------------------------------------------------------------------------
@ -337,17 +341,17 @@ UmSocketSelector::toString() const
void
UmModuleIPs::addIP( in_addr_t ip )
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
boost::mutex::scoped_lock lock( fUmModuleMutex );
#ifdef MOD_CONN_DEBUG
std::ostringstream oss;
char ipString[INET_ADDRSTRLEN];
oss << " UM " << fUmModuleName << "; adding IP: " <<
UmIPSocketConns::nwToString(ip,ipString) << std::endl;
std::cout << oss.str();
std::ostringstream oss;
char ipString[INET_ADDRSTRLEN];
oss << " UM " << fUmModuleName << "; adding IP: " <<
UmIPSocketConns::nwToString(ip, ipString) << std::endl;
std::cout << oss.str();
#endif
fUmIPSocketConns.push_back( SP_UM_IPCONNS(new UmIPSocketConns(ip)) );
fUmIPSocketConns.push_back( SP_UM_IPCONNS(new UmIPSocketConns(ip)) );
}
//------------------------------------------------------------------------------
@ -359,38 +363,41 @@ UmModuleIPs::addIP( in_addr_t ip )
//------------------------------------------------------------------------------
bool
UmModuleIPs::addSocketConn(
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
{
bool bConnAdded = false;
bool bConnAdded = false;
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i=0; i<fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
#ifdef MOD_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; adding connection " <<
ioSock->toString() << std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; adding connection " <<
ioSock->toString() << std::endl;
std::cout << oss.str();
#endif
fUmIPSocketConns[i]->addSocketConn ( ioSock, writeLock );
bConnAdded = true;
fUmIPSocketConns[i]->addSocketConn ( ioSock, writeLock );
bConnAdded = true;
//..Initialize fNextUmIPSocketIdx if this is the first socket/port
// connection for this UM.
if ( fNextUmIPSocketIdx == NEXT_IP_SOCKET_UNASSIGNED)
fNextUmIPSocketIdx = i;
break;
}
}
//..Initialize fNextUmIPSocketIdx if this is the first socket/port
// connection for this UM.
if ( fNextUmIPSocketIdx == NEXT_IP_SOCKET_UNASSIGNED)
fNextUmIPSocketIdx = i;
return bConnAdded;
break;
}
}
return bConnAdded;
}
//------------------------------------------------------------------------------
@ -400,32 +407,35 @@ UmModuleIPs::addSocketConn(
void
UmModuleIPs::delSocketConn( const IOSocket& ioSock )
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i=0; i<fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
boost::mutex::scoped_lock lock( fUmModuleMutex );
for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i)
{
sockaddr sa = ioSock.sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
if (fUmIPSocketConns[i]->ipAddress() == sinp->sin_addr.s_addr)
{
#ifdef MOD_CONN_DEBUG
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; deleting connection "<<
ioSock.toString() << std::endl;
std::cout << oss.str();
std::ostringstream oss;
oss << "UM " << fUmModuleName << "; deleting connection " <<
ioSock.toString() << std::endl;
std::cout << oss.str();
#endif
fUmIPSocketConns[i]->delSocketConn ( ioSock );
fUmIPSocketConns[i]->delSocketConn ( ioSock );
//..If we just deleted the last connection for this IP, then ad-
// vance fNextUmIPSocketIdx to an IP address having connections.
if (fUmIPSocketConns[i]->count() == 0)
{
advanceToNextIP();
}
break;
}
}
//..If we just deleted the last connection for this IP, then ad-
// vance fNextUmIPSocketIdx to an IP address having connections.
if (fUmIPSocketConns[i]->count() == 0)
{
advanceToNextIP();
}
break;
}
}
}
//------------------------------------------------------------------------------
@ -439,19 +449,20 @@ UmModuleIPs::delSocketConn( const IOSocket& ioSock )
bool
UmModuleIPs::nextIOSocket( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
{
bool found = false;
bool found = false;
boost::mutex::scoped_lock lock( fUmModuleMutex );
if ((fUmIPSocketConns.size() > 0) &&
(fNextUmIPSocketIdx != NEXT_IP_SOCKET_UNASSIGNED))
{
assert (fNextUmIPSocketIdx < static_cast<int>(fUmIPSocketConns.size()));
fUmIPSocketConns[fNextUmIPSocketIdx]->nextIOSocket( outIos, writeLock );
advanceToNextIP();
found = true;
}
boost::mutex::scoped_lock lock( fUmModuleMutex );
return found;
if ((fUmIPSocketConns.size() > 0) &&
(fNextUmIPSocketIdx != NEXT_IP_SOCKET_UNASSIGNED))
{
assert (fNextUmIPSocketIdx < static_cast<int>(fUmIPSocketConns.size()));
fUmIPSocketConns[fNextUmIPSocketIdx]->nextIOSocket( outIos, writeLock );
advanceToNextIP();
found = true;
}
return found;
}
//------------------------------------------------------------------------------
@ -462,40 +473,40 @@ UmModuleIPs::nextIOSocket( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
void
UmModuleIPs::advanceToNextIP()
{
if ( fUmIPSocketConns.size() > 1 )
{
//..Search to end of IP list for an IP having 1 or more connections
for (int i=fNextUmIPSocketIdx+1;
i<static_cast<int>(fUmIPSocketConns.size());
++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
if ( fUmIPSocketConns.size() > 1 )
{
//..Search to end of IP list for an IP having 1 or more connections
for (int i = fNextUmIPSocketIdx + 1;
i < static_cast<int>(fUmIPSocketConns.size());
++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
//..Wrap back around to the start of the list, to continue the search
for (int i=0; i<fNextUmIPSocketIdx; ++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
//..Wrap back around to the start of the list, to continue the search
for (int i = 0; i < fNextUmIPSocketIdx; ++i)
{
if (fUmIPSocketConns[i]->count() > 0)
{
fNextUmIPSocketIdx = i;
return;
}
}
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
else // special logic to handle 0 fUmIPSocketConns, or a single empty one
{
if ((fUmIPSocketConns.size() == 0) ||
(fUmIPSocketConns[0]->count() == 0))
{
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
}
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
else // special logic to handle 0 fUmIPSocketConns, or a single empty one
{
if ((fUmIPSocketConns.size() == 0) ||
(fUmIPSocketConns[0]->count() == 0))
{
fNextUmIPSocketIdx = NEXT_IP_SOCKET_UNASSIGNED;
}
}
}
//------------------------------------------------------------------------------
@ -504,17 +515,18 @@ UmModuleIPs::advanceToNextIP()
const std::string
UmModuleIPs::toString()
{
std::ostringstream oss;
std::ostringstream oss;
boost::mutex::scoped_lock lock( fUmModuleMutex );
oss << "UM module name: " << fUmModuleName <<
"; nextUmIPIdx: " << fNextUmIPSocketIdx << std::endl;
for (unsigned int i=0; i<fUmIPSocketConns.size(); ++i)
{
oss << fUmIPSocketConns[i]->toString();
}
boost::mutex::scoped_lock lock( fUmModuleMutex );
oss << "UM module name: " << fUmModuleName <<
"; nextUmIPIdx: " << fNextUmIPSocketIdx << std::endl;
return oss.str();
for (unsigned int i = 0; i < fUmIPSocketConns.size(); ++i)
{
oss << fUmIPSocketConns[i]->toString();
}
return oss.str();
}
//------------------------------------------------------------------------------
@ -527,16 +539,18 @@ UmModuleIPs::toString()
//------------------------------------------------------------------------------
void
UmIPSocketConns::addSocketConn(
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock )
{
UmIOSocketData sockData = {
SP_UM_IOSOCK(ioSock), SP_UM_MUTEX(writeLock) };
fIOSockets.push_back( sockData );
UmIOSocketData sockData =
{
SP_UM_IOSOCK(ioSock), SP_UM_MUTEX(writeLock)
};
fIOSockets.push_back( sockData );
//..Initialize fNextIOSocketIdx when we add first connection
if (fIOSockets.size() == 1)
fNextIOSocketIdx = 0;
//..Initialize fNextIOSocketIdx when we add first connection
if (fIOSockets.size() == 1)
fNextIOSocketIdx = 0;
}
//------------------------------------------------------------------------------
@ -553,36 +567,37 @@ UmIPSocketConns::addSocketConn(
void
UmIPSocketConns::delSocketConn( const IOSocket& ioSock )
{
for (unsigned int i=0; i<fIOSockets.size(); ++i)
{
sockaddr sa1 = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp1 = reinterpret_cast<const sockaddr_in*>(&sa1);
sockaddr sa2 = ioSock.sa();
const sockaddr_in* sinp2 = reinterpret_cast<const sockaddr_in*>(&sa2);
if (sinp1->sin_port == sinp2->sin_port)
{
fIOSockets.erase ( fIOSockets.begin() + i );
for (unsigned int i = 0; i < fIOSockets.size(); ++i)
{
sockaddr sa1 = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp1 = reinterpret_cast<const sockaddr_in*>(&sa1);
sockaddr sa2 = ioSock.sa();
const sockaddr_in* sinp2 = reinterpret_cast<const sockaddr_in*>(&sa2);
//..Adjust fNextIOSocketIdx
// 1a. decrement if fNextIOSocketIdx is after deleted connection
// 1b. reset to start of vector if we deleted the last connection
// 2. reset fNextIOSocketIdx to -1 if we have no more connections
if (fIOSockets.size() > 0)
{
if (fNextIOSocketIdx > static_cast<int>(i))
fNextIOSocketIdx--;
if ( fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()) )
fNextIOSocketIdx = 0;
}
else
{
fNextIOSocketIdx = NEXT_IOSOCKET_UNASSIGNED;
}
if (sinp1->sin_port == sinp2->sin_port)
{
fIOSockets.erase ( fIOSockets.begin() + i );
break;
}
}
//..Adjust fNextIOSocketIdx
// 1a. decrement if fNextIOSocketIdx is after deleted connection
// 1b. reset to start of vector if we deleted the last connection
// 2. reset fNextIOSocketIdx to -1 if we have no more connections
if (fIOSockets.size() > 0)
{
if (fNextIOSocketIdx > static_cast<int>(i))
fNextIOSocketIdx--;
if ( fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()) )
fNextIOSocketIdx = 0;
}
else
{
fNextIOSocketIdx = NEXT_IOSOCKET_UNASSIGNED;
}
break;
}
}
}
//------------------------------------------------------------------------------
@ -594,18 +609,19 @@ UmIPSocketConns::delSocketConn( const IOSocket& ioSock )
void
UmIPSocketConns::nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
{
assert (fIOSockets.size() > 0);
assert (fNextIOSocketIdx != NEXT_IOSOCKET_UNASSIGNED);
assert (fNextIOSocketIdx < static_cast<int>(fIOSockets.size()));
assert (fIOSockets.size() > 0);
assert (fNextIOSocketIdx != NEXT_IOSOCKET_UNASSIGNED);
assert (fNextIOSocketIdx < static_cast<int>(fIOSockets.size()));
outIos = fIOSockets[fNextIOSocketIdx].fSock;
writeLock = fIOSockets[fNextIOSocketIdx].fMutex;
outIos = fIOSockets[fNextIOSocketIdx].fSock;
writeLock = fIOSockets[fNextIOSocketIdx].fMutex;
//..Update "next" index, being sure to wrap around to the start
// whenever we reach the end of the vector.
fNextIOSocketIdx++;
if (fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()))
fNextIOSocketIdx = 0;
//..Update "next" index, being sure to wrap around to the start
// whenever we reach the end of the vector.
fNextIOSocketIdx++;
if (fNextIOSocketIdx >= static_cast<int>(fIOSockets.size()))
fNextIOSocketIdx = 0;
}
//------------------------------------------------------------------------------
@ -615,13 +631,14 @@ UmIPSocketConns::nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock )
/* static */ char*
UmIPSocketConns::nwToString( in_addr_t addr, char* ipString )
{
in_addr addrStruct = { addr };
in_addr addrStruct = { addr };
#ifndef _MSC_VER
if (!inet_ntop(AF_INET, &addrStruct, ipString, INET_ADDRSTRLEN))
#endif
strcpy(ipString,"unknown");
return ipString;
if (!inet_ntop(AF_INET, &addrStruct, ipString, INET_ADDRSTRLEN))
#endif
strcpy(ipString, "unknown");
return ipString;
}
//------------------------------------------------------------------------------
@ -630,21 +647,21 @@ UmIPSocketConns::nwToString( in_addr_t addr, char* ipString )
const std::string
UmIPSocketConns::toString() const
{
std::ostringstream oss;
std::ostringstream oss;
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " << UmIPSocketConns::nwToString(fIpAddress,ipString)<<
"; nextIOSocketIdx: " << fNextIOSocketIdx << std::endl;
char ipString[INET_ADDRSTRLEN];
oss << " IPAddress: " << UmIPSocketConns::nwToString(fIpAddress, ipString) <<
"; nextIOSocketIdx: " << fNextIOSocketIdx << std::endl;
for (unsigned int i=0; i<fIOSockets.size(); ++i)
{
sockaddr sa = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
oss << " port: " << ntohs(sinp->sin_port) <<
std::endl;
}
for (unsigned int i = 0; i < fIOSockets.size(); ++i)
{
sockaddr sa = fIOSockets[i].fSock->sa();
const sockaddr_in* sinp = reinterpret_cast<const sockaddr_in*>(&sa);
oss << " port: " << ntohs(sinp->sin_port) <<
std::endl;
}
return oss.str();
return oss.str();
}
} // end of primitiveprocessor namespace

376
primitives/primproc/umsocketselector.h
View File

@ -81,73 +81,73 @@ typedef boost::shared_ptr<boost::mutex> SP_UM_MUTEX;
//------------------------------------------------------------------------------
class UmSocketSelector
{
public:
typedef std::map<in_addr_t,unsigned int> IpAddressUmMap_t;
public:
typedef std::map<in_addr_t, unsigned int> IpAddressUmMap_t;
/** @brief Singleton accessor to UmSocketSelector instance.
*
* This method should be called once from the main thread to perform
* initialization from a single threaded environment.
*/
static UmSocketSelector* instance();
/** @brief Singleton accessor to UmSocketSelector instance.
*
* This method should be called once from the main thread to perform
* initialization from a single threaded environment.
*/
static UmSocketSelector* instance();
/** @brief UmSocketSelector destructor
*
*/
~UmSocketSelector()
{ };
/** @brief UmSocketSelector destructor
*
*/
~UmSocketSelector()
{ };
/** @brief Accessor to total number of UM IP's in Columnstore.xml.
*
* @return Number of UM IP addresses read from Columnstore.xml.
*/
uint32_t ipAddressCount() const;
/** @brief Accessor to total number of UM IP's in Columnstore.xml.
*
* @return Number of UM IP addresses read from Columnstore.xml.
*/
uint32_t ipAddressCount() const;
/** @brief Add a socket/port connection to the connection list.
*
* @param ios (in) socket/port connection to be added.
* @param writeLock (in) mutex to use when writing to ios.
* @return boolean indicating if socket/port connection was added.
*/
bool addConnection( const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock );
/** @brief Add a socket/port connection to the connection list.
*
* @param ios (in) socket/port connection to be added.
* @param writeLock (in) mutex to use when writing to ios.
* @return boolean indicating if socket/port connection was added.
*/
bool addConnection( const SP_UM_IOSOCK& ios,
const SP_UM_MUTEX& writeLock );
/** @brief Delete a socket/port connection from the connection list.
*
* @param ios (in) socket/port connection to be removed.
*/
void delConnection( const IOSocket& ios );
/** @brief Delete a socket/port connection from the connection list.
*
* @param ios (in) socket/port connection to be removed.
*/
void delConnection( const IOSocket& ios );
/** @brief Get the next output IOSocket to use for the specified ios.
*
* @param ios (in) socket/port connection for the incoming message.
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return boolean indicating if operation was successful.
*/
bool nextIOSocket( const IOSocket& ios, SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock );
/** @brief Get the next output IOSocket to use for the specified ios.
*
* @param ios (in) socket/port connection for the incoming message.
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return boolean indicating if operation was successful.
*/
bool nextIOSocket( const IOSocket& ios, SP_UM_IOSOCK& outIos,
SP_UM_MUTEX& writeLock );
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
private:
//...Disable default copy constructor and assignment operator by
// declaring but not defining
UmSocketSelector (const UmSocketSelector& rhs);
UmSocketSelector& operator=(const UmSocketSelector& rhs);
private:
//...Disable default copy constructor and assignment operator by
// declaring but not defining
UmSocketSelector (const UmSocketSelector& rhs);
UmSocketSelector& operator=(const UmSocketSelector& rhs);
UmSocketSelector();
void loadUMModuleInfo();
unsigned int findOrAddUm( const std::string& moduleName );
UmSocketSelector();
void loadUMModuleInfo();
unsigned int findOrAddUm( const std::string& moduleName );
static UmSocketSelector* fpUmSocketSelector;
std::vector<SP_UM_MODIPS> fUmModuleIPs; // UM's and their sockets
static UmSocketSelector* fpUmSocketSelector;
std::vector<SP_UM_MODIPS> fUmModuleIPs; // UM's and their sockets
// std::map that maps an IP address to an index into fUmModuleIPs
IpAddressUmMap_t fIpAddressUmMap;
// std::map that maps an IP address to an index into fUmModuleIPs
IpAddressUmMap_t fIpAddressUmMap;
};
//------------------------------------------------------------------------------
@ -164,80 +164,84 @@ class UmSocketSelector
//------------------------------------------------------------------------------
class UmModuleIPs
{
public:
/** @brief UmModuleIPs constructor.
*
* @param moduleName (in) UM module name for this UmModuleIPs object.
*/
explicit UmModuleIPs ( const std::string& moduleName ) :
fUmModuleName(moduleName),
fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED)
{ }
public:
/** @brief UmModuleIPs constructor.
*
* @param moduleName (in) UM module name for this UmModuleIPs object.
*/
explicit UmModuleIPs ( const std::string& moduleName ) :
fUmModuleName(moduleName),
fNextUmIPSocketIdx(NEXT_IP_SOCKET_UNASSIGNED)
{ }
/** @brief UmModuleIPs destructor.
*
*/
~UmModuleIPs ( )
{ };
/** @brief UmModuleIPs destructor.
*
*/
~UmModuleIPs ( )
{ };
/** @brief Accessor to number of IP's from Columnstore.xml for this UM.
*
* @return Number of IP addresses read from Columnstore.xml for this UM.
*/
uint32_t ipAddressCount() const
{ return fUmIPSocketConns.size(); }
/** @brief Accessor to number of IP's from Columnstore.xml for this UM.
*
* @return Number of IP addresses read from Columnstore.xml for this UM.
*/
uint32_t ipAddressCount() const
{
return fUmIPSocketConns.size();
}
/** @brief Accessor to the module name for this UmModuleIPs object.
*
* @return UM module name.
*/
const std::string& moduleName() const
{ return fUmModuleName; }
/** @brief Accessor to the module name for this UmModuleIPs object.
*
* @return UM module name.
*/
const std::string& moduleName() const
{
return fUmModuleName;
}
/** @brief Add an IP address to this UM module.
*
* @param ip (in) IP address to be added (in network byte order)
*/
void addIP ( in_addr_t ip );
/** @brief Add an IP address to this UM module.
*
* @param ip (in) IP address to be added (in network byte order)
*/
void addIP ( in_addr_t ip );
/** @brief Add specified socket/port to the connection list for this UM.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
* @return boolean indicating if socket/port connection was added.
*/
bool addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Add specified socket/port to the connection list for this UM.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
* @return boolean indicating if socket/port connection was added.
*/
bool addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Get the "next" available socket/port for this UM module.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return bool flag indicating whether a socket/port was available.
*/
bool nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief Get the "next" available socket/port for this UM module.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
* @return bool flag indicating whether a socket/port was available.
*/
bool nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString();
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString();
private:
void advanceToNextIP();
private:
void advanceToNextIP();
static const int32_t NEXT_IP_SOCKET_UNASSIGNED;
std::string fUmModuleName; // UM module name
int32_t fNextUmIPSocketIdx; //index to "next" IP address
boost::mutex fUmModuleMutex;
static const int32_t NEXT_IP_SOCKET_UNASSIGNED;
std::string fUmModuleName; // UM module name
int32_t fNextUmIPSocketIdx; //index to "next" IP address
boost::mutex fUmModuleMutex;
// collection of IP addresses and their corresponding socket/port conns
std::vector<SP_UM_IPCONNS> fUmIPSocketConns;
// collection of IP addresses and their corresponding socket/port conns
std::vector<SP_UM_IPCONNS> fUmIPSocketConns;
};
//------------------------------------------------------------------------------
@ -255,81 +259,85 @@ class UmModuleIPs
//------------------------------------------------------------------------------
class UmIPSocketConns
{
public:
struct UmIOSocketData
{
SP_UM_IOSOCK fSock; // an output IOSocket
SP_UM_MUTEX fMutex; // mutex to be use when writing to fSock
};
public:
struct UmIOSocketData
{
SP_UM_IOSOCK fSock; // an output IOSocket
SP_UM_MUTEX fMutex; // mutex to be use when writing to fSock
};
/** @brief UmIPSocketConns constructor.
*
* @param ip (in) IP address for this UmIPSocketConns object.
*/
explicit UmIPSocketConns( in_addr_t ip ) :
fIpAddress(ip),
fNextIOSocketIdx(NEXT_IOSOCKET_UNASSIGNED)
{ }
/** @brief UmIPSocketConns constructor.
*
* @param ip (in) IP address for this UmIPSocketConns object.
*/
explicit UmIPSocketConns( in_addr_t ip ) :
fIpAddress(ip),
fNextIOSocketIdx(NEXT_IOSOCKET_UNASSIGNED)
{ }
/** @brief UmIPSocketConns destructor.
*
*/
~UmIPSocketConns( )
{ }
/** @brief UmIPSocketConns destructor.
*
*/
~UmIPSocketConns( )
{ }
/** @brief Accessor to the IP address for this UmIPSocketConns object.
*
* @return IP address (in network byte order).
*/
in_addr_t ipAddress ( )
{ return fIpAddress; }
/** @brief Accessor to the IP address for this UmIPSocketConns object.
*
* @return IP address (in network byte order).
*/
in_addr_t ipAddress ( )
{
return fIpAddress;
}
/** @brief Accessor to socket/port connection count for this IP address.
*
* @return socket/port connection count.
*/
uint32_t count( )
{ return fIOSockets.size(); }
/** @brief Accessor to socket/port connection count for this IP address.
*
* @return socket/port connection count.
*/
uint32_t count( )
{
return fIOSockets.size();
}
/** @brief Add specified socket/port to the connection list.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
*/
void addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Add specified socket/port to the connection list.
*
* @param ioSock (in) socket/port to add to the connection list.
* @param writeLock (in) mutex to use when writing to ioSock.
*/
void addSocketConn ( const SP_UM_IOSOCK& ioSock,
const SP_UM_MUTEX& writeLock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Delete specified socket/port from the connection list.
*
* @param ioSock (in) socket/port to delete from the connection list.
*/
void delSocketConn ( const IOSocket& ioSock );
/** @brief Get the "next" available socket/port for this IP address.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
*/
void nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief Get the "next" available socket/port for this IP address.
*
* @param outIos (out) socket/port connection to use for the response.
* @param writeLock (out) mutex to use when writing to outIos.
*/
void nextIOSocket ( SP_UM_IOSOCK& outIos, SP_UM_MUTEX& writeLock );
/** @brief Convert network byte ordered IP address to string
*
* @param ipString (out) IP address string;
* ipString must be an array of size INET_ADDRSTRLEN.
* @return IP address string (same as ipString)
*/
static char* nwToString( in_addr_t addr, char* ipString );
/** @brief Convert network byte ordered IP address to string
*
* @param ipString (out) IP address string;
* ipString must be an array of size INET_ADDRSTRLEN.
* @return IP address string (same as ipString)
*/
static char* nwToString( in_addr_t addr, char* ipString );
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
/** @brief toString method used in logging, debugging, etc.
*
*/
const std::string toString() const;
private:
static const int32_t NEXT_IOSOCKET_UNASSIGNED;
in_addr_t fIpAddress; // IP address in network byte order
int32_t fNextIOSocketIdx;//index to "next" socket/port
std::vector<UmIOSocketData> fIOSockets;//socket/port list for fIpAddress
private:
static const int32_t NEXT_IOSOCKET_UNASSIGNED;
in_addr_t fIpAddress; // IP address in network byte order
int32_t fNextIOSocketIdx;//index to "next" socket/port
std::vector<UmIOSocketData> fIOSockets;//socket/port list for fIpAddress
};
}