database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

631
primitives/blockcache/bcTest.cpp
View File

@ -40,346 +40,405 @@ using namespace BRM;
using namespace dbbc;
using namespace std;
Stats* gPMStatsPtr=NULL;
bool gPMProfOn=false;
uint32_t gSession=0;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
int fLoops=1;
int thr_cnt=1;
uint64_t bfoundTot=0;
uint64_t bnfoundTot=0;
uint64_t rfoundTot=0;
uint64_t rnfoundTot=0;
uint64_t rangeOpCountTot=0;
uint64_t blockOpCountTot=0;
uint64_t noOpCountTot=0;
int fLoops = 1;
int thr_cnt = 1;
uint64_t bfoundTot = 0;
uint64_t bnfoundTot = 0;
uint64_t rfoundTot = 0;
uint64_t rnfoundTot = 0;
uint64_t rangeOpCountTot = 0;
uint64_t blockOpCountTot = 0;
uint64_t noOpCountTot = 0;
struct thr_wait_struct {
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
struct thr_wait_struct
{
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
};
typedef thr_wait_struct thr_wait_t;
uint32_t cacheSize=10240;
uint32_t cacheSize = 10240;
// BlockRequestProcessor BRP(cacheSize, 4, 16);
BlockRequestProcessor* BRP;
BRM::VER_t ver=0xFFFF;
u_int64_t totBlocks=0;
BRM::VER_t ver = 0xFFFF;
u_int64_t totBlocks = 0;
void* thr_client(void* clientArgs) {
blockCacheClient bc(*BRP);
uint64_t bfound=0;
uint64_t bnfound=0;
uint64_t rfound=0;
uint64_t rnfound=0;
uint64_t rangeOpCount=0;
uint64_t blockOpCount=0;
uint64_t noOpCount=0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate=0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
void* thr_client(void* clientArgs)
{
blockCacheClient bc(*BRP);
uint64_t bfound = 0;
uint64_t bnfound = 0;
uint64_t rfound = 0;
uint64_t rnfound = 0;
uint64_t rangeOpCount = 0;
uint64_t blockOpCount = 0;
uint64_t noOpCount = 0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate = 0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
gettimeofday(&tv, NULL);
uint8_t fbData[8192]={0};
FileBuffer* fbPtr=NULL;
int ret=0;
int idx=0;
uint32_t jdx=0;
uint32_t l=0;
uint32_t m=0;
uint32_t start=0, max=0, size=0;
LBIDRange_v& r=range_thr[0];
for(idx=0; idx<fLoops; idx++) {
for (jdx=0; jdx<range_thr.size(); jdx++) {
ret=0;
r = range_thr[jdx];
for(l=0; l<r.size(); l++) {
start = r[l].start;
size = r[l].size;
max = r[l].start+r[l].size;
//cout << "readThr " <<start << " " << max << endl;
for(m=start;m<max; m++) {
//const FileBuffer* ptr=bc.read(m, ver, fbPtr);
bool ptr=bc.read(m, ver, fbData);
//cout << "bc ptr " << ptr << endl;
if (ptr) {
bfound++;
//memcpy(fbData, ptr->getData(), ptr->datLen());
} else {
bnfound++;
//cout << "bc fail:" << m << " " <<start << " " << size << " " << max << endl;
}
ptr=false;
}
}
}
}
gettimeofday(&tv, NULL);
uint8_t fbData[8192] = {0};
FileBuffer* fbPtr = NULL;
int ret = 0;
int idx = 0;
uint32_t jdx = 0;
uint32_t l = 0;
uint32_t m = 0;
uint32_t start = 0, max = 0, size = 0;
LBIDRange_v& r = range_thr[0];
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
for (idx = 0; idx < fLoops; idx++)
{
for (jdx = 0; jdx < range_thr.size(); jdx++)
{
ret = 0;
r = range_thr[jdx];
if (elTime>0) {
avgTot=(bfound+rfound)/elTime;
rangeAvg=(rfound)/elTime;
blkAvg=(bfound)/elTime;
} else {
avgTot=bfound+rfound;
rangeAvg=rfound;
blkAvg=bfound;
}
for (l = 0; l < r.size(); l++)
{
start = r[l].start;
size = r[l].size;
max = r[l].start + r[l].size;
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
//cout << "readThr " <<start << " " << max << endl;
for (m = start; m < max; m++)
{
//const FileBuffer* ptr=bc.read(m, ver, fbPtr);
bool ptr = bc.read(m, ver, fbData);
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot+=bfound;
bnfoundTot+=bnfound;
rfoundTot+=rfound;
rnfoundTot+=rnfound;
rangeOpCountTot+=rangeOpCount;
blockOpCountTot+=blockOpCount;
noOpCountTot+=noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
//cout << "bc ptr " << ptr << endl;
if (ptr)
{
bfound++;
//memcpy(fbData, ptr->getData(), ptr->datLen());
}
else
{
bnfound++;
//cout << "bc fail:" << m << " " <<start << " " << size << " " << max << endl;
}
ptr = false;
}
}
}
}
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfound + rfound) / elTime;
rangeAvg = (rfound) / elTime;
blkAvg = (bfound) / elTime;
}
else
{
avgTot = bfound + rfound;
rangeAvg = rfound;
blkAvg = bfound;
}
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot += bfound;
bnfoundTot += bnfound;
rfoundTot += rfound;
rnfoundTot += rnfound;
rangeOpCountTot += rangeOpCount;
blockOpCountTot += blockOpCount;
noOpCountTot += noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
return NULL;
return NULL;
} // end thr_client
void LoadRange(const LBIDRange_v& v, uint32_t& loadCount)
{
blockCacheClient bc(*BRP);
blockCacheClient bc(*BRP);
uint32_t rCount=0;
for (uint32_t i =0; i<v.size() ; i++)
{
InlineLBIDRange r={v[i].start, v[i].size};
if (r.size+loadCount>cacheSize)
r.size=(r.size+loadCount)-cacheSize;
if (r.size<=1024) {
//cout << "check() " << r.start << " " << r.size << endl;
bc.check(r, ver, rCount );
loadCount+=rCount;
}
rCount=0;
uint32_t rCount = 0;
}
for (uint32_t i = 0; i < v.size() ; i++)
{
InlineLBIDRange r = {v[i].start, v[i].size};
if (r.size + loadCount > cacheSize)
r.size = (r.size + loadCount) - cacheSize;
if (r.size <= 1024)
{
//cout << "check() " << r.start << " " << r.size << endl;
bc.check(r, ver, rCount );
loadCount += rCount;
}
rCount = 0;
}
}
void ReadRange(const LBIDRange_v& v)
{
blockCacheClient bc(*BRP);
int found=0;
int notfound=0;
int ret=0;
for(uint32_t i=0; i<v.size(); i++)
{
const InlineLBIDRange r={v[i].start, v[i].size};
FileBuffer fb(-1, -1);
//cout << "read() " << r.start << " " << r.size << endl;
for(int j=r.start; j<r.start+r.size; j++)
{
if (r.size > 1024)
continue;
ret=bc.read(j, ver, fb);
if (ret)
found++;
else {
notfound++;
}
ret=0;
}
totBlocks+=found;
totBlocks+=notfound;
found=0;
notfound=0;
}
blockCacheClient bc(*BRP);
int found = 0;
int notfound = 0;
int ret = 0;
for (uint32_t i = 0; i < v.size(); i++)
{
const InlineLBIDRange r = {v[i].start, v[i].size};
FileBuffer fb(-1, -1);
//cout << "read() " << r.start << " " << r.size << endl;
for (int j = r.start; j < r.start + r.size; j++)
{
if (r.size > 1024)
continue;
ret = bc.read(j, ver, fb);
if (ret)
found++;
else
{
notfound++;
}
ret = 0;
}
totBlocks += found;
totBlocks += notfound;
found = 0;
notfound = 0;
}
}
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(*BRP);
bool b;
bc.check(lbid, ver, false, b);
blockCacheClient bc(*BRP);
bool b;
bc.check(lbid, ver, false, b);
}
void ReadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
static int found=0, notfound=0;
uint8_t** d=NULL; //[8192];
blockCacheClient bc(*BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if (ret)
found++;
else
notfound++;
static int found = 0, notfound = 0;
uint8_t** d = NULL; //[8192];
blockCacheClient bc(*BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if ((found+notfound)%10000==0)
cout << "found " << found << " notfound " << notfound << endl;
if (ret)
found++;
else
notfound++;
if ((found + notfound) % 10000 == 0)
cout << "found " << found << " notfound " << notfound << endl;
}
//
int main(int argc, char *argv[]) {
int main(int argc, char* argv[])
{
if (argc>=2) thr_cnt=atoi(argv[1]);
if (argc>=3) fLoops=atoi(argv[2]);
if (argc>=4) cacheSize=atoi(argv[3]);
if (thr_cnt<=0) thr_cnt=1;
if (thr_cnt>1024) thr_cnt=1024;
if (fLoops<=0) fLoops=1;
if (argc >= 2) thr_cnt = atoi(argv[1]);
BlockRequestProcessor brp(cacheSize, 4, 16);
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
BRP=&brp;
if (argc >= 3) fLoops = atoi(argv[2]);
cout << "Reading Ranges " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid=3000;
uint32_t totalExt=0;
do {
int ret = dbrm.lookup(oid, r);
if (ret==0 && r.size() > 0) {
lowlbid = (r[0].start/extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start=lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx=0; idx<r.size(); idx++)
totalExt+=r[idx].size;
ranges.push_back(r);
cout << ".";
if (ranges.size()%50==0)
cout <<endl;
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
if (argc >= 4) cacheSize = atoi(argv[3]);
cout << endl << ranges.size() << " ranges found" << endl;
if (thr_cnt <= 0) thr_cnt = 1;
gettimeofday(&tv, NULL);
uint32_t blksLoaded=0;
uint32_t rangesLoaded=0;
cout << "Loading Ranges " << endl;
for (uint32_t i =0; i<ranges.size() && blksLoaded < (cacheSize-1024); i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
cout << ".";
if (i%50==0&&i)
cout <<endl;
}
cout << endl;
if (thr_cnt > 1024) thr_cnt = 1024;
gettimeofday(&tv2, NULL);
cout << endl << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec <<endl;
if (fLoops <= 0) fLoops = 1;
while (ranges.size()>=rangesLoaded) ranges.pop_back();
BlockRequestProcessor brp(cacheSize, 4, 16);
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
BRP = &brp;
ranges.pop_back();
cout << "Reading Ranges " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid = 3000;
uint32_t totalExt = 0;
do
{
int ret = dbrm.lookup(oid, r);
if (ret == 0 && r.size() > 0)
{
lowlbid = (r[0].start / extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start = lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx = 0; idx < r.size(); idx++)
totalExt += r[idx].size;
ranges.push_back(r);
cout << ".";
if (ranges.size() % 50 == 0)
cout << endl;
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
cout << endl << ranges.size() << " ranges found" << endl;
gettimeofday(&tv, NULL);
uint32_t blksLoaded = 0;
uint32_t rangesLoaded = 0;
cout << "Loading Ranges " << endl;
for (uint32_t i = 0; i < ranges.size() && blksLoaded < (cacheSize - 1024); i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
cout << ".";
if (i % 50 == 0 && i)
cout << endl;
}
cout << endl;
gettimeofday(&tv2, NULL);
cout << endl << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec << endl;
while (ranges.size() >= rangesLoaded) ranges.pop_back();
ranges.pop_back();
#ifdef BLAH
for (uint32_t i =0; i<ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i =0; i<ranges.size(); i++)
{
LBIDRange_v rv=ranges[i];
for(uint32_t j=0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for(uint32_t k=l.start; k<l.start+l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
for (uint32_t i = 0; i < ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i = 0; i < ranges.size(); i++)
{
LBIDRange_v rv = ranges[i];
for (uint32_t j = 0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for (uint32_t k = l.start; k < l.start + l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
#endif
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait={0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
//start threads running
cout << "Starting reader threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt*(sizeof(pthread_t)));
for(int i=0; i<thr_cnt; i++) {
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
while (thr_wait.predicate>0) {
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
//start threads running
cout << "Starting reader threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt * (sizeof(pthread_t)));
// join threads back to main
for(int i=0; i<thr_cnt; i++) {
pthread_join(thr_id[i], NULL);
}
for (int i = 0; i < thr_cnt; i++)
{
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
//uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
if (elTime>0) {
avgTot=(bfoundTot+rfoundTot)/elTime;
rangeAvg=(rfoundTot)/elTime;
blkAvg=(bfoundTot)/elTime;
} else {
avgTot=bfoundTot+rfoundTot;
rangeAvg=rfoundTot;
blkAvg=bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot/thr_cnt << endl << endl;
while (thr_wait.predicate > 0)
{
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
// join threads back to main
for (int i = 0; i < thr_cnt; i++)
{
pthread_join(thr_id[i], NULL);
}
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
//uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfoundTot + rfoundTot) / elTime;
rangeAvg = (rfoundTot) / elTime;
blkAvg = (bfoundTot) / elTime;
}
else
{
avgTot = bfoundTot + rfoundTot;
rangeAvg = rfoundTot;
blkAvg = bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl << endl;
return 0;
return 0;
} // end main

5
primitives/blockcache/blockcacheclient.cpp
View File

@ -21,11 +21,12 @@
*
* jrodriguez@calpont.com *
***************************************************************************/
#include <sstream>
#include <string>
#include "blockcacheclient.h"
namespace dbbc {
namespace dbbc
{
}

194
primitives/blockcache/blockcacheclient.h
View File

@ -35,7 +35,7 @@
/**
* @brief API for the Disk Block Buffer Cache
*
*
*
*/
namespace dbbc
@ -45,103 +45,131 @@ class blockCacheClient
{
public:
/**
* @brief ctor requires reference to BlockRequestProcessor object
**/
blockCacheClient(BlockRequestProcessor& brp) : fBCCBrp(&brp) {}
/**
* @brief ctor requires reference to BlockRequestProcessor object
**/
blockCacheClient(BlockRequestProcessor& brp) : fBCCBrp(&brp) {}
/**
* @brief dtor
**/
virtual ~blockCacheClient() {}
/**
* @brief dtor
**/
virtual ~blockCacheClient() {}
/**
* @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache.
**/
inline void check(BRM::LBID_t lbid, const BRM::QueryContext &ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache) {
fBCCBrp->check(lbid, ver, txn, flg, compType, wasBlockInCache); }
/**
* @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache.
**/
inline void check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache)
{
fBCCBrp->check(lbid, ver, txn, flg, compType, wasBlockInCache);
}
/**
* @brief verify all Disk Blocks for the LBID range are loaded into the Cache
**/
inline void check(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType,
uint32_t& rCount) {
fBCCBrp->check(range, ver, txn, compType, rCount); }
/**
* @brief verify all Disk Blocks for the LBID range are loaded into the Cache
**/
inline void check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType,
uint32_t& rCount)
{
fBCCBrp->check(range, ver, txn, compType, rCount);
}
inline FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver, bool flg) {
return fBCCBrp->getBlockPtr(lbid, ver, flg); }
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) {
return fBCCBrp->read(lbid, ver, fb); }
inline FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver, bool flg)
{
return fBCCBrp->getBlockPtr(lbid, ver, flg);
}
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) {
return fBCCBrp->read(lbid, ver, bufferPtr); }
inline const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType,
void* bufferPtr, bool flg, bool &wasCached, bool *wasVersioned = NULL, bool insertIntoCache = true,
bool readFromCache = true) {
return fBCCBrp->getBlock(lbid, ver, txn, compType, bufferPtr, flg, wasCached, wasVersioned, insertIntoCache,
readFromCache); }
inline int getCachedBlocks(const BRM::LBID_t *lbids, const BRM::VER_t *vers, uint8_t **bufferPtrs,
bool *wasCached, uint32_t blockCount)
{ return fBCCBrp->getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount); }
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
return fBCCBrp->read(lbid, ver, fb);
}
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) {
return fBCCBrp->exists(lbid, ver); }
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
return fBCCBrp->read(lbid, ver, bufferPtr);
}
/**
* @brief flush the cache
**/
inline void flushCache() {
fBCCBrp->flushCache(); }
inline const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType,
void* bufferPtr, bool flg, bool& wasCached, bool* wasVersioned = NULL, bool insertIntoCache = true,
bool readFromCache = true)
{
return fBCCBrp->getBlock(lbid, ver, txn, compType, bufferPtr, flg, wasCached, wasVersioned, insertIntoCache,
readFromCache);
}
/**
* @brief flush one LBID@Ver from the cache
**/
inline void flushOne(const BRM::LBID_t& lbid, const BRM::VER_t& ver) {
fBCCBrp->flushOne(lbid, ver); }
inline int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** bufferPtrs,
bool* wasCached, uint32_t blockCount)
{
return fBCCBrp->getCachedBlocks(lbids, vers, bufferPtrs, wasCached, blockCount);
}
/**
* @brief flush specific LBID/version pairs from the cache
**/
inline void flushMany(const LbidAtVer* laVptr, uint32_t cnt) {
fBCCBrp->flushMany(laVptr, cnt); }
/**
* @brief flush all versions of the given lbids from the cache.
**/
inline void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) {
fBCCBrp->flushManyAllversion(laVptr, cnt); }
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
return fBCCBrp->exists(lbid, ver);
}
/**
* @brief Flush all versions of all LBIDs belonging to the given OIDs.
*/
inline void flushOIDs(const uint32_t *oids, uint32_t count) {
fBCCBrp->flushOIDs(oids, count); }
/**
* @brief flush the cache
**/
inline void flushCache()
{
fBCCBrp->flushCache();
}
/**
* @brief Flush all versions of a partition from the given OIDs.
*/
inline void flushPartition(const std::vector<BRM::OID_t> &oids, const std::set<BRM::LogicalPartition> partitions) {
fBCCBrp->flushPartition(oids, partitions); }
/**
* @brief flush one LBID@Ver from the cache
**/
inline void flushOne(const BRM::LBID_t& lbid, const BRM::VER_t& ver)
{
fBCCBrp->flushOne(lbid, ver);
}
/**
* @brief flush specific LBID/version pairs from the cache
**/
inline void flushMany(const LbidAtVer* laVptr, uint32_t cnt)
{
fBCCBrp->flushMany(laVptr, cnt);
}
/**
* @brief flush all versions of the given lbids from the cache.
**/
inline void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt)
{
fBCCBrp->flushManyAllversion(laVptr, cnt);
}
/**
* @brief Flush all versions of all LBIDs belonging to the given OIDs.
*/
inline void flushOIDs(const uint32_t* oids, uint32_t count)
{
fBCCBrp->flushOIDs(oids, count);
}
/**
* @brief Flush all versions of a partition from the given OIDs.
*/
inline void flushPartition(const std::vector<BRM::OID_t>& oids, const std::set<BRM::LogicalPartition> partitions)
{
fBCCBrp->flushPartition(oids, partitions);
}
private:
/**
* @brief pointer to the BlockRequestProcessor object on which the API will operate
**/
BlockRequestProcessor* fBCCBrp;
/**
* @brief pointer to the BlockRequestProcessor object on which the API will operate
**/
BlockRequestProcessor* fBCCBrp;
//do not implement
blockCacheClient(const blockCacheClient& bc);
blockCacheClient& operator=(const blockCacheClient& blk);
//do not implement
blockCacheClient(const blockCacheClient& bc);
blockCacheClient& operator=(const blockCacheClient& blk);
};

349
primitives/blockcache/blockrequestprocessor.cpp
View File

@ -35,210 +35,233 @@ using namespace std;
#include "dbrm.h"
#include "pp_logger.h"
namespace dbbc {
namespace dbbc
{
BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks,
int thrCount,
int blocksPerRead,
uint32_t deleteBlocks,
uint32_t blckSz) :
fbMgr(numBlcks, blckSz, deleteBlocks),
fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead)
int thrCount,
int blocksPerRead,
uint32_t deleteBlocks,
uint32_t blckSz) :
fbMgr(numBlcks, blckSz, deleteBlocks),
fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead)
{
//pthread_mutex_init(&check_mutex, NULL);
config::Config* fConfig=config::Config::makeConfig();
string val = fConfig->getConfig("DBBC", "BRPTracing");
int temp=0;
//pthread_mutex_init(&check_mutex, NULL);
config::Config* fConfig = config::Config::makeConfig();
string val = fConfig->getConfig("DBBC", "BRPTracing");
int temp = 0;
#ifdef _MSC_VER
int tid = GetCurrentThreadId();
int tid = GetCurrentThreadId();
#else
pthread_t tid = pthread_self();
pthread_t tid = pthread_self();
#endif
if (val.length()>0) temp=static_cast<int>(config::Config::fromText(val));
if (val.length() > 0) temp = static_cast<int>(config::Config::fromText(val));
if (temp > 0)
fTrace=true;
else
fTrace=false;
if (temp > 0)
fTrace = true;
else
fTrace = false;
if (fTrace)
{
ostringstream brpLogFileName;
if (fTrace)
{
ostringstream brpLogFileName;
#ifdef _MSC_VER
brpLogFileName << "C:/Calpont/log/trace/brp." << tid;
brpLogFileName << "C:/Calpont/log/trace/brp." << tid;
#else
brpLogFileName << "/var/log/mariadb/columnstore/trace/brp." << tid;
brpLogFileName << "/var/log/mariadb/columnstore/trace/brp." << tid;
#endif
fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate);
}
fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate);
}
}
BlockRequestProcessor::~BlockRequestProcessor()
{
//pthread_mutex_destroy(&check_mutex);
if (fTrace)
fLogFile.close();
//pthread_mutex_destroy(&check_mutex);
if (fTrace)
fLogFile.close();
}
void BlockRequestProcessor::stop() {
fBRPRequestQueue.stop();
fIOMgr.stop();
void BlockRequestProcessor::stop()
{
fBRPRequestQueue.stop();
fIOMgr.stop();
}
int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType, uint32_t& lbidCount) {
uint64_t maxLbid = range.start; // highest existent lbid
uint64_t rangeLen = range.size;
uint64_t idx;
uint64_t adjSz;
struct timespec start_tm;
lbidCount = 0;
int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType, uint32_t& lbidCount)
{
uint64_t maxLbid = range.start; // highest existent lbid
uint64_t rangeLen = range.size;
uint64_t idx;
uint64_t adjSz;
struct timespec start_tm;
lbidCount = 0;
if (fTrace)
clock_gettime(CLOCK_MONOTONIC, &start_tm);
if (fTrace)
clock_gettime(CLOCK_MONOTONIC, &start_tm);
for (idx = 0; fbMgr.exists(maxLbid, ver.currentScn) == true && idx<rangeLen; maxLbid++, idx++)
(void)0;
for (idx = 0; fbMgr.exists(maxLbid, ver.currentScn) == true && idx < rangeLen; maxLbid++, idx++)
(void)0;
if (idx == rangeLen) { // range is already loaded
if (fTrace)
{
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< 0 << " "
<< 0 << " "
<< 0 << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
}
return 0;
}
if (idx == rangeLen) // range is already loaded
{
if (fTrace)
{
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< 0 << " "
<< 0 << " "
<< 0 << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
}
adjSz=rangeLen-idx;
BRM::InlineLBIDRange adjRange;
adjRange.start=maxLbid;
adjRange.size=adjSz;
fileRequest rqstBlk(adjRange, ver, txn, compType);
check(rqstBlk);
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
throw runtime_error(rqstBlk.RequestStatusStr());
lbidCount=rqstBlk.BlocksRead();
return 0;
}
if (fTrace) {
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< adjSz << " "
<< rqstBlk.BlocksRead() << " "
<< rqstBlk.BlocksLoaded() << " "
<< right << fixed << ((double)(start_tm.tv_sec+(1.e-9*start_tm.tv_nsec)))
<< endl;
}
adjSz = rangeLen - idx;
BRM::InlineLBIDRange adjRange;
adjRange.start = maxLbid;
adjRange.size = adjSz;
fileRequest rqstBlk(adjRange, ver, txn, compType);
check(rqstBlk);
return rqstBlk.BlocksLoaded();
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
throw runtime_error(rqstBlk.RequestStatusStr());
lbidCount = rqstBlk.BlocksRead();
if (fTrace)
{
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookupLocal(maxLbid, ver.currentScn, false, oid, dbroot, partNum, segNum, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< adjSz << " "
<< rqstBlk.BlocksRead() << " "
<< rqstBlk.BlocksLoaded() << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
}
return rqstBlk.BlocksLoaded();
} // check
int BlockRequestProcessor::check(fileRequest& rqstBlk) {
rqstBlk.frMutex().lock();
rqstBlk.SetPredicate(fileRequest::SENDING);
sendRequest(rqstBlk); // start file read request
int BlockRequestProcessor::check(fileRequest& rqstBlk)
{
rqstBlk.frMutex().lock();
rqstBlk.SetPredicate(fileRequest::SENDING);
sendRequest(rqstBlk); // start file read request
while(rqstBlk.frPredicate()<fileRequest::COMPLETE)
rqstBlk.frCond().wait(rqstBlk.frMutex());
rqstBlk.frMutex().unlock();
while (rqstBlk.frPredicate() < fileRequest::COMPLETE)
rqstBlk.frCond().wait(rqstBlk.frMutex());
return 0;
rqstBlk.frMutex().unlock();
return 0;
}
// For future use. Not currently used.
int BlockRequestProcessor::check(BRM::LBID_t lbid, const BRM::QueryContext &ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache) {
if (fbMgr.exists(lbid, ver.currentScn)==true) {
wasBlockInCache = true;
return 0;
} else {
wasBlockInCache = false;
fileRequest rqstBlk(lbid, ver, flg, txn, compType);
int ret=check(rqstBlk);
if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) {
throw runtime_error(rqstBlk.RequestStatusStr());
}
return ret;
}
int BlockRequestProcessor::check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache)
{
if (fbMgr.exists(lbid, ver.currentScn) == true)
{
wasBlockInCache = true;
return 0;
}
else
{
wasBlockInCache = false;
fileRequest rqstBlk(lbid, ver, flg, txn, compType);
int ret = check(rqstBlk);
if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
{
throw runtime_error(rqstBlk.RequestStatusStr());
}
return ret;
}
}
const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext &ver, BRM::VER_t txn,
int compType, void* bufferPtr, bool vbFlg, bool &wasCached, bool *versioned, bool insertIntoCache,
bool readFromCache)
const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, BRM::VER_t txn,
int compType, void* bufferPtr, bool vbFlg, bool& wasCached, bool* versioned, bool insertIntoCache,
bool readFromCache)
{
if (readFromCache) {
HashObject_t hashObj(lbid, ver.currentScn, 0);
wasCached = fbMgr.find(hashObj, bufferPtr);
if (wasCached)
return 1;
}
if (readFromCache)
{
HashObject_t hashObj(lbid, ver.currentScn, 0);
wasCached = fbMgr.find(hashObj, bufferPtr);
wasCached = false;
fileRequest rqstBlk(lbid, ver, vbFlg, txn, compType, (uint8_t *) bufferPtr, insertIntoCache);
check(rqstBlk);
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
{
ostringstream os;
os << "BRP::getBlock(): got a BRM lookup error. LBID=" << lbid << " ver=" << ver << " txn="
<< txn << " vbFlg=" << (int) vbFlg;
primitiveprocessor::Logger logger;
logger.logMessage(os.str(), false);
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
{
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
{
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
}
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL) {
throw runtime_error(rqstBlk.RequestStatusStr());
}
if (versioned)
*versioned = rqstBlk.versioned();
return 1;
if (wasCached)
return 1;
}
wasCached = false;
fileRequest rqstBlk(lbid, ver, vbFlg, txn, compType, (uint8_t*) bufferPtr, insertIntoCache);
check(rqstBlk);
if (rqstBlk.RequestStatus() == fileRequest::BRM_LOOKUP_ERROR)
{
ostringstream os;
os << "BRP::getBlock(): got a BRM lookup error. LBID=" << lbid << " ver=" << ver << " txn="
<< txn << " vbFlg=" << (int) vbFlg;
primitiveprocessor::Logger logger;
logger.logMessage(os.str(), false);
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_BRM_LOOKUP), logging::ERR_BRM_LOOKUP);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_EINVAL)
{
throw logging::IDBExcept(logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_O_DIRECT),
logging::ERR_O_DIRECT);
}
else if (rqstBlk.RequestStatus() == fileRequest::FS_ENOENT)
{
throw logging::IDBExcept(
logging::IDBErrorInfo::instance()->errorMsg(logging::ERR_ENOENT),
logging::ERR_ENOENT);
}
else if (rqstBlk.RequestStatus() != fileRequest::SUCCESSFUL)
{
throw runtime_error(rqstBlk.RequestStatusStr());
}
if (versioned)
*versioned = rqstBlk.versioned();
return 1;
}
int BlockRequestProcessor::getCachedBlocks(const BRM::LBID_t *lbids, const BRM::VER_t *vers,
uint8_t **ptrs, bool *wasCached, uint32_t count)
int BlockRequestProcessor::getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers,
uint8_t** ptrs, bool* wasCached, uint32_t count)
{
return fbMgr.bulkFind(lbids, vers, ptrs, wasCached, count);
return fbMgr.bulkFind(lbids, vers, ptrs, wasCached, count);
}

215
primitives/blockcache/blockrequestprocessor.h
View File

@ -38,129 +38,160 @@
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
namespace dbbc {
namespace dbbc
{
typedef std::list<FileBuffer> FileBufferList_t;
/**
* @brief class to control the populating of the Disk Block Buffer Cache and manage Block requests.
**/
class BlockRequestProcessor {
class BlockRequestProcessor
{
public:
/**
* @brief default ctor
**/
BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks=0,
uint32_t blckSz=BLOCK_SIZE);
/**
* @brief default dtor
**/
virtual ~BlockRequestProcessor();
/**
* @brief default ctor
**/
BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0,
uint32_t blckSz = BLOCK_SIZE);
/**
* @brief send a request for disk blocks to the IO manager
**/
int sendRequest(fileRequest& blk) {
return fBRPRequestQueue.push(blk); }
/**
* @brief default dtor
**/
virtual ~BlockRequestProcessor();
/**
* @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not
**/
int check(BRM::LBID_t lbid, const BRM::QueryContext &ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache);
/**
* @brief send a request for disk blocks to the IO manager
**/
int sendRequest(fileRequest& blk)
{
return fBRPRequestQueue.push(blk);
}
/**
* @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not
**/
int check(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, const BRM::VER_t txn, const int compType,
uint32_t& lbidCount);
/**
* @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not
**/
int check(BRM::LBID_t lbid, const BRM::QueryContext& ver, BRM::VER_t txn, bool flg, int compType, bool& wasBlockInCache);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver, bool flg) {
return fbMgr.findPtr(HashObject_t(lbid, ver, flg)); }
/**
* @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not
**/
int check(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, const BRM::VER_t txn, const int compType,
uint32_t& lbidCount);
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) {
return (fbMgr.find(HashObject_t(lbid, ver, 0), fb) ? 1 : 0); }
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver, bool flg)
{
return fbMgr.findPtr(HashObject_t(lbid, ver, flg));
}
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t &ver, void* bufferPtr) {
return (fbMgr.find(HashObject_t(lbid, ver, 0), bufferPtr) ? 1 : 0); }
const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext &ver, BRM::VER_t txn, int compType,
void* bufferPtr, bool flg, bool &wasCached, bool *wasVersioned, bool insertIntoCache,
bool readFromCache);
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
return (fbMgr.find(HashObject_t(lbid, ver, 0), fb) ? 1 : 0);
}
int getCachedBlocks(const BRM::LBID_t *lbids, const BRM::VER_t *vers, uint8_t **ptrs,
bool *wasCached, uint32_t count);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
inline const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
return (fbMgr.find(HashObject_t(lbid, ver, 0), bufferPtr) ? 1 : 0);
}
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver) {
return fbMgr.exists(HashObject_t(lbid, ver, 0)); }
const int getBlock(const BRM::LBID_t& lbid, const BRM::QueryContext& ver, BRM::VER_t txn, int compType,
void* bufferPtr, bool flg, bool& wasCached, bool* wasVersioned, bool insertIntoCache,
bool readFromCache);
/**
* @brief
**/
void flushCache() {
fbMgr.flushCache(); }
int getCachedBlocks(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** ptrs,
bool* wasCached, uint32_t count);
/**
* @brief
**/
void flushOne(BRM::LBID_t lbid, BRM::VER_t ver) {
fbMgr.flushOne(lbid, ver); }
inline bool exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
return fbMgr.exists(HashObject_t(lbid, ver, 0));
}
void flushMany(const LbidAtVer* laVptr, uint32_t cnt) {
fbMgr.flushMany(laVptr, cnt); }
void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt) {
fbMgr.flushManyAllversion(laVptr, cnt); }
/**
* @brief
**/
void flushCache()
{
fbMgr.flushCache();
}
void flushOIDs(const uint32_t *oids, uint32_t count) {
fbMgr.flushOIDs(oids, count); }
/**
* @brief
**/
void flushOne(BRM::LBID_t lbid, BRM::VER_t ver)
{
fbMgr.flushOne(lbid, ver);
}
void flushPartition(const std::vector<BRM::OID_t> &oids, const std::set<BRM::LogicalPartition> &partitions) {
fbMgr.flushPartition(oids, partitions); }
void flushMany(const LbidAtVer* laVptr, uint32_t cnt)
{
fbMgr.flushMany(laVptr, cnt);
}
void setReportingFrequency(const uint32_t d) {
fbMgr.setReportingFrequency(d); }
void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt)
{
fbMgr.flushManyAllversion(laVptr, cnt);
}
uint32_t ReportingFrequency() const {return fbMgr.ReportingFrequency();}
void flushOIDs(const uint32_t* oids, uint32_t count)
{
fbMgr.flushOIDs(oids, count);
}
void flushPartition(const std::vector<BRM::OID_t>& oids, const std::set<BRM::LogicalPartition>& partitions)
{
fbMgr.flushPartition(oids, partitions);
}
void setReportingFrequency(const uint32_t d)
{
fbMgr.setReportingFrequency(d);
}
uint32_t ReportingFrequency() const
{
return fbMgr.ReportingFrequency();
}
std::ostream& formatLRUList(std::ostream& os) const
{
return fbMgr.formatLRUList(os);
}
std::ostream& formatLRUList(std::ostream& os) const {
return fbMgr.formatLRUList(os); }
private:
FileBufferMgr fbMgr;
fileBlockRequestQueue fBRPRequestQueue;
ioManager fIOMgr;
boost::mutex check_mutex;
/**
* helper function for public check functions
**/
int check(fileRequest& rqstBlk);
FileBufferMgr fbMgr;
fileBlockRequestQueue fBRPRequestQueue;
ioManager fIOMgr;
boost::mutex check_mutex;
/**
* send stop requests for IOmanager and request Q
**/
void stop();
/**
* helper function for public check functions
**/
int check(fileRequest& rqstBlk);
std::ofstream fLogFile;
bool fTrace;
/**
* send stop requests for IOmanager and request Q
**/
void stop();
BRM::DBRM fdbrm;
std::ofstream fLogFile;
bool fTrace;
BRM::DBRM fdbrm;
// do not implement
BlockRequestProcessor(const BlockRequestProcessor& brp);
BlockRequestProcessor& operator=(const BlockRequestProcessor& brp);
// do not implement
BlockRequestProcessor(const BlockRequestProcessor& brp);
BlockRequestProcessor& operator=(const BlockRequestProcessor& brp);
};
}

69
primitives/blockcache/fileblockrequestqueue.cpp
View File

@ -30,62 +30,69 @@
using namespace std;
namespace dbbc {
namespace dbbc
{
fileBlockRequestQueue::fileBlockRequestQueue() : queueSize(0), readersWaiting(0)
{
//pthread_mutex_init(&mutex, NULL);
//pthread_cond_init(&notEmpty, NULL);
//pthread_mutex_init(&mutex, NULL);
//pthread_cond_init(&notEmpty, NULL);
}
fileBlockRequestQueue::~fileBlockRequestQueue()
{
//pthread_cond_destroy(&notEmpty);
//pthread_mutex_destroy(&mutex);
//pthread_cond_destroy(&notEmpty);
//pthread_mutex_destroy(&mutex);
}
bool fileBlockRequestQueue::empty() const
{
return (queueSize == 0);
return (queueSize == 0);
}
fileRequest* fileBlockRequestQueue::top() const
{
return fbQueue.front();
return fbQueue.front();
}
int fileBlockRequestQueue::push(fileRequest& blk) {
mutex.lock(); //pthread_mutex_lock(&mutex);
fbQueue.push_back(&blk);
int fileBlockRequestQueue::push(fileRequest& blk)
{
mutex.lock(); //pthread_mutex_lock(&mutex);
fbQueue.push_back(&blk);
// @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which
// caused only one i/o thread to be working at a time.
if (++queueSize >= 1 && readersWaiting > 0)
notEmpty.notify_one(); //pthread_cond_signal(&notEmpty);
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return 0;
// @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which
// caused only one i/o thread to be working at a time.
if (++queueSize >= 1 && readersWaiting > 0)
notEmpty.notify_one(); //pthread_cond_signal(&notEmpty);
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return 0;
}
void fileBlockRequestQueue::stop() {
notEmpty.notify_all(); //pthread_cond_broadcast(&notEmpty);
void fileBlockRequestQueue::stop()
{
notEmpty.notify_all(); //pthread_cond_broadcast(&notEmpty);
}
fileRequest* fileBlockRequestQueue::pop(void) {
mutex.lock(); //pthread_mutex_lock(&mutex);
while (queueSize == 0) {
readersWaiting++;
notEmpty.wait(mutex); //pthread_cond_wait(&notEmpty, &mutex);
readersWaiting--;
}
fileRequest* fileBlockRequestQueue::pop(void)
{
mutex.lock(); //pthread_mutex_lock(&mutex);
fileRequest* blk = fbQueue.front();
fbQueue.pop_front();
--queueSize;
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return blk;
while (queueSize == 0)
{
readersWaiting++;
notEmpty.wait(mutex); //pthread_cond_wait(&notEmpty, &mutex);
readersWaiting--;
}
fileRequest* blk = fbQueue.front();
fbQueue.pop_front();
--queueSize;
mutex.unlock(); //pthread_mutex_unlock(&mutex);
return blk;
}
}

101
primitives/blockcache/fileblockrequestqueue.h
View File

@ -25,7 +25,7 @@
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include <deque>
#include <boost/thread.hpp>
@ -41,8 +41,9 @@
/**
* @brief definition of the block request queue as stl std::priority_queue
**/
namespace dbbc {
namespace dbbc
{
typedef std::deque<fileRequest*> fileBlockRequestQueue_t;
/**
@ -50,63 +51,67 @@ typedef std::deque<fileRequest*> fileBlockRequestQueue_t;
**/
class fileBlockRequestQueue {
class fileBlockRequestQueue
{
public:
/**
* @brief default ctor
**/
fileBlockRequestQueue();
/**
* @brief dtor
**/
virtual ~fileBlockRequestQueue();
/**
* @brief default ctor
**/
fileBlockRequestQueue();
/**
* @brief add a request to the queue
**/
int push(fileRequest& blk);
/**
* @brief dtor
**/
virtual ~fileBlockRequestQueue();
/**
* @brief get the next request from the queue and delete it from the queue
**/
fileRequest* pop(void);
/**
* @brief add a request to the queue
**/
int push(fileRequest& blk);
/**
* @brief true if no reuquests are in the queue. false if there are requests in the queue
**/
bool empty() const;
/**
* @brief get the next request from the queue and delete it from the queue
**/
fileRequest* pop(void);
/**
* @brief number of requests in the queue
**/
uint32_t size() const {return queueSize;}
/**
* @brief true if no reuquests are in the queue. false if there are requests in the queue
**/
bool empty() const;
/**
* @brief number of requests in the queue
**/
uint32_t size() const
{
return queueSize;
}
/**
* @brief queue will stop accecpting requests in preparation for the dtor
**/
void stop();
/**
* @brief queue will stop accecpting requests in preparation for the dtor
**/
void stop();
protected:
boost::mutex mutex;
boost::condition notEmpty;
fileBlockRequestQueue_t fbQueue;
uint32_t queueSize;
uint32_t readersWaiting;
boost::mutex mutex;
boost::condition notEmpty;
fileBlockRequestQueue_t fbQueue;
uint32_t queueSize;
uint32_t readersWaiting;
private:
// do not implement
fileBlockRequestQueue(const fileBlockRequestQueue& Q){}
const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q);
// do not implement
fileBlockRequestQueue(const fileBlockRequestQueue& Q) {}
const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q);
/**
* @brief pointer to the next request to be popped from the queue
**/
fileRequest* top() const;
/**
* @brief pointer to the next request to be popped from the queue
**/
fileRequest* top() const;
};
}
#endif

78
primitives/blockcache/filebuffer.cpp
View File

@ -30,65 +30,71 @@
using namespace std;
namespace dbbc {
namespace dbbc
{
FileBuffer::FileBuffer() : fDataLen(0), fLbid(-1), fVerid(0)
{
}
FileBuffer::FileBuffer(const FileBuffer& rhs) {
FileBuffer::FileBuffer(const FileBuffer& rhs)
{
if (this==NULL || this==&rhs)
return;
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
setData(rhs.fByteData, rhs.fDataLen);
fListLoc=rhs.listLoc();
fDataLen=rhs.fDataLen;
if (this == NULL || this == &rhs)
return;
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
setData(rhs.fByteData, rhs.fDataLen);
fListLoc = rhs.listLoc();
fDataLen = rhs.fDataLen;
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len) {
fLbid = lbid;
fVerid = ver;
fDataLen=len;
setData(data, fDataLen);
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len)
{
fLbid = lbid;
fVerid = ver;
fDataLen = len;
setData(data, fDataLen);
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver) {
fLbid=lbid;
fVerid=ver;
fDataLen=0;
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
fLbid = lbid;
fVerid = ver;
fDataLen = 0;
}
FileBuffer& FileBuffer::operator= (const FileBuffer& rhs) {
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
fDataLen=rhs.fDataLen;
setData(rhs.fByteData, fDataLen);
fListLoc=rhs.listLoc();
return *this;
FileBuffer& FileBuffer::operator= (const FileBuffer& rhs)
{
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
fDataLen = rhs.fDataLen;
setData(rhs.fByteData, fDataLen);
fListLoc = rhs.listLoc();
return *this;
}
void FileBuffer::setData(const uint8_t* d, const int len)
{
if (d==NULL || len <=0)
return;
fDataLen=len;
memcpy(fByteData, d, len);
if (d == NULL || len <= 0)
return;
fDataLen = len;
memcpy(fByteData, d, len);
}
void FileBuffer::setData(const uint8_t *d)
void FileBuffer::setData(const uint8_t* d)
{
fDataLen = 8192;
memcpy(fByteData, d, 8192);
fDataLen = 8192;
memcpy(fByteData, d, 8192);
}
FileBuffer::~FileBuffer() {
FileBuffer::~FileBuffer()
{
}
}

181
primitives/blockcache/filebuffer.h
View File

@ -45,104 +45,139 @@
/**
* @brief represents a disk blockrequest
**/
namespace dbbc {
namespace dbbc
{
typedef struct FBData{
BRM::LBID_t lbid;
BRM::VER_t ver;
uint8_t hits;
typedef struct FBData
{
BRM::LBID_t lbid;
BRM::VER_t ver;
uint8_t hits;
} FBData_t;
//@bug 669 Change to list for least recently used cache
//@bug 669 Change to list for least recently used cache
typedef std::list<FBData_t> filebuffer_list_t;
typedef std::list<FBData_t>::iterator filebuffer_list_iter_t;
class FileBuffer {
class FileBuffer
{
public:
FileBuffer();
/**
* @brief copy ctor
**/
FileBuffer(const FileBuffer& fb);
public:
/**
* @brief the disk block from lbid@ver, and a data block len bytes long
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len);
FileBuffer();
/**
* @brief copy ctor
**/
FileBuffer(const FileBuffer& fb);
/**
* @brief disk block lbid@ver and empty data
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief the disk block from lbid@ver, and a data block len bytes long
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len);
/**
* @brief class dtor
**/
/**
* @brief disk block lbid@ver and empty data
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief class dtor
**/
~FileBuffer();
/**
* @brief set the data value of this block to d have len bytestream
**/
void setData(const uint8_t* d, const int len);
void setData(const uint8_t *d); // assumes len = 8192
/**
* @brief set the data value of this block to d have len bytestream
**/
void setData(const uint8_t* d, const int len);
void setData(const uint8_t* d); // assumes len = 8192
/**
* @brief retrieve the data in byte* format from this data block
**/
inline const uint8_t* getData() const {return fByteData;}
inline uint8_t* getData() {return fByteData;}
/**
* @brief retrieve the data in byte* format from this data block
**/
inline const uint8_t* getData() const
{
return fByteData;
}
inline uint8_t* getData()
{
return fByteData;
}
inline const uint32_t datLen() const {return fDataLen;}
inline const uint32_t datLen() const
{
return fDataLen;
}
/**
* @brief assignment operator
**/
FileBuffer& operator= (const FileBuffer& rhs);
/**
* @brief assignment operator
**/
FileBuffer& operator= (const FileBuffer& rhs);
/**
* @brief equality operator is based on lbid@ver
**/
inline bool operator==(const FileBuffer& rhs) const {
return (fLbid == rhs.fLbid && fVerid == rhs.fVerid);
}
/**
* @brief equality operator is based on lbid@ver
**/
inline bool operator==(const FileBuffer& rhs) const
{
return (fLbid == rhs.fLbid && fVerid == rhs.fVerid);
}
/**
* @brief inequality operator
**/
inline bool operator!=(const FileBuffer& rhs) const {
return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid));
}
/**
* @brief inequality operator
**/
inline bool operator!=(const FileBuffer& rhs) const
{
return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid));
}
FileBuffer* thisPtr() {return this;}
/**
* @brief return the lbid value of disk bloc
**/
inline const BRM::LBID_t Lbid() const {return fLbid;}
inline void Lbid(const BRM::LBID_t l) {fLbid=l;}
FileBuffer* thisPtr()
{
return this;
}
/**
* @brief return the lbid value of disk bloc
**/
inline const BRM::LBID_t Lbid() const
{
return fLbid;
}
inline void Lbid(const BRM::LBID_t l)
{
fLbid = l;
}
/**
* @brief return the version of this disk block. ignored for range retrievals
**/
inline const BRM::VER_t Verid() const {return fVerid;}
inline void Verid(BRM::VER_t v) {fVerid=v;}
/**
* @brief return the version of this disk block. ignored for range retrievals
**/
inline const BRM::VER_t Verid() const
{
return fVerid;
}
inline void Verid(BRM::VER_t v)
{
fVerid = v;
}
/**
* @brief return the number of bytes in this disk blockrequest
**/
/**
* @brief return the number of bytes in this disk blockrequest
**/
inline void listLoc(const filebuffer_list_iter_t& loc) {fListLoc = loc;}
inline void listLoc(const filebuffer_list_iter_t& loc)
{
fListLoc = loc;
}
inline const filebuffer_list_iter_t& listLoc() const {return fListLoc;}
inline const filebuffer_list_iter_t& listLoc() const
{
return fListLoc;
}
private:
uint8_t fByteData[BLOCK_SIZE];
uint32_t fDataLen;
BRM::LBID_t fLbid;
BRM::VER_t fVerid;
filebuffer_list_iter_t fListLoc;
uint8_t fByteData[BLOCK_SIZE];
uint32_t fDataLen;
BRM::LBID_t fLbid;
BRM::VER_t fVerid;
filebuffer_list_iter_t fListLoc;
};
typedef std::vector<FileBuffer> FileBufferPool_t;

1069
primitives/blockcache/filebuffermgr.cpp
View File

File diff suppressed because it is too large Load Diff

244
primitives/blockcache/filebuffermgr.h
View File

@ -46,10 +46,11 @@
/**
* @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and list.
*
*
**/
namespace dbbc {
namespace dbbc
{
/**
* @brief used as the hasher algorithm for the unordered_set used to store the disk blocks
@ -57,168 +58,185 @@ namespace dbbc {
struct FileBufferIndex
{
FileBufferIndex(BRM::LBID_t l, BRM::VER_t v, uint32_t p) : lbid(l), ver(v), poolIdx(p) {}
BRM::LBID_t lbid;
BRM::VER_t ver;
uint32_t poolIdx;
FileBufferIndex(BRM::LBID_t l, BRM::VER_t v, uint32_t p) : lbid(l), ver(v), poolIdx(p) {}
BRM::LBID_t lbid;
BRM::VER_t ver;
uint32_t poolIdx;
};
struct CacheInsert_t {
CacheInsert_t(const BRM::LBID_t &l, const BRM::VER_t &v, const uint8_t *d) :
lbid(l), ver(v), data(d) { }
BRM::LBID_t lbid;
BRM::VER_t ver;
const uint8_t *data;
struct CacheInsert_t
{
CacheInsert_t(const BRM::LBID_t& l, const BRM::VER_t& v, const uint8_t* d) :
lbid(l), ver(v), data(d) { }
BRM::LBID_t lbid;
BRM::VER_t ver;
const uint8_t* data;
};
typedef FileBufferIndex HashObject_t;
class bcHasher
{
public:
inline size_t operator()(const HashObject_t& rhs) const
{
return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL));
}
public:
inline size_t operator()(const HashObject_t& rhs) const
{
return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL));
}
};
class bcEqual
{
public:
inline bool operator()(const HashObject_t& f1, const HashObject_t& f2) const
{
return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver));
}
public:
inline bool operator()(const HashObject_t& f1, const HashObject_t& f2) const
{
return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver));
}
};
inline bool operator<(const HashObject_t& f1, const HashObject_t& f2)
{
return ((f1.lbid < f2.lbid) || (f1.lbid == f2.lbid && f1.ver < f2.ver));
return ((f1.lbid < f2.lbid) || (f1.lbid == f2.lbid && f1.ver < f2.ver));
}
class FileBufferMgr {
class FileBufferMgr
{
public:
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual> filebuffer_uset_t;
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual>::const_iterator filebuffer_uset_iter_t;
typedef std::pair<filebuffer_uset_t::iterator, bool> filebuffer_pair_t; // return type for insert
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual> filebuffer_uset_t;
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual>::const_iterator filebuffer_uset_iter_t;
typedef std::pair<filebuffer_uset_t::iterator, bool> filebuffer_pair_t; // return type for insert
typedef std::deque<uint32_t> emptylist_t;
typedef std::deque<uint32_t> emptylist_t;
/**
* @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz
**/
/**
* @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz
**/
FileBufferMgr(uint32_t numBlcks, uint32_t blckSz=BLOCK_SIZE, uint32_t deleteBlocks = 0);
FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0);
/**
* @brief default dtor
**/
/**
* @brief default dtor
**/
virtual ~FileBufferMgr();
/**
* @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const;
/**
* @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const HashObject_t& fb) const;
/**
* @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const;
/**
* @brief add the Disk Block reference by fb into the Disk Block Buffer Cache
**/
int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data);
/**
* @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const HashObject_t& fb) const;
int bulkInsert(const std::vector<CacheInsert_t> &);
/**
* @brief add the Disk Block reference by fb into the Disk Block Buffer Cache
**/
int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data);
/**
* @brief returns the total number of Disk Blocks in the Cache
**/
uint32_t size() const {return fbSet.size();}
int bulkInsert(const std::vector<CacheInsert_t>&);
/**
* @brief
**/
void flushCache();
/**
* @brief returns the total number of Disk Blocks in the Cache
**/
uint32_t size() const
{
return fbSet.size();
}
/**
* @brief
**/
void flushOne(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief
**/
void flushCache();
/**
* @brief
**/
void flushMany(const LbidAtVer* laVptr, uint32_t cnt);
/**
* @brief flush all versions
**/
void flushManyAllversion(const BRM::LBID_t *laVptr, uint32_t cnt);
/**
* @brief
**/
void flushOne(const BRM::LBID_t lbid, const BRM::VER_t ver);
void flushOIDs(const uint32_t *oids, uint32_t count);
void flushPartition(const std::vector<BRM::OID_t> &oids, const std::set<BRM::LogicalPartition> &partitions);
/**
* @brief
**/
void flushMany(const LbidAtVer* laVptr, uint32_t cnt);
/**
* @brief return the disk Block referenced by fb
**/
/**
* @brief flush all versions
**/
void flushManyAllversion(const BRM::LBID_t* laVptr, uint32_t cnt);
FileBuffer* findPtr(const HashObject_t& keyFb);
void flushOIDs(const uint32_t* oids, uint32_t count);
void flushPartition(const std::vector<BRM::OID_t>& oids, const std::set<BRM::LogicalPartition>& partitions);
bool find(const HashObject_t& keyFb, FileBuffer& fb);
/**
* @brief return the disk Block referenced by fb
**/
/**
* @brief return the disk Block referenced by bufferPtr
**/
FileBuffer* findPtr(const HashObject_t& keyFb);
bool find(const HashObject_t& keyFb, void* bufferPtr);
uint32_t bulkFind(const BRM::LBID_t *lbids, const BRM::VER_t *vers, uint8_t **buffers,
bool *wasCached, uint32_t blockCount);
uint32_t maxCacheSize() const {return fMaxNumBlocks;}
bool find(const HashObject_t& keyFb, FileBuffer& fb);
uint32_t listSize() const {return fbList.size();}
/**
* @brief return the disk Block referenced by bufferPtr
**/
const filebuffer_uset_iter_t end() const {return fbSet.end();}
bool find(const HashObject_t& keyFb, void* bufferPtr);
uint32_t bulkFind(const BRM::LBID_t* lbids, const BRM::VER_t* vers, uint8_t** buffers,
bool* wasCached, uint32_t blockCount);
void setReportingFrequency(const uint32_t d);
const uint32_t ReportingFrequency() const {return fReportFrequency;}
uint32_t maxCacheSize() const
{
return fMaxNumBlocks;
}
std::ostream& formatLRUList(std::ostream& os) const;
uint32_t listSize() const
{
return fbList.size();
}
const filebuffer_uset_iter_t end() const
{
return fbSet.end();
}
void setReportingFrequency(const uint32_t d);
const uint32_t ReportingFrequency() const
{
return fReportFrequency;
}
std::ostream& formatLRUList(std::ostream& os) const;
private:
uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list
uint32_t fBlockSz; // size in bytes size of a data block - probably 8
uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list
uint32_t fBlockSz; // size in bytes size of a data block - probably 8
mutable boost::mutex fWLock;
mutable filebuffer_uset_t fbSet;
mutable boost::mutex fWLock;
mutable filebuffer_uset_t fbSet;
mutable filebuffer_list_t fbList; // rename this
uint32_t fCacheSize;
mutable filebuffer_list_t fbList; // rename this
uint32_t fCacheSize;
FileBufferPool_t fFBPool; // vector<FileBuffer>
uint32_t fDeleteBlocks;
emptylist_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused
FileBufferPool_t fFBPool; // vector<FileBuffer>
uint32_t fDeleteBlocks;
emptylist_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused
void depleteCache();
uint64_t fBlksLoaded; // number of blocks inserted into cache
uint64_t fBlksNotUsed; // number of blocks inserted and not used
uint64_t fReportFrequency; // how many blocks are read between reports
std::ofstream fLog;
config::Config* fConfig;
// do not implement
FileBufferMgr(const FileBufferMgr& fbm);
const FileBufferMgr& operator =(const FileBufferMgr& fbm);
// used by bulkInsert
void updateLRU(const FBData_t &f);
uint32_t doBlockCopy(const BRM::LBID_t &lbid, const BRM::VER_t &ver, const uint8_t *data);
void depleteCache();
uint64_t fBlksLoaded; // number of blocks inserted into cache
uint64_t fBlksNotUsed; // number of blocks inserted and not used
uint64_t fReportFrequency; // how many blocks are read between reports
std::ofstream fLog;
config::Config* fConfig;
// do not implement
FileBufferMgr(const FileBufferMgr& fbm);
const FileBufferMgr& operator =(const FileBufferMgr& fbm);
// used by bulkInsert
void updateLRU(const FBData_t& f);
uint32_t doBlockCopy(const BRM::LBID_t& lbid, const BRM::VER_t& ver, const uint8_t* data);
};
}

83
primitives/blockcache/filerequest.cpp
View File

@ -27,68 +27,69 @@ using namespace std;
#include "filerequest.h"
namespace dbbc {
namespace dbbc
{
fileRequest::fileRequest() :
data(0), fLBID(-1), fVer(-1), fFlg(false), fTxn(-1), fRqstType(LBIDREQUEST), fCompType(0),
cache(true), wasVersioned(false)
data(0), fLBID(-1), fVer(-1), fFlg(false), fTxn(-1), fRqstType(LBIDREQUEST), fCompType(0),
cache(true), wasVersioned(false)
{
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
}
fileRequest::fileRequest(BRM::LBID_t lbid, const BRM::QueryContext &ver, bool flg, BRM::VER_t txn, int compType,
uint8_t *ptr, bool cacheIt) :
data(ptr), fLBID(lbid), fVer(ver), fFlg(flg), fTxn(txn), fRqstType(LBIDREQUEST), fCompType(compType),
cache(cacheIt), wasVersioned(false)
fileRequest::fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, int compType,
uint8_t* ptr, bool cacheIt) :
data(ptr), fLBID(lbid), fVer(ver), fFlg(flg), fTxn(txn), fRqstType(LBIDREQUEST), fCompType(compType),
cache(cacheIt), wasVersioned(false)
{
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = 1;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = 1;
}
fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, BRM::VER_t txn, int compType) :
data(0), fLBID(range.start), fVer(ver), fFlg(false), fTxn(txn), fLength(range.size),
fRqstType(RANGEREQUEST), fCompType(compType), cache(true), wasVersioned(false)
fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, int compType) :
data(0), fLBID(range.start), fVer(ver), fFlg(false), fTxn(txn), fLength(range.size),
fRqstType(RANGEREQUEST), fCompType(compType), cache(true), wasVersioned(false)
{
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = range.size;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLength = range.size;
}
fileRequest::fileRequest(const fileRequest& blk)
{
fLBID = blk.fLBID;
fVer = blk.fVer;
fTxn = blk.fTxn;
fFlg = blk.fFlg;
fRqstType = blk.fRqstType;
fRqstStatusString = blk.fRqstStatusString;
data = blk.data;
fCompType = blk.fCompType;
cache = blk.cache;
wasVersioned = blk.wasVersioned;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
fLBID = blk.fLBID;
fVer = blk.fVer;
fTxn = blk.fTxn;
fFlg = blk.fFlg;
fRqstType = blk.fRqstType;
fRqstStatusString = blk.fRqstStatusString;
data = blk.data;
fCompType = blk.fCompType;
cache = blk.cache;
wasVersioned = blk.wasVersioned;
init(); //resets fFRPredicate, fLength, fblksRead, fblksLoaded, fRqstStatus
}
void fileRequest::init()
{
fFRPredicate = INIT;
fLength = 0;
fblksRead = 0;
fblksLoaded = 0;
fRqstStatus = SUCCESSFUL;
fFRPredicate = INIT;
fLength = 0;
fblksRead = 0;
fblksLoaded = 0;
fRqstStatus = SUCCESSFUL;
}
ostream& operator<<(ostream& os, const fileRequest& rhs)
{
os
<< "LBID: " << rhs.fLBID
<< " ver: " << rhs.fVer
<< " Txn: " << rhs.fTxn
<< " len: " << rhs.fLength
<< " read: " << rhs.fblksRead
<< " load: " << rhs.fblksLoaded
<< " ct: " << rhs.fCompType
;
return os;
os
<< "LBID: " << rhs.fLBID
<< " ver: " << rhs.fVer
<< " Txn: " << rhs.fTxn
<< " len: " << rhs.fLength
<< " read: " << rhs.fblksRead
<< " load: " << rhs.fblksLoaded
<< " ct: " << rhs.fCompType
;
return os;
}
}

395
primitives/blockcache/filerequest.h
View File

@ -41,202 +41,285 @@
/**
* @brief request class for the fileblockrequestqueue and the iomanager
**/
namespace dbbc {
namespace dbbc
{
class fileRequest {
class fileRequest
{
public:
/**
* @brief default ctor
**/
fileRequest();
/**
* @brief default ctor
**/
fileRequest();
/**
* @brief copy constructor
**/
fileRequest(const fileRequest& blk);
/**
* @brief copy constructor
**/
fileRequest(const fileRequest& blk);
/**
* @brief request for the disk block lbid@ver
* note, useCache tells IOManager to cache the loaded blocks.
**/
fileRequest(BRM::LBID_t lbid, const BRM::QueryContext &ver, bool flg, BRM::VER_t txn, int compType,
uint8_t *ptr=0, bool useCache = true);
/**
* @brief request for the disk block lbid@ver
* note, useCache tells IOManager to cache the loaded blocks.
**/
fileRequest(BRM::LBID_t lbid, const BRM::QueryContext& ver, bool flg, BRM::VER_t txn, int compType,
uint8_t* ptr = 0, bool useCache = true);
/**
* @brief request a range of disk blocks
**/
fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext &ver, BRM::VER_t txn, int compType);
/**
* @brief request a range of disk blocks
**/
fileRequest(const BRM::InlineLBIDRange& range, const BRM::QueryContext& ver, BRM::VER_t txn, int compType);
/**
* @brief class dtor
**/
/**
* @brief class dtor
**/
virtual ~fileRequest() {};
/**
* @brief less-than operator
**/
bool operator< (const fileRequest & rhs) const {
return fLength < rhs.fLength;
}
/**
* @brief less-than operator
**/
bool operator< (const fileRequest& rhs) const
{
return fLength < rhs.fLength;
}
/**
* @brief greater-than operator
**/
bool operator> (const fileRequest & rhs) const {
return fLength > rhs.fLength;
}
/**
* @brief greater-than operator
**/
bool operator> (const fileRequest& rhs) const
{
return fLength > rhs.fLength;
}
/**
* @brief equality operator
**/
bool operator== (const fileRequest & rhs) const {
return fLength == rhs.fLength;
}
/**
* @brief equality operator
**/
bool operator== (const fileRequest& rhs) const
{
return fLength == rhs.fLength;
}
enum request_status_enum {
SUCCESSFUL,
FAILED,
BRM_LOOKUP_ERROR,
FS_EINVAL,
FS_ENOENT
};
enum request_status_enum
{
SUCCESSFUL,
FAILED,
BRM_LOOKUP_ERROR,
FS_EINVAL,
FS_ENOENT
};
enum request_type_enum {
LBIDREQUEST,
RANGEREQUEST
};
enum request_type_enum
{
LBIDREQUEST,
RANGEREQUEST
};
/**
* @brief used to manage request processing synchronzation
**/
enum predicate_status_enum {
INIT,
SENDING,
READING,
COMPLETE,
STOP
};
/**
* @brief used to manage request processing synchronzation
**/
enum predicate_status_enum
{
INIT,
SENDING,
READING,
COMPLETE,
STOP
};
/**
* @brief lbid requested
**/
const BRM::LBID_t Lbid() const {return fLBID;}
/**
* @brief lbid requested
**/
const BRM::LBID_t Lbid() const
{
return fLBID;
}
/**
* @brief version of the lbid requested
**/
const BRM::QueryContext Ver() const {return fVer;}
/**
* @brief version of the lbid requested
**/
const BRM::QueryContext Ver() const
{
return fVer;
}
/**
* @brief VBBM flag of the LBID/Ver
**/
const bool Flg() const {return fFlg;}
/**
* @brief VBBM flag of the LBID/Ver
**/
const bool Flg() const
{
return fFlg;
}
const BRM::VER_t Txn() const { return fTxn;}
const BRM::VER_t Txn() const
{
return fTxn;
}
const int CompType() const { return fCompType;}
const int CompType() const
{
return fCompType;
}
/**
* @brief number of blocks requested
**/
const uint32_t BlocksRequested() const {return fLength;}
/**
* @brief number of blocks requested
**/
const uint32_t BlocksRequested() const
{
return fLength;
}
/**
* @brief setter for blocks requested
**/
void BlocksRequested(const int l) {fLength=l;}
/**
* @brief setter for blocks requested
**/
void BlocksRequested(const int l)
{
fLength = l;
}
/**
* @brief number of blocks read from disk
**/
const uint32_t BlocksRead() const {return fblksRead;}
const uint32_t BlocksLoaded() const {return fblksLoaded;}
/**
* @brief number of blocks read from disk
**/
const uint32_t BlocksRead() const
{
return fblksRead;
}
const uint32_t BlocksLoaded() const
{
return fblksLoaded;
}
/**
* @brief setter for blocks read from disk
**/
void BlocksRead(const int l) {fblksRead=l;}
void BlocksLoaded(const int l) {fblksLoaded=l;}
/**
* @brief setter for blocks read from disk
**/
void BlocksRead(const int l)
{
fblksRead = l;
}
void BlocksLoaded(const int l)
{
fblksLoaded = l;
}
/**
* @brief did the request succeed for fail (0-success; else failure)
**/
int RequestStatus() const {return fRqstStatus;}
/**
* @brief did the request succeed for fail (0-success; else failure)
**/
int RequestStatus() const
{
return fRqstStatus;
}
/**
* @brief setter for the request status
**/
void RequestStatus(int s) {fRqstStatus=s;}
/**
* @brief setter for the request status
**/
void RequestStatus(int s)
{
fRqstStatus = s;
}
void CompType(int compType) { fCompType = compType; }
void CompType(int compType)
{
fCompType = compType;
}
/**
* @brief if request failed, this is the error message string
**/
std::string RequestStatusStr() const {return fRqstStatusString;}
/**
* @brief if request failed, this is the error message string
**/
std::string RequestStatusStr() const
{
return fRqstStatusString;
}
/**
* @brief setter for the request status error message string
**/
void RequestStatusStr(const std::string& s) {fRqstStatusString=s;}
/**
* @brief setter for the request status error message string
**/
void RequestStatusStr(const std::string& s)
{
fRqstStatusString = s;
}
/**
* @brief return BLOCK or RANGE requested
**/
int RequestType() const {return fRqstType;}
/**
* @brief return BLOCK or RANGE requested
**/
int RequestType() const
{
return fRqstType;
}
/**
* @brief mutex to control synchronzation of request processing
**/
boost::mutex& frMutex() const {return fFRMutex;}
/**
* @brief mutex to control synchronzation of request processing
**/
boost::mutex& frMutex() const
{
return fFRMutex;
}
/**
* @brief condition variable. signal when request is complete
**/
boost::condition& frCond() const {return fFRCond;}
/**
* @brief condition variable. signal when request is complete
**/
boost::condition& frCond() const
{
return fFRCond;
}
/**
* @brief
**/
const enum predicate_status_enum& frPredicate() const {return fFRPredicate;}
/**
* @brief
**/
const enum predicate_status_enum& frPredicate() const
{
return fFRPredicate;
}
/**
* @brief setter for the predicate
**/
void SetPredicate(const enum predicate_status_enum& p) {fFRPredicate=p;}
/**
* @brief setter for the predicate
**/
void SetPredicate(const enum predicate_status_enum& p)
{
fFRPredicate = p;
}
uint8_t *data;
uint8_t* data;
friend std::ostream& operator<<(std::ostream& os, const fileRequest& rhs);
friend std::ostream& operator<<(std::ostream& os, const fileRequest& rhs);
// tells IOManager to cache the loaded blocks or not
bool useCache() { return cache; }
void useCache(bool c) { cache = c; }
bool versioned() { return wasVersioned; }
void versioned(bool b) { wasVersioned = b; }
// tells IOManager to cache the loaded blocks or not
bool useCache()
{
return cache;
}
void useCache(bool c)
{
cache = c;
}
bool versioned()
{
return wasVersioned;
}
void versioned(bool b)
{
wasVersioned = b;
}
private:
void init();
void init();
BRM::LBID_t fLBID;
BRM::QueryContext fVer;
bool fFlg;
BRM::VER_t fTxn;
mutable boost::mutex fFRMutex;
mutable boost::condition fFRCond;
predicate_status_enum fFRPredicate;
uint32_t fLength; // lbids requested
uint32_t fblksRead; // lbids read
uint32_t fblksLoaded; // lbids loaded into cache
int fRqstStatus;
std::string fRqstStatusString;
enum request_type_enum fRqstType;
int fCompType;
bool cache;
bool wasVersioned;
BRM::LBID_t fLBID;
BRM::QueryContext fVer;
bool fFlg;
BRM::VER_t fTxn;
mutable boost::mutex fFRMutex;
mutable boost::condition fFRCond;
predicate_status_enum fFRPredicate;
uint32_t fLength; // lbids requested
uint32_t fblksRead; // lbids read
uint32_t fblksLoaded; // lbids loaded into cache
int fRqstStatus;
std::string fRqstStatusString;
enum request_type_enum fRqstType;
int fCompType;
bool cache;
bool wasVersioned;
};
}

102
primitives/blockcache/fsutils.cpp
View File

@ -25,7 +25,7 @@ using namespace std;
#include <boost/tokenizer.hpp>
#include <boost/filesystem.hpp>
using namespace boost;
namespace fs=boost::filesystem;
namespace fs = boost::filesystem;
#include "fsutils.h"
#include "exceptclasses.h"
@ -35,41 +35,49 @@ namespace
const string resolveInDir(const string& dir, const string& name)
{
idbassert(!dir.empty() && !name.empty());
string ret;
fs::path path(dir);
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
path /= name;
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
idbassert(!dir.empty() && !name.empty());
string ret;
fs::path path(dir);
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
path /= name;
if (!fs::exists(path))
return ret;
idbassert(fs::exists(path));
#ifndef _MSC_VER
if (!fs::is_symlink(path))
return ret;
idbassert(fs::is_symlink(path));
char* realname = (char*)alloca(PATH_MAX+1);
ssize_t realnamelen = readlink(path.string().c_str(), realname, PATH_MAX);
if (realnamelen <= 0)
return ret;
realname[realnamelen] = 0;
fs::path linkname(realname);
fs::path realpath("/dev");
realpath /= linkname.filename();
ret = realpath.string();
if (!fs::is_symlink(path))
return ret;
idbassert(fs::is_symlink(path));
char* realname = (char*)alloca(PATH_MAX + 1);
ssize_t realnamelen = readlink(path.string().c_str(), realname, PATH_MAX);
if (realnamelen <= 0)
return ret;
realname[realnamelen] = 0;
fs::path linkname(realname);
fs::path realpath("/dev");
realpath /= linkname.filename();
ret = realpath.string();
#endif
return ret;
return ret;
}
inline const string label2dev(const string& name)
{
return resolveInDir("/dev/disk/by-label", name);
return resolveInDir("/dev/disk/by-label", name);
}
inline const string uuid2dev(const string& name)
{
return resolveInDir("/dev/disk/by-uuid", name);
return resolveInDir("/dev/disk/by-uuid", name);
}
}
@ -79,33 +87,35 @@ namespace fsutils
const string symname2devname(const string& sympath)
{
string ret;
string ret;
#ifndef _MSC_VER
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
boost::char_separator<char> sep("=");
tokenizer tokens(sympath, sep);
tokenizer::iterator tok_iter = tokens.begin();
typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
boost::char_separator<char> sep("=");
tokenizer tokens(sympath, sep);
tokenizer::iterator tok_iter = tokens.begin();
idbassert(tok_iter != tokens.end());
string symtype = *tok_iter;
if (symtype != "LABEL" && symtype != "UUID")
return ret;
idbassert(tok_iter != tokens.end());
string symtype = *tok_iter;
idbassert(symtype == "LABEL" || symtype == "UUID");
if (symtype != "LABEL" && symtype != "UUID")
return ret;
++tok_iter;
idbassert(tok_iter != tokens.end());
string symname = *tok_iter;
idbassert(symtype == "LABEL" || symtype == "UUID");
++tok_iter;
idbassert(tok_iter == tokens.end());
++tok_iter;
idbassert(tok_iter != tokens.end());
string symname = *tok_iter;
++tok_iter;
idbassert(tok_iter == tokens.end());
if (symtype == "LABEL")
ret = label2dev(symname);
else if (symtype == "UUID")
ret = uuid2dev(symname);
if (symtype == "LABEL")
ret = label2dev(symname);
else if (symtype == "UUID")
ret = uuid2dev(symname);
#endif
return ret;
return ret;
}
}

2172
primitives/blockcache/iomanager.cpp
View File

File diff suppressed because it is too large Load Diff

134
primitives/blockcache/iomanager.h
View File

@ -21,7 +21,7 @@
//
// C++ Interface: iomanager
//
// Description:
// Description:
//
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>
@ -45,25 +45,36 @@
//#define SHARED_NOTHING_DEMO_2
namespace dbbc {
namespace dbbc
{
class ioManager
{
class ioManager {
public:
ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount,
int bsPerRead);
//ioManager(FileBufferMgr& fbm, int thrCount);
~ioManager();
int readerCount() const {return fThreadCount;}
fileRequest* getNextRequest();
void go(void);
void stop();
FileBufferMgr& fileBufferManager() {return fIOMfbMgr;}
config::Config* configPtr() {return fConfig;}
ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount,
int bsPerRead);
//ioManager(FileBufferMgr& fbm, int thrCount);
~ioManager();
int readerCount() const
{
return fThreadCount;
}
fileRequest* getNextRequest();
void go(void);
void stop();
FileBufferMgr& fileBufferManager()
{
return fIOMfbMgr;
}
config::Config* configPtr()
{
return fConfig;
}
const int localLbidLookup(BRM::LBID_t lbid,
BRM::VER_t verid,
BRM::VER_t verid,
bool vbFlag,
BRM::OID_t& oid,
uint16_t& dbRoot,
@ -72,55 +83,76 @@ public:
uint32_t& fileBlockOffset);
void buildOidFileName(const BRM::OID_t oid,
const uint16_t dbRoot,
const uint16_t partNum,
const uint32_t segNum,
char* file_name);
const uint16_t dbRoot,
const uint16_t partNum,
const uint32_t segNum,
char* file_name);
const uint32_t getExtentRows() { return fdbrm.getExtentRows(); }
const uint32_t getExtentRows()
{
return fdbrm.getExtentRows();
}
uint32_t blocksPerRead;
uint32_t blocksPerRead;
bool IOTrace() const { return fIOTrace;}
bool IOTrace() const
{
return fIOTrace;
}
uint32_t MaxOpenFiles() const { return fMaxOpenFiles;}
uint32_t MaxOpenFiles() const
{
return fMaxOpenFiles;
}
uint32_t DecreaseOpenFilesCount() const { return fDecreaseOpenFilesCount;}
uint32_t DecreaseOpenFilesCount() const
{
return fDecreaseOpenFilesCount;
}
bool FDCacheTrace() const { return fFDCacheTrace;}
bool FDCacheTrace() const
{
return fFDCacheTrace;
}
void handleBlockReadError ( fileRequest* fr,
const std::string& errMsg, bool *copyLocked, int errorCode=fileRequest::FAILED );
void handleBlockReadError ( fileRequest* fr,
const std::string& errMsg, bool* copyLocked, int errorCode = fileRequest::FAILED );
std::ofstream& FDTraceFile()
{
return fFDTraceFile;
}
BRM::DBRM* dbrm()
{
return &fdbrm;
}
std::ofstream& FDTraceFile() {return fFDTraceFile;}
BRM::DBRM* dbrm() { return &fdbrm;}
#ifdef SHARED_NOTHING_DEMO_2
uint32_t pmCount;
uint32_t pmCount;
#endif
private:
FileBufferMgr& fIOMfbMgr;
fileBlockRequestQueue& fIOMRequestQueue;
int fThreadCount;
boost::thread_group fThreadArr;
void createReaders();
config::Config* fConfig;
BRM::DBRM fdbrm;
WriteEngine::FileOp fFileOp;
// do not implement
ioManager();
ioManager(const ioManager& iom);
const ioManager& operator=(const ioManager& iom);
bool fIOTrace;
uint32_t fMaxOpenFiles;
uint32_t fDecreaseOpenFilesCount;
bool fFDCacheTrace;
std::ofstream fFDTraceFile;
FileBufferMgr& fIOMfbMgr;
fileBlockRequestQueue& fIOMRequestQueue;
int fThreadCount;
boost::thread_group fThreadArr;
void createReaders();
config::Config* fConfig;
BRM::DBRM fdbrm;
WriteEngine::FileOp fFileOp;
// do not implement
ioManager();
ioManager(const ioManager& iom);
const ioManager& operator=(const ioManager& iom);
bool fIOTrace;
uint32_t fMaxOpenFiles;
uint32_t fDecreaseOpenFilesCount;
bool fFDCacheTrace;
std::ofstream fFDTraceFile;
};

302
primitives/blockcache/stats.cpp
View File

@ -50,118 +50,128 @@ namespace
void pause_(unsigned delay)
{
struct timespec req;
struct timespec rem;
struct timespec req;
struct timespec rem;
req.tv_sec = delay;
req.tv_nsec = 0;
req.tv_sec = delay;
req.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
#ifdef _MSC_VER
Sleep(req.tv_sec * 1000);
Sleep(req.tv_sec * 1000);
#else
again:
if (nanosleep(&req, &rem) != 0)
if (rem.tv_sec > 0 || rem.tv_nsec > 0) {
req = rem;
goto again;
}
if (nanosleep(&req, &rem) != 0)
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
goto again;
}
#endif
}
const string timestr()
{
// Get a timestamp for output.
struct tm tm;
struct timeval tv;
// Get a timestamp for output.
struct tm tm;
struct timeval tv;
gettimeofday(&tv, 0);
localtime_r(reinterpret_cast<time_t*>(&tv.tv_sec), &tm);
gettimeofday(&tv, 0);
localtime_r(reinterpret_cast<time_t*>(&tv.tv_sec), &tm);
ostringstream oss;
oss << setfill('0')
<< setw(2) << tm.tm_hour << ':'
<< setw(2) << tm.tm_min << ':'
<< setw(2) << tm.tm_sec << '.'
<< setw(4) << tv.tv_usec/100;
ostringstream oss;
oss << setfill('0')
<< setw(2) << tm.tm_hour << ':'
<< setw(2) << tm.tm_min << ':'
<< setw(2) << tm.tm_sec << '.'
<< setw(4) << tv.tv_usec / 100;
return oss.str();
return oss.str();
}
class TraceFile
{
public:
TraceFile(uint32_t sessionID, const char* name)
{
if (sessionID > 0 )
{
const char* outName;
if (name == 0)
outName = "lbids";
else
outName = name;
ostringstream oss;
TraceFile(uint32_t sessionID, const char* name)
{
if (sessionID > 0 )
{
const char* outName;
if (name == 0)
outName = "lbids";
else
outName = name;
ostringstream oss;
#ifdef _MSC_VER
oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID;
oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID;
#else
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID;
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID;
#endif
oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
}
}
oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
}
}
~TraceFile()
{
}
~TraceFile()
{
}
void close()
{
if (oFile)
{
oFile->close();
}
}
void close()
{
if (oFile)
{
oFile->close();
}
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event='\0')
{
*oFile << oid << ' ' << timestr() << ' ' << lbid
<< ' ' << thdid;
if (event != '\0')
*oFile << ' ' << event;
*oFile << endl;
oFile->flush();
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
*oFile << oid << ' ' << timestr() << ' ' << lbid
<< ' ' << thdid;
if (event != '\0')
*oFile << ' ' << event;
*oFile << endl;
oFile->flush();
}
private:
//Compiler defaults okay
//TraceFile(const TraceFile& rhs);
//TraceFile operator=(const TraceFile& rhs);
boost::shared_ptr<ofstream> oFile;
//Compiler defaults okay
//TraceFile(const TraceFile& rhs);
//TraceFile operator=(const TraceFile& rhs);
boost::shared_ptr<ofstream> oFile;
};
struct TraceFileInfo
{
TraceFileInfo(uint32_t session=0, const char* name=0) : traceFile(session, name), lastTouched(0) { }
~TraceFileInfo() { }
TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) { }
~TraceFileInfo() { }
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event='\0')
{
traceFile.log(oid, lbid, thdid, event);
lastTouched = time(0);
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
traceFile.log(oid, lbid, thdid, event);
lastTouched = time(0);
}
void close() { traceFile.close(); }
void close()
{
traceFile.close();
}
TraceFile traceFile;
time_t lastTouched;
TraceFile traceFile;
time_t lastTouched;
private:
//Compiler defaults okay
//TraceFileInfo(const TraceFileInfo& rhs);
//TraceFileInfo operator=(const TraceFileInfo& rhs);
//Compiler defaults okay
//TraceFileInfo(const TraceFileInfo& rhs);
//TraceFileInfo operator=(const TraceFileInfo& rhs);
};
//map a session id to a trace file
@ -174,49 +184,51 @@ mutex traceFileMapMutex;
class StatMon
{
public:
StatMon()
{
StatMon()
{
#ifndef _MSC_VER
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
pthread_sigmask(SIG_BLOCK, &sigset, 0);
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
pthread_sigmask(SIG_BLOCK, &sigset, 0);
#endif
}
void operator()() const
{
//struct timespec ts = { 60 * 1, 0 };
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter;
TraceFileMap_t::iterator end;
for (;;)
{
lk.unlock();
time_t beforeSleep = time(0);
//nanosleep(&ts, 0);
pause_(60);
lk.lock();
iter = traceFileMap.begin();
end = traceFileMap.end();
while (iter != end)
{
if (iter->second.lastTouched < beforeSleep)
{
//remove this session trace file
iter->second.close();
traceFileMap.erase(iter++);
}
else
++iter;
}
}
}
}
void operator()() const
{
//struct timespec ts = { 60 * 1, 0 };
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter;
TraceFileMap_t::iterator end;
for (;;)
{
lk.unlock();
time_t beforeSleep = time(0);
//nanosleep(&ts, 0);
pause_(60);
lk.lock();
iter = traceFileMap.begin();
end = traceFileMap.end();
while (iter != end)
{
if (iter->second.lastTouched < beforeSleep)
{
//remove this session trace file
iter->second.close();
traceFileMap.erase(iter++);
}
else
++iter;
}
}
}
private:
//Compiler defaults okay
//StatMon(const StatMon& rhs);
//StatMon operator=(const StatMon& rhs);
//Compiler defaults okay
//StatMon(const StatMon& rhs);
//StatMon operator=(const StatMon& rhs);
};
}
@ -225,52 +237,56 @@ namespace dbbc
{
Stats::Stats() :
fMonitorp(0)
fMonitorp(0)
{
fMonitorp = new boost::thread(StatMon());
fMonitorp = new boost::thread(StatMon());
}
Stats::Stats(const char *name) :
fMonitorp(0), fName(name)
Stats::Stats(const char* name) :
fMonitorp(0), fName(name)
{
fMonitorp = new boost::thread(StatMon());
//fName << name;
fMonitorp = new boost::thread(StatMon());
//fName << name;
}
Stats::~Stats()
{
delete fMonitorp;
delete fMonitorp;
}
void Stats::touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session)
{
if (lbid < 0 || session == 0) return;
if (lbid < 0 || session == 0) return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid);
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid);
}
void Stats::markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event)
{
if (lbid < 0 || session == 0) return;
if (lbid < 0 || session == 0) return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session, fName);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid, event);
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session, fName);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid, event);
}
}

42
primitives/blockcache/stats.h
View File

@ -42,32 +42,32 @@ namespace dbbc
class Stats
{
public:
Stats();
Stats(const char *name);
virtual ~Stats();
Stats();
Stats(const char* name);
virtual ~Stats();
void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session=0);
void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event);
void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0);
void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event);
inline BRM::OID_t lbid2oid(uint64_t lbid)
{
BRM::OID_t oid;
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo);
return oid;
}
inline BRM::OID_t lbid2oid(uint64_t lbid)
{
BRM::OID_t oid;
uint16_t dbroot;
uint32_t partNum;
uint16_t segNum;
uint32_t fbo;
brm.lookupLocal(lbid, 0, false, oid, dbroot, partNum, segNum, fbo);
return oid;
}
private:
Stats(const Stats& rhs);
Stats& operator=(const Stats& rhs);
Stats(const Stats& rhs);
Stats& operator=(const Stats& rhs);
boost::thread* fMonitorp;
BRM::DBRM brm;
//ostringstream fName;
const char* fName;
boost::thread* fMonitorp;
BRM::DBRM brm;
//ostringstream fName;
const char* fName;
};

569
primitives/blockcache/tdriver.cpp
View File

@ -40,316 +40,369 @@ using namespace BRM;
using namespace dbbc;
using namespace std;
Stats* gPMStatsPtr=NULL;
bool gPMProfOn=false;
uint32_t gSession=0;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
int fLoops=1;
int thr_cnt=1;
uint64_t bfoundTot=0;
uint64_t bnfoundTot=0;
uint64_t rfoundTot=0;
uint64_t rnfoundTot=0;
uint64_t rangeOpCountTot=0;
uint64_t blockOpCountTot=0;
uint64_t noOpCountTot=0;
int fLoops = 1;
int thr_cnt = 1;
uint64_t bfoundTot = 0;
uint64_t bnfoundTot = 0;
uint64_t rfoundTot = 0;
uint64_t rnfoundTot = 0;
uint64_t rangeOpCountTot = 0;
uint64_t blockOpCountTot = 0;
uint64_t noOpCountTot = 0;
struct thr_wait_struct {
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
struct thr_wait_struct
{
int predicate;
pthread_mutex_t fMutex;
pthread_cond_t fCond;
vector<LBIDRange_v> range_thr;
};
typedef thr_wait_struct thr_wait_t;
const int32_t cacheSize=175000;
const int32_t cacheSize = 175000;
BlockRequestProcessor BRP(cacheSize, 4, 16);
BRM::VER_t ver=0xFFFF;
u_int64_t totBlocks=0;
BRM::VER_t ver = 0xFFFF;
u_int64_t totBlocks = 0;
void* thr_client(void* clientArgs) {
blockCacheClient bc(BRP);
uint64_t bfound=0;
uint64_t bnfound=0;
uint64_t rfound=0;
uint64_t rnfound=0;
uint64_t rangeOpCount=0;
uint64_t blockOpCount=0;
uint64_t noOpCount=0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate=0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
void* thr_client(void* clientArgs)
{
blockCacheClient bc(BRP);
uint64_t bfound = 0;
uint64_t bnfound = 0;
uint64_t rfound = 0;
uint64_t rnfound = 0;
uint64_t rangeOpCount = 0;
uint64_t blockOpCount = 0;
uint64_t noOpCount = 0;
thr_wait_t* clientWait = (thr_wait_t*)clientArgs;
struct timeval tv, tv2;
uint32_t randstate = 0;
randstate = static_cast<uint32_t>(tv.tv_usec);
pthread_mutex_lock(&clientWait->fMutex);
clientWait->predicate++;
pthread_mutex_unlock(&clientWait->fMutex);
vector<LBIDRange_v>& range_thr = clientWait->range_thr;
gettimeofday(&tv, NULL);
int j=0;
int s=0;
uint8_t fb[8192]={0};
LBIDRange_v& r=range_thr[0];
for(int idx=0; idx<fLoops; idx++) {
for (int jdx=0; jdx<range_thr.size(); jdx++) {
uint32_t lbid=0;
bool b;
int ret=0;
r = range_thr[jdx];
for(int l=0; l<r.size(); l++) {
for(int m=r[l].start;m<r[l].start+r[l].size; m++) {
//ret=bc.getBlock(m, ver, &fb, false, b);
ret=bc.read(m, ver, &fb);
if (ret) {
bfound++;
} else {
bnfound++;
}
}
}
}
}
gettimeofday(&tv, NULL);
int j = 0;
int s = 0;
uint8_t fb[8192] = {0};
LBIDRange_v& r = range_thr[0];
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
for (int idx = 0; idx < fLoops; idx++)
{
for (int jdx = 0; jdx < range_thr.size(); jdx++)
{
uint32_t lbid = 0;
bool b;
int ret = 0;
r = range_thr[jdx];
if (elTime>0) {
avgTot=(bfound+rfound)/elTime;
rangeAvg=(rfound)/elTime;
blkAvg=(bfound)/elTime;
} else {
avgTot=bfound+rfound;
rangeAvg=rfound;
blkAvg=bfound;
}
for (int l = 0; l < r.size(); l++)
{
for (int m = r[l].start; m < r[l].start + r[l].size; m++)
{
//ret=bc.getBlock(m, ver, &fb, false, b);
ret = bc.read(m, ver, &fb);
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
if (ret)
{
bfound++;
}
else
{
bnfound++;
}
}
}
}
}
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot+=bfound;
bnfoundTot+=bnfound;
rfoundTot+=rfound;
rnfoundTot+=rnfound;
rangeOpCountTot+=rangeOpCount;
blockOpCountTot+=blockOpCount;
noOpCountTot+=noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfound + rfound) / elTime;
rangeAvg = (rfound) / elTime;
blkAvg = (bfound) / elTime;
}
else
{
avgTot = bfound + rfound;
rangeAvg = rfound;
blkAvg = bfound;
}
cout << "thr(" << pthread_self() << ") tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCount << " pass " << bfound << " fail " << bnfound <<
" Blks/Sec Blk " << blkAvg << endl << endl;
pthread_mutex_lock(&clientWait->fMutex);
bfoundTot += bfound;
bnfoundTot += bnfound;
rfoundTot += rfound;
rnfoundTot += rnfound;
rangeOpCountTot += rangeOpCount;
blockOpCountTot += blockOpCount;
noOpCountTot += noOpCount;
clientWait->predicate--;
pthread_cond_signal(&clientWait->fCond);
pthread_mutex_unlock(&clientWait->fMutex);
return NULL;
return NULL;
} // end thr_client
void LoadRange(const LBIDRange_v& v, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
blockCacheClient bc(BRP);
uint32_t rCount=0;
for (uint32_t i =0; i<v.size() ; i++)
{
const InlineLBIDRange r={v[i].start, v[i].size};
if (r.size<=1024) {
bc.check(r, ver, rCount );
loadCount+=rCount;
}
rCount=0;
uint32_t rCount = 0;
}
for (uint32_t i = 0; i < v.size() ; i++)
{
const InlineLBIDRange r = {v[i].start, v[i].size};
if (r.size <= 1024)
{
bc.check(r, ver, rCount );
loadCount += rCount;
}
rCount = 0;
}
}
void ReadRange(const LBIDRange_v& v)
{
blockCacheClient bc(BRP);
int found=0;
int notfound=0;
int ret=0;
for(uint32_t i=0; i<v.size(); i++)
{
const InlineLBIDRange r={v[i].start, v[i].size};
FileBuffer fb(-1, -1);
for(int j=r.start; j<r.start+r.size; j++)
{
if (r.size > 1024)
continue;
ret=bc.read(j, ver, fb);
if (ret)
found++;
else
notfound++;
ret=0;
}
totBlocks+=found;
totBlocks+=notfound;
found=0;
notfound=0;
}
blockCacheClient bc(BRP);
int found = 0;
int notfound = 0;
int ret = 0;
for (uint32_t i = 0; i < v.size(); i++)
{
const InlineLBIDRange r = {v[i].start, v[i].size};
FileBuffer fb(-1, -1);
for (int j = r.start; j < r.start + r.size; j++)
{
if (r.size > 1024)
continue;
ret = bc.read(j, ver, fb);
if (ret)
found++;
else
notfound++;
ret = 0;
}
totBlocks += found;
totBlocks += notfound;
found = 0;
notfound = 0;
}
}
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
}
void ReadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
static int found=0, notfound=0;
uint8_t d[8192];
blockCacheClient bc(BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if (ret)
found++;
else
notfound++;
static int found = 0, notfound = 0;
uint8_t d[8192];
blockCacheClient bc(BRP);
//FileBuffer fb(-1, -1);
//bc.read(lbid, ver, fb);
int ret = bc.read(lbid, ver, d);
if ((found+notfound)%10000==0)
cout << "found " << found << " notfound " << notfound << endl;
if (ret)
found++;
else
notfound++;
if ((found + notfound) % 10000 == 0)
cout << "found " << found << " notfound " << notfound << endl;
}
//
int main(int argc, char *argv[]) {
int main(int argc, char* argv[])
{
if (argc>=2) thr_cnt=atoi(argv[1]);
if (argc>=3) fLoops=atoi(argv[2]);
if (thr_cnt<=0) thr_cnt=1;
if (thr_cnt>1024) thr_cnt=1024;
if (fLoops<=0) fLoops=1;
if (argc >= 2) thr_cnt = atoi(argv[1]);
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
if (argc >= 3) fLoops = atoi(argv[2]);
cout << "Starting " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid=3000;
uint32_t totalExt=0;
do {
int ret = dbrm.lookup(oid, r);
if (ret==0 && r.size() > 0) {
lowlbid = (r[0].start/extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start=lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx=0; idx<r.size(); idx++)
totalExt+=r[idx].size;
ranges.push_back(r);
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
if (thr_cnt <= 0) thr_cnt = 1;
cout << ranges.size() << " ranges found" << endl;
if (thr_cnt > 1024) thr_cnt = 1024;
gettimeofday(&tv, NULL);
uint32_t blksLoaded=0;
int rangesLoaded=0;
for (uint32_t i =0; i<ranges.size() && blksLoaded < cacheSize; i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
}
cout << endl;
if (fLoops <= 0) fLoops = 1;
gettimeofday(&tv2, NULL);
cout << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec <<endl;
LBIDRange_v r;
vector<LBIDRange_v> ranges;
DBRM dbrm;
uint32_t hwm, lowfbo, highfbo, fbo, extentSize, lowlbid;
struct timeval tv, tv2;
while (ranges.size() > rangesLoaded) ranges.pop_back();
cout << "Starting " << endl;
extentSize = dbrm.getExtentSize();
BRM::OID_t oid = 3000;
uint32_t totalExt = 0;
do
{
int ret = dbrm.lookup(oid, r);
if (ret == 0 && r.size() > 0)
{
lowlbid = (r[0].start / extentSize) * extentSize;
dbrm.lookup(r[0].start, ver, false, oid, fbo); // need the oid
dbrm.getHWM(oid, hwm);
lowfbo = fbo - (r[0].start - lowlbid);
highfbo = lowfbo + extentSize;
r[0].start = lowlbid;
if (hwm < highfbo)
r[0].size = hwm - lowfbo + 1;
else
r[0].size = extentSize;
for (uint32_t idx = 0; idx < r.size(); idx++)
totalExt += r[idx].size;
ranges.push_back(r);
}
oid++;
}
while ( (r.size() > 0 || oid < 900000) );
cout << ranges.size() << " ranges found" << endl;
gettimeofday(&tv, NULL);
uint32_t blksLoaded = 0;
int rangesLoaded = 0;
for (uint32_t i = 0; i < ranges.size() && blksLoaded < cacheSize; i++)
{
LoadRange(ranges[i], blksLoaded);
rangesLoaded++;
}
cout << endl;
gettimeofday(&tv2, NULL);
cout << "Loaded: " << blksLoaded << " blks " << rangesLoaded << " ranges sec: " << tv2.tv_sec - tv.tv_sec << endl;
while (ranges.size() > rangesLoaded) ranges.pop_back();
#ifdef BLAH
for (uint32_t i =0; i<ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i =0; i<ranges.size(); i++)
{
LBIDRange_v rv=ranges[i];
for(uint32_t j=0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for(uint32_t k=l.start; k<l.start+l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
for (uint32_t i = 0; i < ranges; i++)
ReadRange(ranges[i]);
for (uint32_t i = 0; i < ranges.size(); i++)
{
LBIDRange_v rv = ranges[i];
for (uint32_t j = 0; j < rv.size(); j++)
{
const InlineLBIDRange l = {rv[j].start, rv[j].size};
for (uint32_t k = l.start; k < l.start + l.size; k++)
{
LoadLbid(k, ver);
ReadLbid(k, ver);
}
}
}
#endif
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait={0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
//start threads running
cout << "Starting driver threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt*(sizeof(pthread_t)));
for(int i=0; i<thr_cnt; i++) {
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
pthread_t thr_id[thr_cnt];
thr_wait_t thr_wait = {0, PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, ranges};
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
while (thr_wait.predicate>0) {
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
//start threads running
cout << "Starting driver threads" << endl;
gettimeofday(&tv, NULL);
memset(thr_id, 0, thr_cnt * (sizeof(pthread_t)));
// join threads back to main
for(int i=0; i<thr_cnt; i++) {
pthread_join(thr_id[i], NULL);
}
for (int i = 0; i < thr_cnt; i++)
{
pthread_create(&thr_id[i], NULL, thr_client, &thr_wait);
}
gettimeofday(&tv2, NULL);
time_t tm=time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t)-1]=0;
uint32_t elTime=tv2.tv_sec-tv.tv_sec;
uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot=0;
uint64_t rangeAvg=0;
uint64_t blkAvg=0;
// waiting until all threads have indicated completion
pthread_mutex_lock(&thr_wait.fMutex);
if (elTime>0) {
avgTot=(bfoundTot+rfoundTot)/elTime;
rangeAvg=(rfoundTot)/elTime;
blkAvg=(bfoundTot)/elTime;
} else {
avgTot=bfoundTot+rfoundTot;
rangeAvg=rfoundTot;
blkAvg=bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec-tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot/thr_cnt << endl << endl;
while (thr_wait.predicate > 0)
{
pthread_cond_wait(&thr_wait.fCond, &thr_wait.fMutex);
}
pthread_mutex_unlock(&thr_wait.fMutex);
// join threads back to main
for (int i = 0; i < thr_cnt; i++)
{
pthread_join(thr_id[i], NULL);
}
gettimeofday(&tv2, NULL);
time_t tm = time(0);
char t[50];
ctime_r(&tm, t);
t[strlen(t) - 1] = 0;
uint32_t elTime = tv2.tv_sec - tv.tv_sec;
uint64_t total = bfoundTot + rfoundTot;
uint64_t avgTot = 0;
uint64_t rangeAvg = 0;
uint64_t blkAvg = 0;
if (elTime > 0)
{
avgTot = (bfoundTot + rfoundTot) / elTime;
rangeAvg = (rfoundTot) / elTime;
blkAvg = (bfoundTot) / elTime;
}
else
{
avgTot = bfoundTot + rfoundTot;
rangeAvg = rfoundTot;
blkAvg = bfoundTot;
}
cout << "Summary tm " << t << " " << (tv2.tv_sec - tv.tv_sec) << endl <<
"\tBlk: c " << blockOpCountTot << " pass " << bfoundTot << " fail " << bnfoundTot <<
//"\tRng: c "<< rangeOpCountTot << " pass " << rfoundTot << " fail " << rnfoundTot << endl <<
//"\tNoOp: c " << noOpCountTot << " Total " << total << endl <<
//"\tblks/sec Blk " << blkAvg << " Rng " << rangeAvg << " Tot " << avgTot << " Thr " << avgTot/thr_cnt << endl << endl;
" Blks/Sec Blk " << blkAvg << " Thr " << avgTot / thr_cnt << endl << endl;
return 0;
return 0;
} // end main