database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
utils/testbc/blockcacheclient.cpp
View File

@ -21,67 +21,79 @@
*
* jrodriguez@calpont.com *
***************************************************************************/
#include <sstream>
#include <string>
#include "blockcacheclient.h"
namespace dbbc {
namespace dbbc
{
blockCacheClient::blockCacheClient() {
fBCCBrp=NULL;
blockCacheClient::blockCacheClient()
{
fBCCBrp = NULL;
}
blockCacheClient::blockCacheClient(BlockRequestProcessor& brp) {
fBCCBrp=&brp;
blockCacheClient::blockCacheClient(BlockRequestProcessor& brp)
{
fBCCBrp = &brp;
}
blockCacheClient::~blockCacheClient() {
blockCacheClient::~blockCacheClient()
{
}
void blockCacheClient::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache) {
fBCCBrp->check(lbid, ver, flg, wasBlockInCache);
void blockCacheClient::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache)
{
fBCCBrp->check(lbid, ver, flg, wasBlockInCache);
}
void blockCacheClient::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount) {
fBCCBrp->check(range, ver, rCount);
void blockCacheClient::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount)
{
fBCCBrp->check(range, ver, rCount);
}
FileBuffer* blockCacheClient::getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver) {
FileBuffer* fb = fBCCBrp->getBlockPtr(lbid, ver);
return fb;
FileBuffer* blockCacheClient::getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver)
{
FileBuffer* fb = fBCCBrp->getBlockPtr(lbid, ver);
return fb;
}
const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) {
int ret = fBCCBrp->read(lbid, ver, fb);
return ret;
const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
int ret = fBCCBrp->read(lbid, ver, fb);
return ret;
}
const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) {
int ret = fBCCBrp->read(lbid, ver, bufferPtr);
return ret;
const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
int ret = fBCCBrp->read(lbid, ver, bufferPtr);
return ret;
}
const int blockCacheClient::read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver) {
int ret = fBCCBrp->read(range, fbList, ver);
return ret;
const int blockCacheClient::read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver)
{
int ret = fBCCBrp->read(range, fbList, ver);
return ret;
}
const int blockCacheClient::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, bool &wasCached) {
int ret = fBCCBrp->getBlock(lbid, ver, bufferPtr, flg, wasCached);
return ret;
const int blockCacheClient::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, bool& wasCached)
{
int ret = fBCCBrp->getBlock(lbid, ver, bufferPtr, flg, wasCached);
return ret;
}
bool blockCacheClient::exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
return fBCCBrp->exists(lbid, ver);
return fBCCBrp->exists(lbid, ver);
}
void blockCacheClient::flushCache() {
fBCCBrp->flushCache();
void blockCacheClient::flushCache()
{
fBCCBrp->flushCache();
}
}

106
utils/testbc/blockcacheclient.h
View File

@ -35,73 +35,75 @@
/**
* @brief API for the Disk Block Buffer Cache
*
*
*
*/
namespace dbbc {
class blockCacheClient {
namespace dbbc
{
class blockCacheClient
{
public:
/**
* @brief ctor requires reference to BlockRequestProcessor object
**/
blockCacheClient(BlockRequestProcessor& brp);
/**
* @brief ctor requires reference to BlockRequestProcessor object
**/
blockCacheClient(BlockRequestProcessor& brp);
/**
* @brief dtor
**/
/**
* @brief dtor
**/
virtual ~blockCacheClient();
/**
* @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache.
**/
void check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache);
/**
* @brief verify all Disk Blocks for the LBID range are loaded into the Cache
**/
void check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount);
/**
* @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache.
**/
void check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache);
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb);
/**
* @brief verify all Disk Blocks for the LBID range are loaded into the Cache
**/
void check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount);
FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver);
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr);
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb);
/**
* @brief retrieve all disk Blocks in the LBIDRange range and insert them into fbList
**/
const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver);
const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr,
bool flg, bool &wasCached);
FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver);
bool exists(BRM::LBID_t lbid, BRM::VER_t ver);
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr);
/**
* @brief flush the cache
**/
void flushCache();
/**
* @brief retrieve all disk Blocks in the LBIDRange range and insert them into fbList
**/
const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver);
const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr,
bool flg, bool& wasCached);
bool exists(BRM::LBID_t lbid, BRM::VER_t ver);
/**
* @brief flush the cache
**/
void flushCache();
private:
/**
* @brief pointer to the BlockRequestProcessor object on which the API will operate
**/
BlockRequestProcessor* fBCCBrp;
//do not implement
blockCacheClient();
blockCacheClient(const blockCacheClient& bc);
const blockCacheClient& operator=(const blockCacheClient& blk);
/**
* @brief pointer to the BlockRequestProcessor object on which the API will operate
**/
BlockRequestProcessor* fBCCBrp;
//do not implement
blockCacheClient();
blockCacheClient(const blockCacheClient& bc);
const blockCacheClient& operator=(const blockCacheClient& blk);
};
}

288
utils/testbc/blockrequestprocessor.cpp
View File

@ -34,15 +34,16 @@
#include <boost/date_time/posix_time/posix_time.hpp>
using namespace std;
namespace dbbc {
namespace dbbc
{
BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks,
int thrCount,
int blocksPerRead,
uint32_t deleteBlocks,
uint32_t blckSz) :
fbMgr(numBlcks, blckSz, deleteBlocks),
fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead)
int thrCount,
int blocksPerRead,
uint32_t deleteBlocks,
uint32_t blckSz) :
fbMgr(numBlcks, blckSz, deleteBlocks),
fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead)
{
}
@ -51,170 +52,191 @@ BlockRequestProcessor::~BlockRequestProcessor()
{
}
void BlockRequestProcessor::stop() {
fBRPRequestQueue.stop();
fIOMgr.stop();
void BlockRequestProcessor::stop()
{
fBRPRequestQueue.stop();
fIOMgr.stop();
}
int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount) {
uint64_t maxLbid = range.start; // highest existent lbid
uint64_t rangeLen = range.size;
uint64_t idx;
uint64_t adjSz;
struct timespec start_tm;
int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount)
{
uint64_t maxLbid = range.start; // highest existent lbid
uint64_t rangeLen = range.size;
uint64_t idx;
uint64_t adjSz;
struct timespec start_tm;
if (fTrace) clock_gettime(CLOCK_MONOTONIC, &start_tm);
if (fTrace) clock_gettime(CLOCK_MONOTONIC, &start_tm);
for (idx=0; fbMgr.exists(maxLbid, ver)==true&&idx<rangeLen; maxLbid++, idx++);
for (idx = 0; fbMgr.exists(maxLbid, ver) == true && idx < rangeLen; maxLbid++, idx++);
if (idx==rangeLen) { // range is already loaded
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookup(maxLbid, ver, false, oid, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< 0 << " "
<< 0 << " "
<< 0 << " "
<< right << fixed << ((double)(start_tm.tv_sec+(1.e-9*start_tm.tv_nsec)))
<< endl;
return 0;
}
if (idx == rangeLen) // range is already loaded
{
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookup(maxLbid, ver, false, oid, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< 0 << " "
<< 0 << " "
<< 0 << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
return 0;
}
adjSz=rangeLen-idx;
BRM::InlineLBIDRange adjRange;
adjRange.start=maxLbid;
adjRange.size=adjSz;
fileRequest rqstBlk(adjRange, ver);
check(rqstBlk);
lbidCount=rqstBlk.BlocksRead();
adjSz = rangeLen - idx;
BRM::InlineLBIDRange adjRange;
adjRange.start = maxLbid;
adjRange.size = adjSz;
fileRequest rqstBlk(adjRange, ver);
check(rqstBlk);
lbidCount = rqstBlk.BlocksRead();
if (fTrace) {
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookup(maxLbid, ver, false, oid, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< adjSz << " "
<< rqstBlk.BlocksRead() << " "
<< rqstBlk.BlocksLoaded() << " "
<< right << fixed << ((double)(start_tm.tv_sec+(1.e-9*start_tm.tv_nsec)))
<< endl;
}
if (fTrace)
{
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookup(maxLbid, ver, false, oid, fbo);
fLogFile
<< oid << " "
<< maxLbid << " "
<< fbo << " "
<< rangeLen << " "
<< adjSz << " "
<< rqstBlk.BlocksRead() << " "
<< rqstBlk.BlocksLoaded() << " "
<< right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec)))
<< endl;
}
return rqstBlk.BlocksLoaded();
return rqstBlk.BlocksLoaded();
} // check
int BlockRequestProcessor::check(fileRequest& rqstBlk) {
pthread_mutex_lock(&rqstBlk.frMutex());
rqstBlk.SetPredicate(fileRequest::SENDING);
sendRequest(rqstBlk); // start file read request
int BlockRequestProcessor::check(fileRequest& rqstBlk)
{
pthread_mutex_lock(&rqstBlk.frMutex());
rqstBlk.SetPredicate(fileRequest::SENDING);
sendRequest(rqstBlk); // start file read request
while(rqstBlk.frPredicate()<fileRequest::COMPLETE)
pthread_cond_wait(&rqstBlk.frCond(), &rqstBlk.frMutex());
pthread_mutex_unlock(&rqstBlk.frMutex());
while (rqstBlk.frPredicate() < fileRequest::COMPLETE)
pthread_cond_wait(&rqstBlk.frCond(), &rqstBlk.frMutex());
return 0;
pthread_mutex_unlock(&rqstBlk.frMutex());
return 0;
}
int BlockRequestProcessor::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache) {
if (fbMgr.exists(lbid, ver)==true) {
wasBlockInCache = true;
return 0;
} else {
wasBlockInCache = false;
fileRequest rqstBlk(lbid, ver, flg);
int ret=check(rqstBlk);
return ret;
}
int BlockRequestProcessor::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache)
{
if (fbMgr.exists(lbid, ver) == true)
{
wasBlockInCache = true;
return 0;
}
else
{
wasBlockInCache = false;
fileRequest rqstBlk(lbid, ver, flg);
int ret = check(rqstBlk);
return ret;
}
}
int BlockRequestProcessor::sendRequest(fileRequest& blk)
{
int ret = fBRPRequestQueue.push(blk);
return ret;
int ret = fBRPRequestQueue.push(blk);
return ret;
}
const int BlockRequestProcessor::read(const BRM::InlineLBIDRange& range, FileBufferList_t& readList, const BRM::VER_t ver)
{
int blksLoaded=0;
HashObject_t fb = {0, 0, 0};
for(int idx=0; (uint64_t)idx<range.size; idx++) {
fb.lbid=range.start+idx;
fb.ver=ver;
fb.poolIdx=0;
FileBuffer fbRet(-1, -1);
bool ret = false; //fbMgr.find(fb, fbRet);
if (ret) {
blksLoaded++;
readList.push_back(fbRet);
}
}
return blksLoaded;
int blksLoaded = 0;
HashObject_t fb = {0, 0, 0};
for (int idx = 0; (uint64_t)idx < range.size; idx++)
{
fb.lbid = range.start + idx;
fb.ver = ver;
fb.poolIdx = 0;
FileBuffer fbRet(-1, -1);
bool ret = false; //fbMgr.find(fb, fbRet);
if (ret)
{
blksLoaded++;
readList.push_back(fbRet);
}
}
return blksLoaded;
}
FileBuffer* BlockRequestProcessor::getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver ) {
HashObject_t hashObj = {lbid, ver, 0};
FileBuffer* fb = fbMgr.findPtr(hashObj);
return fb;
}
const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb) {
HashObject_t hashObj = {lbid, ver, 0};
bool ret = fbMgr.find(hashObj, fb);
if (ret==true)
return 1;
else
return 0;
}
const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr) {
HashObject_t hashObj = {lbid, ver, 0};
bool ret = fbMgr.find(hashObj, bufferPtr);
if (ret==true)
return 1;
else
return 0;
}
const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, bool &wasCached)
FileBuffer* BlockRequestProcessor::getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver )
{
HashObject_t hashObj = {lbid, ver, 0};
wasCached = fbMgr.find(hashObj, bufferPtr);
if (wasCached)
return 1;
wasCached = false;
fileRequest rqstBlk(lbid, ver, flg, (uint8_t *) bufferPtr);
check(rqstBlk);
return 1;
HashObject_t hashObj = {lbid, ver, 0};
FileBuffer* fb = fbMgr.findPtr(hashObj);
return fb;
}
const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
HashObject_t hashObj = {lbid, ver, 0};
bool ret = fbMgr.find(hashObj, fb);
if (ret == true)
return 1;
else
return 0;
}
const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
HashObject_t hashObj = {lbid, ver, 0};
bool ret = fbMgr.find(hashObj, bufferPtr);
if (ret == true)
return 1;
else
return 0;
}
const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg, bool& wasCached)
{
HashObject_t hashObj = {lbid, ver, 0};
wasCached = fbMgr.find(hashObj, bufferPtr);
if (wasCached)
return 1;
wasCached = false;
fileRequest rqstBlk(lbid, ver, flg, (uint8_t*) bufferPtr);
check(rqstBlk);
return 1;
}
bool BlockRequestProcessor::exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
HashObject_t ho = {lbid, ver, 0};
HashObject_t ho = {lbid, ver, 0};
return fbMgr.exists(ho);
return fbMgr.exists(ho);
}
void BlockRequestProcessor::flushCache() {
fbMgr.flushCache();
void BlockRequestProcessor::flushCache()
{
fbMgr.flushCache();
}
/**
const uint32_t BlockRequestProcessor::resize(const uint32_t s)
@ -225,7 +247,7 @@ const uint32_t BlockRequestProcessor::resize(const uint32_t s)
**/
ostream& BlockRequestProcessor::formatLRUList(ostream& os) const
{
return fbMgr.formatLRUList(os);
return fbMgr.formatLRUList(os);
}
} // namespace dbbc

144
utils/testbc/blockrequestprocessor.h
View File

@ -36,101 +36,103 @@
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
namespace dbbc {
namespace dbbc
{
typedef std::list<FileBuffer> FileBufferList_t;
/**
* @brief class to control the populating of the Disk Block Buffer Cache and manage Block requests.
**/
class BlockRequestProcessor {
class BlockRequestProcessor
{
public:
/**
* @brief default ctor
**/
BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks=0,
uint32_t blckSz=BLOCK_SIZE);
/**
* @brief default dtor
**/
/**
* @brief default ctor
**/
BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0,
uint32_t blckSz = BLOCK_SIZE);
/**
* @brief default dtor
**/
virtual ~BlockRequestProcessor();
/**
* @brief send a request for disk blocks to the IO manager
**/
int sendRequest(fileRequest& blk);
/**
* @brief send a request for disk blocks to the IO manager
**/
int sendRequest(fileRequest& blk);
/**
* @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not
**/
int check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache);
/**
* @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not
**/
int check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not
**/
int check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache);
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb);
/**
* @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not
**/
int check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr);
/**
* @brief retrieve the LBIDRange of disk blocks from the block cache
**/
const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver);
const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr,
bool flg, bool &wasCached);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver);
bool exists(BRM::LBID_t lbid, BRM::VER_t ver);
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb);
/**
* @brief
**/
void flushCache();
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr);
//const uint32_t resize(const uint32_t s);
/**
* @brief retrieve the LBIDRange of disk blocks from the block cache
**/
const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver);
const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr,
bool flg, bool& wasCached);
bool exists(BRM::LBID_t lbid, BRM::VER_t ver);
/**
* @brief
**/
void flushCache();
//const uint32_t resize(const uint32_t s);
std::ostream& formatLRUList(std::ostream& os) const;
std::ostream& formatLRUList(std::ostream& os) const;
private:
FileBufferMgr fbMgr;
fileBlockRequestQueue fBRPRequestQueue;
ioManager fIOMgr;
pthread_mutex_t check_mutex;
/**
* helper function for public check functions
**/
int check(fileRequest& rqstBlk);
FileBufferMgr fbMgr;
fileBlockRequestQueue fBRPRequestQueue;
ioManager fIOMgr;
pthread_mutex_t check_mutex;
/**
* send stop requests for IOmanager and request Q
**/
void stop();
/**
* helper function for public check functions
**/
int check(fileRequest& rqstBlk);
std::ofstream fLogFile;
bool fTrace;
/**
* send stop requests for IOmanager and request Q
**/
void stop();
BRM::DBRM fdbrm;
std::ofstream fLogFile;
bool fTrace;
BRM::DBRM fdbrm;
// do not implement
BlockRequestProcessor(const BlockRequestProcessor& brp);
BlockRequestProcessor& operator=(const BlockRequestProcessor& brp);
// do not implement
BlockRequestProcessor(const BlockRequestProcessor& brp);
BlockRequestProcessor& operator=(const BlockRequestProcessor& brp);
};
}

69
utils/testbc/fileblockrequestqueue.cpp
View File

@ -28,62 +28,69 @@
using namespace std;
namespace dbbc {
namespace dbbc
{
fileBlockRequestQueue::fileBlockRequestQueue() : queueSize(0), readersWaiting(0)
{
pthread_mutex_init(&mutex, NULL);
pthread_cond_init(&notEmpty, NULL);
pthread_mutex_init(&mutex, NULL);
pthread_cond_init(&notEmpty, NULL);
}
fileBlockRequestQueue::~fileBlockRequestQueue()
{
pthread_cond_destroy(&notEmpty);
pthread_mutex_destroy(&mutex);
pthread_cond_destroy(&notEmpty);
pthread_mutex_destroy(&mutex);
}
bool fileBlockRequestQueue::empty() const
{
return (queueSize == 0);
return (queueSize == 0);
}
fileRequest* fileBlockRequestQueue::top() const
{
return fbQueue.front();
return fbQueue.front();
}
int fileBlockRequestQueue::push(fileRequest& blk) {
pthread_mutex_lock(&mutex);
fbQueue.push_back(&blk);
int fileBlockRequestQueue::push(fileRequest& blk)
{
pthread_mutex_lock(&mutex);
fbQueue.push_back(&blk);
// @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which
// caused only one i/o thread to be working at a time.
if (++queueSize >= 1 && readersWaiting > 0)
pthread_cond_signal(&notEmpty);
pthread_mutex_unlock(&mutex);
return 0;
// @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size was 1 which
// caused only one i/o thread to be working at a time.
if (++queueSize >= 1 && readersWaiting > 0)
pthread_cond_signal(&notEmpty);
pthread_mutex_unlock(&mutex);
return 0;
}
void fileBlockRequestQueue::stop() {
pthread_cond_broadcast(&notEmpty);
void fileBlockRequestQueue::stop()
{
pthread_cond_broadcast(&notEmpty);
}
fileRequest* fileBlockRequestQueue::pop(void) {
pthread_mutex_lock(&mutex);
while (queueSize == 0) {
readersWaiting++;
pthread_cond_wait(&notEmpty, &mutex);
readersWaiting--;
}
fileRequest* fileBlockRequestQueue::pop(void)
{
pthread_mutex_lock(&mutex);
fileRequest* blk = fbQueue.front();
fbQueue.pop_front();
--queueSize;
pthread_mutex_unlock(&mutex);
return blk;
while (queueSize == 0)
{
readersWaiting++;
pthread_cond_wait(&notEmpty, &mutex);
readersWaiting--;
}
fileRequest* blk = fbQueue.front();
fbQueue.pop_front();
--queueSize;
pthread_mutex_unlock(&mutex);
return blk;
}
}

101
utils/testbc/fileblockrequestqueue.h
View File

@ -25,7 +25,7 @@
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include <deque>
#include <pthread.h>
@ -40,8 +40,9 @@
/**
* @brief definition of the block request queue as stl std::priority_queue
**/
namespace dbbc {
namespace dbbc
{
typedef std::deque<fileRequest*> fileBlockRequestQueue_t;
/**
@ -49,63 +50,67 @@ typedef std::deque<fileRequest*> fileBlockRequestQueue_t;
**/
class fileBlockRequestQueue {
class fileBlockRequestQueue
{
public:
/**
* @brief default ctor
**/
fileBlockRequestQueue();
/**
* @brief dtor
**/
virtual ~fileBlockRequestQueue();
/**
* @brief default ctor
**/
fileBlockRequestQueue();
/**
* @brief add a request to the queue
**/
int push(fileRequest& blk);
/**
* @brief dtor
**/
virtual ~fileBlockRequestQueue();
/**
* @brief get the next request from the queue and delete it from the queue
**/
fileRequest* pop(void);
/**
* @brief add a request to the queue
**/
int push(fileRequest& blk);
/**
* @brief true if no reuquests are in the queue. false if there are requests in the queue
**/
bool empty() const;
/**
* @brief get the next request from the queue and delete it from the queue
**/
fileRequest* pop(void);
/**
* @brief number of requests in the queue
**/
uint32_t size() const {return queueSize;}
/**
* @brief true if no reuquests are in the queue. false if there are requests in the queue
**/
bool empty() const;
/**
* @brief number of requests in the queue
**/
uint32_t size() const
{
return queueSize;
}
/**
* @brief queue will stop accecpting requests in preparation for the dtor
**/
void stop();
/**
* @brief queue will stop accecpting requests in preparation for the dtor
**/
void stop();
protected:
pthread_mutex_t mutex;
pthread_cond_t notEmpty;
fileBlockRequestQueue_t fbQueue;
uint32_t queueSize;
uint32_t readersWaiting;
pthread_mutex_t mutex;
pthread_cond_t notEmpty;
fileBlockRequestQueue_t fbQueue;
uint32_t queueSize;
uint32_t readersWaiting;
private:
// do not implement
fileBlockRequestQueue(const fileBlockRequestQueue& Q){}
const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q);
// do not implement
fileBlockRequestQueue(const fileBlockRequestQueue& Q) {}
const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q);
/**
* @brief pointer to the next request to be popped from the queue
**/
fileRequest* top() const;
/**
* @brief pointer to the next request to be popped from the queue
**/
fileRequest* top() const;
};
}
#endif

70
utils/testbc/filebuffer.cpp
View File

@ -30,56 +30,62 @@
using namespace std;
namespace dbbc {
namespace dbbc
{
FileBuffer::FileBuffer(const FileBuffer& rhs) {
FileBuffer::FileBuffer(const FileBuffer& rhs)
{
if (this==NULL || this==&rhs)
return;
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
setData(rhs.fByteData, rhs.fDataLen);
fListLoc=rhs.listLoc();
fDataLen=rhs.fDataLen;
if (this == NULL || this == &rhs)
return;
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
setData(rhs.fByteData, rhs.fDataLen);
fListLoc = rhs.listLoc();
fDataLen = rhs.fDataLen;
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len) {
fLbid = lbid;
fVerid = ver;
fDataLen=len;
setData(data, fDataLen);
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len)
{
fLbid = lbid;
fVerid = ver;
fDataLen = len;
setData(data, fDataLen);
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver) {
fLbid=lbid;
fVerid=ver;
fDataLen=0;
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
fLbid = lbid;
fVerid = ver;
fDataLen = 0;
}
FileBuffer& FileBuffer::operator= (const FileBuffer& rhs) {
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
fDataLen=rhs.fDataLen;
setData(rhs.fByteData, fDataLen);
fListLoc=rhs.listLoc();
return *this;
FileBuffer& FileBuffer::operator= (const FileBuffer& rhs)
{
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
fDataLen = rhs.fDataLen;
setData(rhs.fByteData, fDataLen);
fListLoc = rhs.listLoc();
return *this;
}
void FileBuffer::setData(const uint8_t* d, const int len)
{
if (d==NULL || len <=0)
return;
fDataLen=len;
memcpy(fByteData, d, len);
if (d == NULL || len <= 0)
return;
fDataLen = len;
memcpy(fByteData, d, len);
}
FileBuffer::~FileBuffer() {
FileBuffer::~FileBuffer()
{
}
}

177
utils/testbc/filebuffer.h
View File

@ -45,107 +45,142 @@
/**
* @brief represents a disk blockrequest
**/
namespace dbbc {
namespace dbbc
{
//Set block cache alogorithm to last recently used by defining LRU.
//Otherwise it will be FIFO.
typedef struct FBData{
BRM::LBID_t lbid;
BRM::VER_t ver;
typedef struct FBData
{
BRM::LBID_t lbid;
BRM::VER_t ver;
} FBData_t;
//@bug 669 Change to list for last recently used cache
//@bug 669 Change to list for last recently used cache
typedef std::list<FBData_t> filebuffer_list_t;
typedef std::list<FBData_t>::iterator filebuffer_list_iter_t;
class FileBuffer {
class FileBuffer
{
public:
/**
* @brief copy ctor
**/
FileBuffer(const FileBuffer& fb);
/**
* @brief the disk block from lbid@ver, and a data block len bytes long
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len);
/**
* @brief copy ctor
**/
FileBuffer(const FileBuffer& fb);
/**
* @brief disk block lbid@ver and empty data
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief the disk block from lbid@ver, and a data block len bytes long
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len);
/**
* @brief class dtor
**/
/**
* @brief disk block lbid@ver and empty data
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief class dtor
**/
~FileBuffer();
/**
* @brief set the data value of this block to d have len bytestream
**/
void setData(const uint8_t* d, const int len=8192);
/**
* @brief set the data value of this block to d have len bytestream
**/
void setData(const uint8_t* d, const int len = 8192);
/**
* @brief retrieve the data in byte* format from this data block
**/
const uint8_t* getData() const {return fByteData;}
uint8_t* getData() {return fByteData;}
/**
* @brief retrieve the data in byte* format from this data block
**/
const uint8_t* getData() const
{
return fByteData;
}
uint8_t* getData()
{
return fByteData;
}
const uint32_t datLen() const {return fDataLen;}
const uint32_t datLen() const
{
return fDataLen;
}
/**
* @brief assignment operator
**/
FileBuffer& operator= (const FileBuffer& rhs);
/**
* @brief assignment operator
**/
FileBuffer& operator= (const FileBuffer& rhs);
/**
* @brief equality operator is based on lbid@ver
**/
bool operator==(const FileBuffer& rhs) const {
return (fLbid == rhs.fLbid && fVerid == rhs.fVerid);
}
/**
* @brief equality operator is based on lbid@ver
**/
bool operator==(const FileBuffer& rhs) const
{
return (fLbid == rhs.fLbid && fVerid == rhs.fVerid);
}
/**
* @brief inequality operator
**/
bool operator!=(const FileBuffer& rhs) const {
return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid));
}
/**
* @brief inequality operator
**/
bool operator!=(const FileBuffer& rhs) const
{
return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid));
}
FileBuffer* thisPtr() {return this;}
/**
* @brief return the lbid value of disk bloc
**/
const BRM::LBID_t Lbid() const {return fLbid;}
void Lbid(const BRM::LBID_t l) {fLbid=l;}
FileBuffer* thisPtr()
{
return this;
}
/**
* @brief return the lbid value of disk bloc
**/
const BRM::LBID_t Lbid() const
{
return fLbid;
}
void Lbid(const BRM::LBID_t l)
{
fLbid = l;
}
/**
* @brief return the version of this disk block. ignored for range retrievals
**/
const BRM::VER_t Verid() const {return fVerid;}
void Verid(BRM::VER_t v) {fVerid=v;}
/**
* @brief return the version of this disk block. ignored for range retrievals
**/
const BRM::VER_t Verid() const
{
return fVerid;
}
void Verid(BRM::VER_t v)
{
fVerid = v;
}
/**
* @brief return the number of bytes in this disk blockrequest
**/
/**
* @brief return the number of bytes in this disk blockrequest
**/
void listLoc(const filebuffer_list_iter_t& loc) {fListLoc = loc;}
void listLoc(const filebuffer_list_iter_t& loc)
{
fListLoc = loc;
}
const filebuffer_list_iter_t& listLoc() const {return fListLoc;}
const filebuffer_list_iter_t& listLoc() const
{
return fListLoc;
}
private:
uint8_t fByteData[WriteEngine::BYTE_PER_BLOCK];
uint32_t fDataLen;
uint8_t fByteData[WriteEngine::BYTE_PER_BLOCK];
uint32_t fDataLen;
BRM::LBID_t fLbid;
BRM::VER_t fVerid;
filebuffer_list_iter_t fListLoc;
BRM::LBID_t fLbid;
BRM::VER_t fVerid;
filebuffer_list_iter_t fListLoc;
// do not implement
FileBuffer() {};
// do not implement
FileBuffer() {};
};
typedef std::vector<FileBuffer> FileBufferPool_t;

362
utils/testbc/filebuffermgr.cpp
View File

@ -41,116 +41,131 @@ extern dbbc::Stats* gPMStatsPtr;
extern bool gPMProfOn;
extern uint32_t gSession;
namespace dbbc {
namespace dbbc
{
FileBufferMgr::FileBufferMgr(const uint32_t numBlcks, const uint32_t blkSz, const uint32_t deleteBlocks)
:fMaxNumBlocks(numBlcks),
fBlockSz(blkSz),
fWLock(),
fbSet(),
fbList(),
fCacheSize(0),
fFBPool(),
aging(false),
fDeleteBlocks(deleteBlocks),
fEmptyPoolSlots()
: fMaxNumBlocks(numBlcks),
fBlockSz(blkSz),
fWLock(),
fbSet(),
fbList(),
fCacheSize(0),
fFBPool(),
aging(false),
fDeleteBlocks(deleteBlocks),
fEmptyPoolSlots()
{
fFBPool.reserve(numBlcks);
fEmptyPoolSlots.reserve(deleteBlocks);
fFBPool.reserve(numBlcks);
fEmptyPoolSlots.reserve(deleteBlocks);
}
FileBufferMgr::~FileBufferMgr()
{
flushCache();
flushCache();
}
void FileBufferMgr::flushCache()
{
mutex::scoped_lock lk(fWLock);
fbList.clear();
fbSet.clear();
fFBPool.clear();
fEmptyPoolSlots.clear();
//TODO:: re-init blocks in pool and HWM
mutex::scoped_lock lk(fWLock);
fbList.clear();
fbSet.clear();
fFBPool.clear();
fEmptyPoolSlots.clear();
//TODO:: re-init blocks in pool and HWM
}
bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const {
const HashObject_t fb ={lbid, ver, 0};
const bool b = exists(fb);
return b;
bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const
{
const HashObject_t fb = {lbid, ver, 0};
const bool b = exists(fb);
return b;
} // bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const
FileBuffer* FileBufferMgr::findPtr(const HashObject_t& keyFb) {
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(keyFb);
if (fbSet.end()!=it)
{
FileBuffer* fb=&(fFBPool[it->poolIdx]);
fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() );
return fb;
}
FileBuffer* FileBufferMgr::findPtr(const HashObject_t& keyFb)
{
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(keyFb);
return NULL;
if (fbSet.end() != it)
{
FileBuffer* fb = &(fFBPool[it->poolIdx]);
fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() );
return fb;
}
return NULL;
} // end findPtr(const HashObject_t& keyFB)
bool FileBufferMgr::find(const HashObject_t& keyFb, FileBuffer& fb) {
bool ret = false;
bool FileBufferMgr::find(const HashObject_t& keyFb, FileBuffer& fb)
{
bool ret = false;
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(keyFb);
if (fbSet.end()!=it)
{
fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() );
lk.unlock();
fb = fFBPool[it->poolIdx];
ret = true;
}
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(keyFb);
return ret;
if (fbSet.end() != it)
{
fbList.splice( fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc() );
lk.unlock();
fb = fFBPool[it->poolIdx];
ret = true;
}
return ret;
} // end find(const HashObject_t& keyFB, HashObject_t& fb)
bool FileBufferMgr::find(const HashObject_t& keyFb, void* bufferPtr) {
bool ret = false;
bool FileBufferMgr::find(const HashObject_t& keyFb, void* bufferPtr)
{
bool ret = false;
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L');
mutex::scoped_lock lk(fWLock);
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M');
filebuffer_uset_iter_t it = fbSet.find(keyFb);
if (fbSet.end()!=it)
{
//@bug 669 LRU cache, move block to front of list as last recently used.
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
lk.unlock();
memcpy(bufferPtr, (fFBPool[it->poolIdx]).getData(), 8);
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U');
ret = true;
}
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L');
return ret;
mutex::scoped_lock lk(fWLock);
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M');
filebuffer_uset_iter_t it = fbSet.find(keyFb);
if (fbSet.end() != it)
{
//@bug 669 LRU cache, move block to front of list as last recently used.
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
lk.unlock();
memcpy(bufferPtr, (fFBPool[it->poolIdx]).getData(), 8);
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U');
ret = true;
}
return ret;
} // end find(const FileBuffer& keyFB, void* bufferPtr)
bool FileBufferMgr::exists(const HashObject_t& fb) const {
bool find_bool=false;
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(fb);
if (it != fbSet.end())
{
find_bool = true;
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
}
bool FileBufferMgr::exists(const HashObject_t& fb) const
{
bool find_bool = false;
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(fb);
return find_bool;
if (it != fbSet.end())
{
find_bool = true;
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
}
return find_bool;
}
// default insert operation.
@ -161,126 +176,131 @@ bool FileBufferMgr::exists(const HashObject_t& fb) const {
const int FileBufferMgr::insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data)
{
int ret=0;
int ret = 0;
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I');
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I');
mutex::scoped_lock lk(fWLock);
HashObject_t fbIndex = { lbid, ver, 0};
filebuffer_pair_t pr = fbSet.insert(fbIndex);
if (pr.second) {
// It was inserted (it wasn't there before)
// Right now we have an invalid cache: we have inserted an entry with a -1 index.
// We need to fix this quickly...
fCacheSize++;
FBData_t fbdata = {lbid, ver};
fbList.push_front(fbdata);
}
else {
// if it's a duplicate there's nothing to do
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D');
return ret;
}
mutex::scoped_lock lk(fWLock);
HashObject_t fbIndex = { lbid, ver, 0};
filebuffer_pair_t pr = fbSet.insert(fbIndex);
uint32_t pi = INT_MAX;
if (fCacheSize > maxCacheSize())
{
// If the insert above caused the cache to exceed its max size, find the lru block in
// the cache and use its pool index to store the block data.
FBData_t &fbdata = fbList.back(); //the lru block
HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0};
filebuffer_uset_iter_t iter = fbSet.find( lastFB ); //should be there
if (pr.second)
{
// It was inserted (it wasn't there before)
// Right now we have an invalid cache: we have inserted an entry with a -1 index.
// We need to fix this quickly...
fCacheSize++;
FBData_t fbdata = {lbid, ver};
fbList.push_front(fbdata);
}
else
{
// if it's a duplicate there's nothing to do
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D');
idbassert(iter != fbSet.end());
pi = iter->poolIdx;
idbassert(pi < maxCacheSize());
idbassert(pi < fFBPool.size());
return ret;
}
/* Why does this iterator return a const HashObject_t? */
HashObject_t &ref = const_cast<HashObject_t &>(*pr.first);
ref.poolIdx = pi;
uint32_t pi = INT_MAX;
//replace the lru block with this block
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool[pi] = fb;
fFBPool[pi].setData(data, BLOCK_SIZE);
fbSet.erase(iter);
fbList.pop_back();
if (fCacheSize > maxCacheSize())
{
// If the insert above caused the cache to exceed its max size, find the lru block in
// the cache and use its pool index to store the block data.
FBData_t& fbdata = fbList.back(); //the lru block
HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0};
filebuffer_uset_iter_t iter = fbSet.find( lastFB ); //should be there
fCacheSize--;
depleteCache();
ret=1;
}
else
{
if ( ! fEmptyPoolSlots.empty() )
{
pi = fEmptyPoolSlots.back();
fEmptyPoolSlots.pop_back();
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool[pi] = fb;
fFBPool[pi].setData(data, 8);
}
else
{
pi = fFBPool.size();
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool.push_back(fb);
fFBPool[pi].setData(data, 8);
}
idbassert(iter != fbSet.end());
pi = iter->poolIdx;
idbassert(pi < maxCacheSize());
idbassert(pi < fFBPool.size());
/* Why does this iterator return a const? */
HashObject_t &ref = const_cast<HashObject_t &>(*pr.first);
ref.poolIdx = pi;
ret=1;
}
/* Why does this iterator return a const HashObject_t? */
HashObject_t& ref = const_cast<HashObject_t&>(*pr.first);
ref.poolIdx = pi;
idbassert(pi < fFBPool.size());
fFBPool[pi].listLoc(fbList.begin());
//replace the lru block with this block
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool[pi] = fb;
fFBPool[pi].setData(data, BLOCK_SIZE);
fbSet.erase(iter);
fbList.pop_back();
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J');
fCacheSize--;
depleteCache();
ret = 1;
}
else
{
if ( ! fEmptyPoolSlots.empty() )
{
pi = fEmptyPoolSlots.back();
fEmptyPoolSlots.pop_back();
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool[pi] = fb;
fFBPool[pi].setData(data, 8);
}
else
{
pi = fFBPool.size();
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool.push_back(fb);
fFBPool[pi].setData(data, 8);
}
idbassert(fCacheSize <= maxCacheSize());
/* Why does this iterator return a const? */
HashObject_t& ref = const_cast<HashObject_t&>(*pr.first);
ref.poolIdx = pi;
ret = 1;
}
idbassert(pi < fFBPool.size());
fFBPool[pi].listLoc(fbList.begin());
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J');
idbassert(fCacheSize <= maxCacheSize());
// idbassert(fCacheSize == fbSet.size());
// idbassert(fCacheSize == fbList.size());
return ret;
return ret;
}
void FileBufferMgr::depleteCache()
void FileBufferMgr::depleteCache()
{
for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i)
{
FBData_t fbdata(fbList.back()); //the lru block
HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0};
filebuffer_uset_iter_t iter = fbSet.find( lastFB );
for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i)
{
FBData_t fbdata(fbList.back()); //the lru block
HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0};
filebuffer_uset_iter_t iter = fbSet.find( lastFB );
idbassert(iter != fbSet.end());
uint32_t idx = iter->poolIdx;
idbassert(idx < fFBPool.size());
//Save position in FileBuffer pool for reuse.
fEmptyPoolSlots.push_back(idx);
fbSet.erase(iter);
fbList.pop_back();
fCacheSize--;
}
idbassert(iter != fbSet.end());
uint32_t idx = iter->poolIdx;
idbassert(idx < fFBPool.size());
//Save position in FileBuffer pool for reuse.
fEmptyPoolSlots.push_back(idx);
fbSet.erase(iter);
fbList.pop_back();
fCacheSize--;
}
}
ostream& FileBufferMgr::formatLRUList(ostream& os) const
{
filebuffer_list_t::const_iterator iter=fbList.begin();
filebuffer_list_t::const_iterator end=fbList.end();
filebuffer_list_t::const_iterator iter = fbList.begin();
filebuffer_list_t::const_iterator end = fbList.end();
while (iter != end)
{
os << iter->lbid << '\t' << iter->ver << endl;
++iter;
}
while (iter != end)
{
os << iter->lbid << '\t' << iter->ver << endl;
++iter;
}
return os;
return os;
}
}

191
utils/testbc/filebuffermgr.h
View File

@ -37,150 +37,165 @@
/**
* @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and list.
*
*
**/
namespace dbbc {
namespace dbbc
{
/**
* @brief used as the hasher algorithm for the unordered_set used to store the disk blocks
**/
typedef struct {
BRM::LBID_t lbid;
BRM::VER_t ver;
uint32_t poolIdx;
} FileBufferIndex_t;
typedef struct
{
BRM::LBID_t lbid;
BRM::VER_t ver;
uint32_t poolIdx;
} FileBufferIndex_t;
typedef FileBufferIndex_t HashObject_t;
class bcHasher
{
public:
size_t operator()(const HashObject_t& rhs) const
{
return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL));
}
public:
size_t operator()(const HashObject_t& rhs) const
{
return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL));
}
};
class bcEqual
{
public:
size_t operator()(const HashObject_t& f1, const HashObject_t& f2) const
{
return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver));
}
public:
size_t operator()(const HashObject_t& f1, const HashObject_t& f2) const
{
return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver));
}
};
inline bool operator<(const HashObject_t& f1, const HashObject_t& f2)
{
//return ((f1.lbid < f2.lbid) || (f1.ver < f2.ver));
//return ((f1.lbid < f2.lbid) || (f1.ver < f2.ver));
#if 1
if (f1.lbid < f2.lbid)
return true;
else if (f1.lbid == f2.lbid)
return (f1.ver < f2.ver);
if (f1.lbid < f2.lbid)
return true;
else if (f1.lbid == f2.lbid)
return (f1.ver < f2.ver);
return false;
return false;
#else
bcHasher bh1, bh2;
return (bh1(f1) < bh2(f2));
bcHasher bh1, bh2;
return (bh1(f1) < bh2(f2));
#endif
}
class FileBufferMgr {
class FileBufferMgr
{
public:
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual> filebuffer_uset_t;
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual>::const_iterator filebuffer_uset_iter_t;
typedef std::pair<filebuffer_uset_t::iterator, bool> filebuffer_pair_t; // return type for insert
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual> filebuffer_uset_t;
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual>::const_iterator filebuffer_uset_iter_t;
typedef std::pair<filebuffer_uset_t::iterator, bool> filebuffer_pair_t; // return type for insert
typedef std::vector<uint32_t> intvec_t;
typedef std::vector<uint32_t> intvec_t;
/**
* @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz
**/
/**
* @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz
**/
FileBufferMgr(uint32_t numBlcks, uint32_t blckSz=BLOCK_SIZE, uint32_t deleteBlocks = 0);
FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0);
/**
* @brief default dtor
**/
/**
* @brief default dtor
**/
virtual ~FileBufferMgr();
/**
* @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const;
/**
* @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const HashObject_t& fb) const;
/**
* @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const;
/**
* @brief add the Disk Block reference by fb into the Disk Block Buffer Cache
**/
const int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data);
/**
* @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache otherwise return FALSE.
**/
bool exists(const HashObject_t& fb) const;
/**
* @brief returns the total number of Disk Blocks in the Cache
**/
uint32_t size() const {return fbSet.size();}
/**
* @brief add the Disk Block reference by fb into the Disk Block Buffer Cache
**/
const int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data);
/**
* @brief
**/
void flushCache();
/**
* @brief returns the total number of Disk Blocks in the Cache
**/
uint32_t size() const
{
return fbSet.size();
}
/**
* @brief return the disk Block referenced by fb
**/
/**
* @brief
**/
void flushCache();
FileBuffer* findPtr(const HashObject_t& keyFb);
/**
* @brief return the disk Block referenced by fb
**/
bool find(const HashObject_t& keyFb, FileBuffer& fb);
FileBuffer* findPtr(const HashObject_t& keyFb);
/**
* @brief return the disk Block referenced by bufferPtr
**/
bool find(const HashObject_t& keyFb, FileBuffer& fb);
bool find(const HashObject_t& keyFb, void* bufferPtr);
uint32_t maxCacheSize() const {return fMaxNumBlocks;}
/**
* @brief return the disk Block referenced by bufferPtr
**/
uint32_t listSize() const {return fbList.size();}
bool find(const HashObject_t& keyFb, void* bufferPtr);
const filebuffer_uset_iter_t end() const {return fbSet.end();}
uint32_t maxCacheSize() const
{
return fMaxNumBlocks;
}
void displayCounts() const;
uint32_t listSize() const
{
return fbList.size();
}
std::ostream& formatLRUList(std::ostream& os) const;
const filebuffer_uset_iter_t end() const
{
return fbSet.end();
}
void displayCounts() const;
std::ostream& formatLRUList(std::ostream& os) const;
private:
uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list
uint32_t fBlockSz; // size in bytes size of a data block - probably 8
uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list
uint32_t fBlockSz; // size in bytes size of a data block - probably 8
mutable boost::mutex fWLock;
mutable filebuffer_uset_t fbSet;
mutable boost::mutex fWLock;
mutable filebuffer_uset_t fbSet;
mutable filebuffer_list_t fbList; // rename this
uint32_t fCacheSize;
mutable filebuffer_list_t fbList; // rename this
uint32_t fCacheSize;
FileBufferPool_t fFBPool; // vector<FileBuffer>
// do not implement
FileBufferMgr(const FileBufferMgr& fbm);
const FileBufferMgr& operator =(const FileBufferMgr& fbm);
bool aging;
FileBufferPool_t fFBPool; // vector<FileBuffer>
uint32_t fDeleteBlocks;
intvec_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused
// do not implement
FileBufferMgr(const FileBufferMgr& fbm);
const FileBufferMgr& operator =(const FileBufferMgr& fbm);
bool aging;
void depleteCache();
uint32_t fDeleteBlocks;
intvec_t fEmptyPoolSlots; //keep track of FBPool slots that can be reused
void depleteCache();
};
}
#endif

84
utils/testbc/filerequest.cpp
View File

@ -30,75 +30,77 @@
using namespace std;
namespace dbbc {
namespace dbbc
{
fileRequest::fileRequest()
{
fLBID=-1;
fVer=-1;
data = NULL;
init();
fRqstType = LBIDREQUEST;
fLBID = -1;
fVer = -1;
data = NULL;
init();
fRqstType = LBIDREQUEST;
}
fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg) :
data(NULL), fLBID(lbid), fVer(ver), fFlg(flg)
data(NULL), fLBID(lbid), fVer(ver), fFlg(flg)
{
init();
fLength=1;
fRqstType = LBIDREQUEST;
init();
fLength = 1;
fRqstType = LBIDREQUEST;
}
fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t *ptr) :
fLBID(lbid), fVer(ver), fFlg(flg)
fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr) :
fLBID(lbid), fVer(ver), fFlg(flg)
{
init();
fLength=1;
fRqstType = LBIDREQUEST;
data = ptr;
init();
fLength = 1;
fRqstType = LBIDREQUEST;
data = ptr;
}
fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver) :
data(NULL), fLBID(range.start), fVer(ver), fFlg(false), fLength(range.size),
fRqstType(RANGEREQUEST)
data(NULL), fLBID(range.start), fVer(ver), fFlg(false), fLength(range.size),
fRqstType(RANGEREQUEST)
{
init();
fLength=range.size;
init();
fLength = range.size;
}
fileRequest::fileRequest(const fileRequest& blk)
{
fLBID=blk.fLBID;
fVer=blk.fVer;
fLength=blk.fLength;
fblksRead=blk.fblksRead;
fRqstType = blk.fRqstType;
fRqstStatus=blk.fRqstStatus;
data = blk.data;
init();
fLBID = blk.fLBID;
fVer = blk.fVer;
fLength = blk.fLength;
fblksRead = blk.fblksRead;
fRqstType = blk.fRqstType;
fRqstStatus = blk.fRqstStatus;
data = blk.data;
init();
}
void fileRequest::init() {
if (pthread_mutex_init(&fFRMutex, NULL)!=0)
throw runtime_error("mutex initialization failure");
if (pthread_cond_init(&fFRCond, NULL)!=0)
throw runtime_error("cond var initialization failure");
fFRPredicate=INIT;
fLength=0;
fblksRead=0;
fRqstStatus=0;
void fileRequest::init()
{
if (pthread_mutex_init(&fFRMutex, NULL) != 0)
throw runtime_error("mutex initialization failure");
if (pthread_cond_init(&fFRCond, NULL) != 0)
throw runtime_error("cond var initialization failure");
fFRPredicate = INIT;
fLength = 0;
fblksRead = 0;
fRqstStatus = 0;
}
fileRequest::~fileRequest()
{
pthread_mutex_destroy(&fFRMutex);
pthread_cond_destroy(&fFRCond);
pthread_mutex_destroy(&fFRMutex);
pthread_cond_destroy(&fFRCond);
}
}

326
utils/testbc/filerequest.h
View File

@ -36,168 +36,224 @@
/**
* @brief request class for the fileblockrequestqueue and the iomanager
**/
namespace dbbc {
namespace dbbc
{
class fileRequest {
class fileRequest
{
public:
/**
* @brief default ctor
**/
fileRequest();
/**
* @brief copy constructor
**/
fileRequest(const fileRequest& blk);
/**
* @brief request for the disk block lbid@ver
**/
fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg);
/**
* @brief default ctor
**/
fileRequest();
fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t *ptr);
/**
* @brief request a range of disk blocks
**/
fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver);
/**
* @brief copy constructor
**/
fileRequest(const fileRequest& blk);
/**
* @brief class dtor
**/
/**
* @brief request for the disk block lbid@ver
**/
fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg);
fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr);
/**
* @brief request a range of disk blocks
**/
fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver);
/**
* @brief class dtor
**/
virtual ~fileRequest();
/**
* @brief less-than operator
**/
bool operator< (const fileRequest & rhs) const {
return fLength < rhs.fLength;
}
/**
* @brief less-than operator
**/
bool operator< (const fileRequest& rhs) const
{
return fLength < rhs.fLength;
}
/**
* @brief greater-than operator
**/
bool operator> (const fileRequest & rhs) const {
return fLength > rhs.fLength;
}
/**
* @brief greater-than operator
**/
bool operator> (const fileRequest& rhs) const
{
return fLength > rhs.fLength;
}
/**
* @brief equality operator
**/
bool operator== (const fileRequest & rhs) const {
return fLength == rhs.fLength;
}
enum request_status_enum {
SUCCESSFUL,
FAILED
};
/**
* @brief equality operator
**/
bool operator== (const fileRequest& rhs) const
{
return fLength == rhs.fLength;
}
enum request_type_enum {
LBIDREQUEST,
RANGEREQUEST
};
enum request_status_enum
{
SUCCESSFUL,
FAILED
};
/**
* @brief used to manage request processing synchronzation
**/
enum predicate_status_enum {
INIT,
SENDING,
READING,
COMPLETE,
STOP
};
enum request_type_enum
{
LBIDREQUEST,
RANGEREQUEST
};
/**
* @brief lbid requested
**/
const BRM::LBID_t Lbid() const {return fLBID;}
/**
* @brief used to manage request processing synchronzation
**/
enum predicate_status_enum
{
INIT,
SENDING,
READING,
COMPLETE,
STOP
};
/**
* @brief version of the lbid requested
**/
const BRM::VER_t Ver() const {return fVer;}
/**
* @brief VBBM flag of the LBID/Ver
**/
const bool Flg() const {return fFlg;}
/**
* @brief lbid requested
**/
const BRM::LBID_t Lbid() const
{
return fLBID;
}
/**
* @brief number of blocks requested
**/
const uint32_t BlocksRequested() const {return fLength;}
/**
* @brief version of the lbid requested
**/
const BRM::VER_t Ver() const
{
return fVer;
}
/**
* @brief setter for blocks requested
**/
void BlocksRequested(const int l) {fLength=l;}
/**
* @brief VBBM flag of the LBID/Ver
**/
const bool Flg() const
{
return fFlg;
}
/**
* @brief number of blocks read from disk
**/
const uint32_t BlocksRead() const {return fblksRead;}
const uint32_t BlocksLoaded() const {return fblksLoaded;}
/**
* @brief number of blocks requested
**/
const uint32_t BlocksRequested() const
{
return fLength;
}
/**
* @brief setter for blocks read from disk
**/
void BlocksRead(const int l) {fblksRead=l;}
void BlocksLoaded(const int l) {fblksLoaded=l;}
/**
* @brief setter for blocks requested
**/
void BlocksRequested(const int l)
{
fLength = l;
}
/**
* @brief did the request succeed for fail
**/
int RequestStatus() const {return fRqstStatus;}
/**
* @brief number of blocks read from disk
**/
const uint32_t BlocksRead() const
{
return fblksRead;
}
const uint32_t BlocksLoaded() const
{
return fblksLoaded;
}
/**
* @brief setter for the request status
**/
void RequestStatus(int s) {fRqstStatus=s;}
/**
* @brief setter for blocks read from disk
**/
void BlocksRead(const int l)
{
fblksRead = l;
}
void BlocksLoaded(const int l)
{
fblksLoaded = l;
}
/**
* @brief return BLOCK or RANGE requested
**/
int RequestType() const {return fRqstType;}
/**
* @brief did the request succeed for fail
**/
int RequestStatus() const
{
return fRqstStatus;
}
/**
* @brief mutex to control synchronzation of request processing
**/
pthread_mutex_t& frMutex() const {return fFRMutex;}
/**
* @brief setter for the request status
**/
void RequestStatus(int s)
{
fRqstStatus = s;
}
/**
* @brief condition variable. signal when request is complete
**/
pthread_cond_t& frCond() const {return fFRCond;}
/**
* @brief return BLOCK or RANGE requested
**/
int RequestType() const
{
return fRqstType;
}
/**
* @brief
**/
const enum predicate_status_enum& frPredicate() const {return fFRPredicate;}
/**
* @brief mutex to control synchronzation of request processing
**/
pthread_mutex_t& frMutex() const
{
return fFRMutex;
}
/**
* @brief setter for the predicate
**/
void SetPredicate(const enum predicate_status_enum& p) {fFRPredicate=p;}
uint8_t *data;
/**
* @brief condition variable. signal when request is complete
**/
pthread_cond_t& frCond() const
{
return fFRCond;
}
/**
* @brief
**/
const enum predicate_status_enum& frPredicate() const
{
return fFRPredicate;
}
/**
* @brief setter for the predicate
**/
void SetPredicate(const enum predicate_status_enum& p)
{
fFRPredicate = p;
}
uint8_t* data;
private:
void init();
BRM::LBID_t fLBID;
BRM::VER_t fVer;
bool fFlg;
mutable pthread_mutex_t fFRMutex;
mutable pthread_cond_t fFRCond;
predicate_status_enum fFRPredicate;
uint32_t fLength; // lbids requested
uint32_t fblksRead; // lbids read
uint32_t fblksLoaded; // lbids loaded into cache
int fRqstStatus;
enum request_type_enum fRqstType;
void init();
BRM::LBID_t fLBID;
BRM::VER_t fVer;
bool fFlg;
mutable pthread_mutex_t fFRMutex;
mutable pthread_cond_t fFRCond;
predicate_status_enum fFRPredicate;
uint32_t fLength; // lbids requested
uint32_t fblksRead; // lbids read
uint32_t fblksLoaded; // lbids loaded into cache
int fRqstStatus;
enum request_type_enum fRqstType;
};
}
#endif

416
utils/testbc/iomanager.cpp
View File

@ -19,7 +19,7 @@
//
// C++ Implementation: iomanager
//
// Description:
// Description:
//
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>
@ -51,258 +51,298 @@ using namespace std;
using namespace config;
#include "iomanager.h"
namespace {
namespace
{
using namespace dbbc;
using namespace std;
void timespec_sub(const struct timespec &tv1,
const struct timespec &tv2,
double &tm)
void timespec_sub(const struct timespec& tv1,
const struct timespec& tv2,
double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9*(tv2.tv_nsec - tv1.tv_nsec);
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
struct IOMThreadArg {
ioManager* iom;
int32_t thdId;
struct IOMThreadArg
{
ioManager* iom;
int32_t thdId;
};
typedef IOMThreadArg IOMThreadArg_t;
void* thr_popper(void* arg) {
ioManager* iom = ((IOMThreadArg*)arg)->iom;
int32_t iomThdId = ((IOMThreadArg*)arg)->thdId;
FileBufferMgr* fbm;
int totalRqst=0;
fileRequest* fr=0;
BRM::LBID_t lbid=0;
BRM::OID_t oid=0;
BRM::VER_t ver=0;
int blocksLoaded=0;
int blocksRead=0;
const unsigned pageSize = 4096;
fbm = &iom->fileBufferManager();
char fileName[WriteEngine::FILE_NAME_SIZE];
const uint64_t fileBlockSize = BLOCK_SIZE;
uint32_t offset=0;
bool flg=false;
char* fileNamePtr=fileName;
uint64_t longSeekOffset=0;
int err;
uint32_t dlen = 0, acc, readSize, blocksThisRead, j;
uint32_t blocksRequested=0;
ssize_t i;
uint32_t sz=0;
char* alignedbuff=0;
boost::scoped_array<char> realbuff;
pthread_t threadId=0;
ostringstream iomLogFileName;
ofstream lFile;
void* thr_popper(void* arg)
{
ioManager* iom = ((IOMThreadArg*)arg)->iom;
int32_t iomThdId = ((IOMThreadArg*)arg)->thdId;
FileBufferMgr* fbm;
int totalRqst = 0;
fileRequest* fr = 0;
BRM::LBID_t lbid = 0;
BRM::OID_t oid = 0;
BRM::VER_t ver = 0;
int blocksLoaded = 0;
int blocksRead = 0;
const unsigned pageSize = 4096;
fbm = &iom->fileBufferManager();
char fileName[WriteEngine::FILE_NAME_SIZE];
const uint64_t fileBlockSize = BLOCK_SIZE;
uint32_t offset = 0;
bool flg = false;
char* fileNamePtr = fileName;
uint64_t longSeekOffset = 0;
int err;
uint32_t dlen = 0, acc, readSize, blocksThisRead, j;
uint32_t blocksRequested = 0;
ssize_t i;
uint32_t sz = 0;
char* alignedbuff = 0;
boost::scoped_array<char> realbuff;
pthread_t threadId = 0;
ostringstream iomLogFileName;
ofstream lFile;
threadId=pthread_self();
threadId = pthread_self();
uint32_t readBufferSz=iom->blocksPerRead * BLOCK_SIZE+pageSize;
uint32_t readBufferSz = iom->blocksPerRead * BLOCK_SIZE + pageSize;
realbuff.reset(new char[readBufferSz]);
if (realbuff.get() == 0) {
cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
return 0;
}
realbuff.reset(new char[readBufferSz]);
if (realbuff.get() == 0)
{
cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
return 0;
}
#if __WORDSIZE > 32
//alignedbuff=(char*)((((ptrdiff_t)&realbuff[0] + pageSize - 1) / pageSize) * pageSize);
// pagesize == (1 << 12)
alignedbuff=(char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize);
//alignedbuff=(char*)((((ptrdiff_t)&realbuff[0] + pageSize - 1) / pageSize) * pageSize);
// pagesize == (1 << 12)
alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize);
#else
//alignedbuff=(char*)(((((ptrdiff_t)&realbuff[0] & 0xffffffff) + pageSize - 1) / pageSize) * pageSize);
alignedbuff=(char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize);
//alignedbuff=(char*)(((((ptrdiff_t)&realbuff[0] & 0xffffffff) + pageSize - 1) / pageSize) * pageSize);
alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize);
#endif
idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize);
idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0);
idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize);
idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0);
for ( ; ; ) {
for ( ; ; )
{
fr = iom->getNextRequest();
lbid = fr->Lbid();
ver = fr->Ver();
flg = fr->Flg();
blocksLoaded=0;
blocksRead=0;
dlen = fr->BlocksRequested();
blocksRequested = fr->BlocksRequested();
err = iom->lbidLookup(lbid, ver, flg, oid, offset);
if (err < 0) {
cerr << "lbid=" << lbid << " ver=" << ver << " flg=" << (flg ? 1 : 0) << endl;
throw runtime_error("thr_popper: BRM lookup failure");
}
fr = iom->getNextRequest();
lbid = fr->Lbid();
ver = fr->Ver();
flg = fr->Flg();
blocksLoaded = 0;
blocksRead = 0;
dlen = fr->BlocksRequested();
blocksRequested = fr->BlocksRequested();
longSeekOffset=(uint64_t)offset * (uint64_t)fileBlockSize;
totalRqst++;
sz=0;
err = iom->lbidLookup(lbid, ver, flg, oid, offset);
uint32_t readCount=0;
uint32_t bytesRead=0;
uint32_t jend = blocksRequested/iom->blocksPerRead;
for (j = 0; j <= jend; j++) {
if (err < 0)
{
cerr << "lbid=" << lbid << " ver=" << ver << " flg=" << (flg ? 1 : 0) << endl;
throw runtime_error("thr_popper: BRM lookup failure");
}
blocksThisRead = std::min(dlen, iom->blocksPerRead);
readSize = blocksThisRead * BLOCK_SIZE;
longSeekOffset = (uint64_t)offset * (uint64_t)fileBlockSize;
totalRqst++;
sz = 0;
acc = 0;
while (acc < readSize) {
i = readSize; //pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset);
/* XXXPAT: Need to decide how to handle errors here */
if (i < 0 && errno == EINTR) {
continue;
}
else if (i < 0) {
perror("thr_popper::read");
return 0; // shuts down this thread,
// probably not the right thing to do
}
else if (i == 0) {
try {
} catch (exception& exc) {
cerr << "FileName Err:" << exc.what() << endl;
}
cerr << "thr_popper: Early EOF in file " << fileNamePtr << endl;
return 0;
}
acc += i;
longSeekOffset += (uint64_t)i;
readCount++;
bytesRead+=i;
} // while(acc...
blocksRead+=blocksThisRead;
uint32_t readCount = 0;
uint32_t bytesRead = 0;
uint32_t jend = blocksRequested / iom->blocksPerRead;
for (i = 0; (unsigned)i < blocksThisRead; ++i) {
if (fbm->insert( (lbid+i) + (j*iom->blocksPerRead), ver, (uint8_t*)&alignedbuff[i*BLOCK_SIZE]))
++blocksLoaded;
}
for (j = 0; j <= jend; j++)
{
dlen -= blocksThisRead;
} // for (j...
blocksThisRead = std::min(dlen, iom->blocksPerRead);
readSize = blocksThisRead * BLOCK_SIZE;
fr->BlocksRead(blocksRead);
fr->BlocksLoaded(blocksLoaded);
acc = 0;
if (fr->data != 0 && blocksRequested == 1)
memcpy(fr->data, alignedbuff, BLOCK_SIZE);
while (acc < readSize)
{
i = readSize; //pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset);
pthread_mutex_lock(&fr->frMutex());
fr->SetPredicate(fileRequest::COMPLETE);
pthread_cond_signal(&fr->frCond());
pthread_mutex_unlock(&fr->frMutex());
/* XXXPAT: Need to decide how to handle errors here */
if (i < 0 && errno == EINTR)
{
continue;
}
else if (i < 0)
{
perror("thr_popper::read");
return 0; // shuts down this thread,
// probably not the right thing to do
}
else if (i == 0)
{
try
{
} // for(;;)
}
catch (exception& exc)
{
cerr << "FileName Err:" << exc.what() << endl;
}
lFile.close();
cerr << "thr_popper: Early EOF in file " << fileNamePtr << endl;
return 0;
}
return 0;
acc += i;
longSeekOffset += (uint64_t)i;
readCount++;
bytesRead += i;
} // while(acc...
blocksRead += blocksThisRead;
for (i = 0; (unsigned)i < blocksThisRead; ++i)
{
if (fbm->insert( (lbid + i) + (j * iom->blocksPerRead), ver, (uint8_t*)&alignedbuff[i * BLOCK_SIZE]))
++blocksLoaded;
}
dlen -= blocksThisRead;
} // for (j...
fr->BlocksRead(blocksRead);
fr->BlocksLoaded(blocksLoaded);
if (fr->data != 0 && blocksRequested == 1)
memcpy(fr->data, alignedbuff, BLOCK_SIZE);
pthread_mutex_lock(&fr->frMutex());
fr->SetPredicate(fileRequest::COMPLETE);
pthread_cond_signal(&fr->frCond());
pthread_mutex_unlock(&fr->frMutex());
} // for(;;)
lFile.close();
return 0;
} // end thr_popper
} // anonymous namespace
namespace dbbc {
ioManager::ioManager(FileBufferMgr& fbm,
fileBlockRequestQueue& fbrq,
int thrCount,
int bsPerRead):
blocksPerRead(bsPerRead),
fIOMfbMgr(fbm),
fIOMRequestQueue(fbrq)
namespace dbbc
{
if (thrCount<=0)
thrCount=1;
if (thrCount > 256)
thrCount=256;
fConfig = Config::makeConfig();
string val = fConfig->getConfig("DBBC", "IOMTracing");
int temp=0;
if (val.length()>0) temp=static_cast<int>(Config::fromText(val));
if (temp > 0)
fIOTrace=true;
else
fIOTrace=false;
ioManager::ioManager(FileBufferMgr& fbm,
fileBlockRequestQueue& fbrq,
int thrCount,
int bsPerRead):
blocksPerRead(bsPerRead),
fIOMfbMgr(fbm),
fIOMRequestQueue(fbrq)
{
fThreadCount=thrCount;
go();
if (thrCount <= 0)
thrCount = 1;
if (thrCount > 256)
thrCount = 256;
fConfig = Config::makeConfig();
string val = fConfig->getConfig("DBBC", "IOMTracing");
int temp = 0;
if (val.length() > 0) temp = static_cast<int>(Config::fromText(val));
if (temp > 0)
fIOTrace = true;
else
fIOTrace = false;
fThreadCount = thrCount;
go();
} // ioManager
void ioManager::buildOidFileName(const BRM::OID_t oid, char* file_name) {
void ioManager::buildOidFileName(const BRM::OID_t oid, char* file_name)
{
if (fFileOp.getFileName(oid, file_name) != WriteEngine::NO_ERROR) {
file_name[0]=0;
throw std::runtime_error("fileOp.getFileName failed");
}
if (fFileOp.getFileName(oid, file_name) != WriteEngine::NO_ERROR)
{
file_name[0] = 0;
throw std::runtime_error("fileOp.getFileName failed");
}
}
BRM::LBID_t ioManager::lbidLookup(BRM::LBID_t lbid,
BRM::VER_t verid,
bool vbFlag,
BRM::OID_t& oid,
uint32_t& offset)
BRM::VER_t verid,
bool vbFlag,
BRM::OID_t& oid,
uint32_t& offset)
{
int rc = fdbrm.lookup(lbid, verid, vbFlag, oid, offset);
return rc;
int rc = fdbrm.lookup(lbid, verid, vbFlag, oid, offset);
return rc;
}
int ioManager::createReaders() {
int realCnt=0;
IOMThreadArg_t fThdArgArr[256];
for (int idx=0; idx<fThreadCount; idx++){
fThdArgArr[realCnt].iom = this;
fThdArgArr[realCnt].thdId = realCnt;
int ret = pthread_create(&fThreadArr[realCnt], 0, thr_popper, &fThdArgArr[realCnt]);
if (ret!=0)
perror("createReaders::pthread_create");
else
realCnt++;
}
fThreadCount=realCnt;
return fThreadCount;
int ioManager::createReaders()
{
int realCnt = 0;
IOMThreadArg_t fThdArgArr[256];
for (int idx = 0; idx < fThreadCount; idx++)
{
fThdArgArr[realCnt].iom = this;
fThdArgArr[realCnt].thdId = realCnt;
int ret = pthread_create(&fThreadArr[realCnt], 0, thr_popper, &fThdArgArr[realCnt]);
if (ret != 0)
perror("createReaders::pthread_create");
else
realCnt++;
}
fThreadCount = realCnt;
return fThreadCount;
}
ioManager::~ioManager()
{
stop();
stop();
}
void ioManager::go(void) {
createReaders();
void ioManager::go(void)
{
createReaders();
}
void ioManager::stop() {
for(int idx=0; idx <fThreadCount; idx++) {
pthread_detach(fThreadArr[idx]);
}
void ioManager::stop()
{
for (int idx = 0; idx < fThreadCount; idx++)
{
pthread_detach(fThreadArr[idx]);
}
}
fileRequest* ioManager::getNextRequest() {
fileRequest* blk = 0;
try {
blk = fIOMRequestQueue.pop();
return blk;
} catch (exception& e) {
cerr << "ioManager::getNextRequest() ERROR " << endl;
}
fileRequest* ioManager::getNextRequest()
{
fileRequest* blk = 0;
try
{
blk = fIOMRequestQueue.pop();
return blk;
}
catch (exception& e)
{
cerr << "ioManager::getNextRequest() ERROR " << endl;
}
return blk;
return blk;
}
}

95
utils/testbc/iomanager.h
View File

@ -21,7 +21,7 @@
//
// C++ Interface: iomanager
//
// Description:
// Description:
//
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>
@ -42,52 +42,69 @@
#include "fileblockrequestqueue.h"
#include "filebuffermgr.h"
namespace dbbc {
namespace dbbc
{
class ioManager
{
class ioManager {
public:
ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount,
int bsPerRead);
//ioManager(FileBufferMgr& fbm, int thrCount);
~ioManager();
int readerCount() const {return fThreadCount;}
fileRequest* getNextRequest();
void go(void);
void stop();
FileBufferMgr& fileBufferManager() {return fIOMfbMgr;}
config::Config* configPtr() {return fConfig;}
BRM::LBID_t lbidLookup(BRM::LBID_t lbid,
BRM::VER_t verid,
bool vbFlag,
BRM::OID_t& oid,
uint32_t& offset);
void buildOidFileName(const BRM::OID_t oid, char* file_name);
uint32_t getExtentSize() { return fdbrm.getExtentSize(); }
ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount,
int bsPerRead);
//ioManager(FileBufferMgr& fbm, int thrCount);
~ioManager();
int readerCount() const
{
return fThreadCount;
}
fileRequest* getNextRequest();
void go(void);
void stop();
FileBufferMgr& fileBufferManager()
{
return fIOMfbMgr;
}
config::Config* configPtr()
{
return fConfig;
}
BRM::LBID_t lbidLookup(BRM::LBID_t lbid,
BRM::VER_t verid,
bool vbFlag,
BRM::OID_t& oid,
uint32_t& offset);
void buildOidFileName(const BRM::OID_t oid, char* file_name);
uint32_t blocksPerRead;
uint32_t getExtentSize()
{
return fdbrm.getExtentSize();
}
bool IOTrace() const { return fIOTrace;}
uint32_t blocksPerRead;
bool IOTrace() const
{
return fIOTrace;
}
private:
FileBufferMgr& fIOMfbMgr;
fileBlockRequestQueue& fIOMRequestQueue;
int fThreadCount;
pthread_t fPoppertid;
pthread_t fThreadArr[256];
int createReaders();
config::Config* fConfig;
BRM::DBRM fdbrm;
WriteEngine::FileOp fFileOp;
// do not implement
ioManager();
ioManager(const ioManager& iom);
const ioManager& operator=(const ioManager& iom);
bool fIOTrace;
FileBufferMgr& fIOMfbMgr;
fileBlockRequestQueue& fIOMRequestQueue;
int fThreadCount;
pthread_t fPoppertid;
pthread_t fThreadArr[256];
int createReaders();
config::Config* fConfig;
BRM::DBRM fdbrm;
WriteEngine::FileOp fFileOp;
// do not implement
ioManager();
ioManager(const ioManager& iom);
const ioManager& operator=(const ioManager& iom);
bool fIOTrace;
};
}

39
utils/testbc/logger.cpp
View File

@ -33,34 +33,35 @@ namespace primitiveprocessor
{
Logger::Logger() :
fMl1(LoggingID(28))
fMl1(LoggingID(28))
{
fMsgMap[logging::M0000] = Message(logging::M0000);
fMsgMap[logging::M0016] = Message(logging::M0016);
fMsgMap[logging::M0045] = Message(logging::M0045);
fMsgMap[logging::M0053] = Message(logging::M0053);
fMsgMap[logging::M0000] = Message(logging::M0000);
fMsgMap[logging::M0016] = Message(logging::M0016);
fMsgMap[logging::M0045] = Message(logging::M0045);
fMsgMap[logging::M0053] = Message(logging::M0053);
}
void Logger::logMessage(const Message::MessageID mid,
const Message::Args& args,
bool critical)
{
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
msgIter->second.reset();
msgIter->second.format(args);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
if (critical)
{
fMl1.logCriticalMessage(msgIter->second);
}
else
{
fMl1.logWarningMessage(msgIter->second);
}
msgIter->second.reset();
msgIter->second.format(args);
if (critical)
{
fMl1.logCriticalMessage(msgIter->second);
}
else
{
fMl1.logWarningMessage(msgIter->second);
}
}
}

32
utils/testbc/logger.h
View File

@ -37,29 +37,29 @@ namespace primitiveprocessor
class Logger
{
public:
Logger();
Logger();
void logMessage(const logging::Message::MessageID mid,
void logMessage(const logging::Message::MessageID mid,
const logging::Message::Args& args,
bool critical=false);
bool critical = false);
void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 )
{
logging::Message::Args args;
args.add(msg);
logMessage(mid, args, true);
}
void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0 )
{
logging::Message::Args args;
args.add(msg);
logMessage(mid, args, true);
}
private:
// defaults okay
//Logger(const Logger& rhs);
//Logger& operator=(const Logger& rhs);
// defaults okay
//Logger(const Logger& rhs);
//Logger& operator=(const Logger& rhs);
typedef std::map<logging::Message::MessageID, logging::Message> MsgMap;
typedef std::map<logging::Message::MessageID, logging::Message> MsgMap;
MsgMap fMsgMap;
boost::mutex fLogLock;
logging::MessageLog fMl1;
MsgMap fMsgMap;
boost::mutex fLogLock;
logging::MessageLog fMl1;
};

297
utils/testbc/stats.cpp
View File

@ -44,111 +44,120 @@ namespace
void pause_(unsigned delay)
{
struct timespec req;
struct timespec rem;
struct timespec req;
struct timespec rem;
req.tv_sec = delay;
req.tv_nsec = 0;
req.tv_sec = delay;
req.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
again:
if (nanosleep(&req, &rem) != 0)
if (rem.tv_sec > 0 || rem.tv_nsec > 0) {
req = rem;
goto again;
}
if (nanosleep(&req, &rem) != 0)
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
goto again;
}
}
const string timestr()
{
// Get a timestamp for output.
struct tm tm;
struct timeval tv;
// Get a timestamp for output.
struct tm tm;
struct timeval tv;
gettimeofday(&tv, 0);
localtime_r(&tv.tv_sec, &tm);
gettimeofday(&tv, 0);
localtime_r(&tv.tv_sec, &tm);
ostringstream oss;
oss << setfill('0')
<< setw(2) << tm.tm_hour << ':'
<< setw(2) << tm.tm_min << ':'
<< setw(2) << tm.tm_sec << '.'
<< setw(4) << tv.tv_usec/100;
ostringstream oss;
oss << setfill('0')
<< setw(2) << tm.tm_hour << ':'
<< setw(2) << tm.tm_min << ':'
<< setw(2) << tm.tm_sec << '.'
<< setw(4) << tv.tv_usec / 100;
return oss.str();
return oss.str();
}
class TraceFile
{
public:
TraceFile(uint32_t sessionID, const char* name)
{
if (sessionID > 0 )
{
const char* outName;
if (name == 0)
outName = "lbids";
else
outName = name;
ostringstream oss;
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID;
oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
}
}
TraceFile(uint32_t sessionID, const char* name)
{
if (sessionID > 0 )
{
const char* outName;
~TraceFile()
{
}
if (name == 0)
outName = "lbids";
else
outName = name;
void close()
{
if (oFile)
{
oFile->close();
}
}
ostringstream oss;
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID;
oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
}
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event='\0')
{
*oFile << oid << ' ' << timestr() << ' ' << lbid
<< ' ' << thdid;
if (event != '\0')
*oFile << ' ' << event;
*oFile << endl;
oFile->flush();
}
~TraceFile()
{
}
void close()
{
if (oFile)
{
oFile->close();
}
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
*oFile << oid << ' ' << timestr() << ' ' << lbid
<< ' ' << thdid;
if (event != '\0')
*oFile << ' ' << event;
*oFile << endl;
oFile->flush();
}
private:
//Compiler defaults okay
//TraceFile(const TraceFile& rhs);
//TraceFile operator=(const TraceFile& rhs);
shared_ptr<ofstream> oFile;
//Compiler defaults okay
//TraceFile(const TraceFile& rhs);
//TraceFile operator=(const TraceFile& rhs);
shared_ptr<ofstream> oFile;
};
struct TraceFileInfo
{
TraceFileInfo(uint32_t session=0, const char* name=0) : traceFile(session, name), lastTouched(0) { }
~TraceFileInfo() { }
TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0) { }
~TraceFileInfo() { }
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event='\0')
{
traceFile.log(oid, lbid, thdid, event);
lastTouched = time(0);
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
traceFile.log(oid, lbid, thdid, event);
lastTouched = time(0);
}
void close() { traceFile.close(); }
void close()
{
traceFile.close();
}
TraceFile traceFile;
time_t lastTouched;
TraceFile traceFile;
time_t lastTouched;
private:
//Compiler defaults okay
//TraceFileInfo(const TraceFileInfo& rhs);
//TraceFileInfo operator=(const TraceFileInfo& rhs);
//Compiler defaults okay
//TraceFileInfo(const TraceFileInfo& rhs);
//TraceFileInfo operator=(const TraceFileInfo& rhs);
};
//map a session id to a trace file
@ -161,47 +170,49 @@ mutex traceFileMapMutex;
class StatMon
{
public:
StatMon()
{
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
pthread_sigmask(SIG_BLOCK, &sigset, 0);
}
void operator()() const
{
//struct timespec ts = { 60 * 1, 0 };
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter;
TraceFileMap_t::iterator end;
for (;;)
{
lk.unlock();
time_t beforeSleep = time(0);
//nanosleep(&ts, 0);
pause_(60);
lk.lock();
iter = traceFileMap.begin();
end = traceFileMap.end();
while (iter != end)
{
if (iter->second.lastTouched < beforeSleep)
{
//remove this session trace file
iter->second.close();
traceFileMap.erase(iter++);
}
else
++iter;
}
}
}
StatMon()
{
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
pthread_sigmask(SIG_BLOCK, &sigset, 0);
}
void operator()() const
{
//struct timespec ts = { 60 * 1, 0 };
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter;
TraceFileMap_t::iterator end;
for (;;)
{
lk.unlock();
time_t beforeSleep = time(0);
//nanosleep(&ts, 0);
pause_(60);
lk.lock();
iter = traceFileMap.begin();
end = traceFileMap.end();
while (iter != end)
{
if (iter->second.lastTouched < beforeSleep)
{
//remove this session trace file
iter->second.close();
traceFileMap.erase(iter++);
}
else
++iter;
}
}
}
private:
//Compiler defaults okay
//StatMon(const StatMon& rhs);
//StatMon operator=(const StatMon& rhs);
//Compiler defaults okay
//StatMon(const StatMon& rhs);
//StatMon operator=(const StatMon& rhs);
};
}
@ -210,52 +221,56 @@ namespace dbbc
{
Stats::Stats() :
fMonitorp(0)
fMonitorp(0)
{
fMonitorp = new thread(StatMon());
fMonitorp = new thread(StatMon());
}
Stats::Stats(const char *name) :
fMonitorp(0), fName(name)
Stats::Stats(const char* name) :
fMonitorp(0), fName(name)
{
fMonitorp = new thread(StatMon());
//fName << name;
fMonitorp = new thread(StatMon());
//fName << name;
}
Stats::~Stats()
{
delete fMonitorp;
delete fMonitorp;
}
void Stats::touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session)
{
if (lbid < 0 || session == 0) return;
if (lbid < 0 || session == 0) return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid);
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid);
}
void Stats::markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event)
{
if (lbid < 0 || session == 0) return;
if (lbid < 0 || session == 0) return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session, fName);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid, event);
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session, fName);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid, event);
}
}

36
utils/testbc/stats.h
View File

@ -37,29 +37,29 @@ namespace dbbc
class Stats
{
public:
Stats();
Stats(const char *name);
virtual ~Stats();
Stats();
Stats(const char* name);
virtual ~Stats();
void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session=0);
void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event);
void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0);
void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event);
inline BRM::OID_t lbid2oid(uint64_t lbid)
{
BRM::OID_t oid;
uint32_t fbo;
brm.lookup(lbid, 0, false, oid, fbo);
return oid;
}
inline BRM::OID_t lbid2oid(uint64_t lbid)
{
BRM::OID_t oid;
uint32_t fbo;
brm.lookup(lbid, 0, false, oid, fbo);
return oid;
}
private:
Stats(const Stats& rhs);
Stats& operator=(const Stats& rhs);
Stats(const Stats& rhs);
Stats& operator=(const Stats& rhs);
boost::thread* fMonitorp;
BRM::DBRM brm;
//ostringstream fName;
const char* fName;
boost::thread* fMonitorp;
BRM::DBRM brm;
//ostringstream fName;
const char* fName;
};

430
utils/testbc/testbc.cpp
View File

@ -45,158 +45,169 @@ using namespace std;
using namespace logging;
using namespace primitiveprocessor;
Stats* gPMStatsPtr=NULL;
bool gPMProfOn=false;
uint32_t gSession=0;
uint32_t lastRangeListIdx=0;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
uint32_t lastRangeListIdx = 0;
void timespec_sub(const struct timespec &tv1,
const struct timespec &tv2,
double &tm)
void timespec_sub(const struct timespec& tv1,
const struct timespec& tv2,
double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9*(tv2.tv_nsec - tv1.tv_nsec);
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
namespace primitiveprocessor
{
Logger ml;
Logger ml;
}
class BCTest {
class BCTest
{
public:
struct OidRanges {
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid=o;
hwm=h;
ranges=r;
}
}; //struct OidRanges
struct OidRanges
{
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid = o;
hwm = h;
ranges = r;
}
}; //struct OidRanges
BCTest(const int cacheSz=64*1024, int readThr=2, int readAhead=1024);
BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024);
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t>OidRangesList_t;
OidRangesList_t OidRangesList;
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t>OidRangesList_t;
OidRangesList_t OidRangesList;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
BlockRequestProcessor BRP;
BlockRequestProcessor BRP;
}; // class BCTest
BCTest::BCTest(int cacheSz, int readThr, int readAhead) :
fCacheSz(cacheSz),
fReadThr(readThr),
fReadAhead(readAhead),
BRP(fCacheSz, fReadThr, fReadAhead)
fCacheSz(cacheSz),
fReadThr(readThr),
fReadAhead(readAhead),
BRP(fCacheSz, fReadThr, fReadAhead)
{
setUp();
setUp();
} // ctor
//
void BCTest::setUp()
{
LBIDRange_v r;
HWM_t hwm;
OID_t oid=1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable=0;
int i=0;
fExtentSize=dbrm.getExtentSize();
LBIDRange_v r;
HWM_t hwm;
OID_t oid = 1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable = 0;
int i = 0;
fExtentSize = dbrm.getExtentSize();
while ( oid < 5000 )
{
int ret=0;
ret = dbrm.lookup(oid, r);
if (ret==0 && r.size()>0) {
dbrm.getHWM(oid, hwm);
maxBlocksAvailable+=(r.size()*extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
//cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm=0;
r.clear();
}
oid++;
}
while ( oid < 5000 )
{
int ret = 0;
ret = dbrm.lookup(oid, r);
//cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i=0;
if (ret == 0 && r.size() > 0)
{
dbrm.getHWM(oid, hwm);
maxBlocksAvailable += (r.size() * extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
//cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm = 0;
r.clear();
}
oid++;
}
//cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i = 0;
} // setUp()
int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
uint32_t rCount=0;
blockCacheClient bc(BRP);
uint32_t rCount = 0;
for (uint32_t i =0; i<o.ranges.size() ; i++)
{
const InlineLBIDRange r={o.ranges[i].start, o.ranges[i].size};
if (r.size>0) {
bc.check(r, 0, rCount);
//cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount+=rCount;
}
rCount=0;
} // for
for (uint32_t i = 0; i < o.ranges.size() ; i++)
{
const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size};
//cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
if (r.size > 0)
{
bc.check(r, 0, rCount);
//cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount += rCount;
}
return loadCount;
rCount = 0;
} // for
//cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
return loadCount;
} // LoadOid
int BCTest::ReadOidRanges(const OidRanges_t& v)
{
blockCacheClient bc(BRP);
int32_t readBlocks=0;
int32_t missBlocks=0;
//int ret;
for(uint32_t i=0; i<v.ranges.size(); i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r={v.ranges[i].start, v.ranges[i].size};
for(int j=r.start; readBlocks<fCacheSz && j<r.start+r.size; j++)
{
//ret=0;
//ret=bc.read(j, 0, fb);
FileBuffer* ptr =bc.getBlockPtr(j, 0);
if (ptr)
readBlocks++;
else
missBlocks++;
}
blockCacheClient bc(BRP);
int32_t readBlocks = 0;
int32_t missBlocks = 0;
//cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl;
}
//int ret;
for (uint32_t i = 0; i < v.ranges.size(); i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size};
return readBlocks;
for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++)
{
//ret=0;
//ret=bc.read(j, 0, fb);
FileBuffer* ptr = bc.getBlockPtr(j, 0);
if (ptr)
readBlocks++;
else
missBlocks++;
}
//cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl;
}
return readBlocks;
} // ReadRange
@ -204,157 +215,164 @@ int BCTest::ReadOidRanges(const OidRanges_t& v)
//
void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
} // LoadLbid
// get one block out of block cache
//
void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
uint8_t d[8192]={'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
uint8_t d[8192] = {'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
} // ReadLbid
struct loadThr
{
loadThr::loadThr(BCTest& bc, int reps=1) : fBC(bc), fReps(reps) {}
loadThr::loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {}
void operator()()
{
uint32_t loadedBlocks=0;
uint32_t oidBlocks;
uint32_t i=0;
uint32_t rc=0;
void operator()()
{
uint32_t loadedBlocks = 0;
uint32_t oidBlocks;
uint32_t i = 0;
uint32_t rc = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
for(uint32_t j=0; j<fReps; j++)
for(i=0; /*loadedBlocks<cacheSize &&*/ i<fBC.OidRangesList.size(); i++)
{
oidBlocks=0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
/**
cout << "."
<< "-- " << i << " " << fBC.OidRangesList[i].oid
<< " h: " << fBC.OidRangesList[i].hwm
<< "/" << oidBlocks
<< endl;
**/
loadedBlocks+=oidBlocks;
}
clock_gettime(CLOCK_REALTIME, &tm1);
clock_gettime(CLOCK_REALTIME, &tm2);
double tm3;
timespec_sub(tm1, tm2, tm3);
lastRangeListIdx=i;
for (uint32_t j = 0; j < fReps; j++)
for (i = 0; /*loadedBlocks<cacheSize &&*/ i < fBC.OidRangesList.size(); i++)
{
oidBlocks = 0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
/**
cout << "."
<< "-- " << i << " " << fBC.OidRangesList[i].oid
<< " h: " << fBC.OidRangesList[i].hwm
<< "/" << oidBlocks
<< endl;
**/
loadedBlocks += oidBlocks;
}
cout << "loadtest ld: " << loadedBlocks
<< " sz: " << fBC.fCacheSz
//<< " last: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
clock_gettime(CLOCK_REALTIME, &tm2);
double tm3;
timespec_sub(tm1, tm2, tm3);
lastRangeListIdx = i;
} // operator()
cout << "loadtest ld: " << loadedBlocks
<< " sz: " << fBC.fCacheSz
//<< " last: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
BCTest& fBC;
uint32_t fReps;
struct timespec tm1;
struct timespec tm2;
} // operator()
BCTest& fBC;
uint32_t fReps;
struct timespec tm1;
struct timespec tm2;
};
struct readThr
{
readThr::readThr(BCTest& bc, int reps=1) : fBC(bc), fReps(reps) {}
readThr::readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {}
void operator()()
{
uint32_t rc=0;
uint32_t loadedBlocks=0;
for (uint32_t k=0; k<fReps; k++) {
for(uint32_t i=0; i<fBC.OidRangesList.size() ; i++)
{
rc = fBC.ReadOidRanges(fBC.OidRangesList[i]);
cout << ".";
//cout << "-- ReadTest " << OidRangesList[i].oid << " h: " << OidRangesList[i].hwm << "/" << rc << endl;
loadedBlocks+=rc;
rc=0;
}
}
cout << "loadtest " << loadedBlocks << " " << rc << endl<<endl;
void operator()()
{
uint32_t rc = 0;
uint32_t loadedBlocks = 0;
} // operator()
for (uint32_t k = 0; k < fReps; k++)
{
for (uint32_t i = 0; i < fBC.OidRangesList.size() ; i++)
{
rc = fBC.ReadOidRanges(fBC.OidRangesList[i]);
cout << ".";
//cout << "-- ReadTest " << OidRangesList[i].oid << " h: " << OidRangesList[i].hwm << "/" << rc << endl;
loadedBlocks += rc;
rc = 0;
}
}
cout << "loadtest " << loadedBlocks << " " << rc << endl << endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
BCTest& fBC;
uint32_t fReps;
};
void usage()
{
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
}
//
int main(int argc, char* argv[]) {
int main(int argc, char* argv[])
{
int cacheSz=128; // K number of blocks
int thr=1;
int ra=1024;
int clients=1;
int reps=1;
int cacheSz = 128; // K number of blocks
int thr = 1;
int ra = 1024;
int clients = 1;
int reps = 1;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz=atoi(argv[1])*1024;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz = atoi(argv[1]) * 1024;
if (argc > 2 && atoi(argv[2]) > 0)
thr=atoi(argv[2]);
if (argc > 2 && atoi(argv[2]) > 0)
thr = atoi(argv[2]);
if (argc > 3 && atoi(argv[3]) > 0)
ra=atoi(argv[3]);
if (argc > 3 && atoi(argv[3]) > 0)
ra = atoi(argv[3]);
if (argc > 4 && atoi(argv[4]) > 0)
clients=atoi(argv[4]);
if (argc > 4 && atoi(argv[4]) > 0)
clients = atoi(argv[4]);
if (argc > 5 && atoi(argv[5]) > 0)
reps=atoi(argv[5]);
if (argc > 5 && atoi(argv[5]) > 0)
reps = atoi(argv[5]);
BCTest bc(cacheSz, thr, ra);
BCTest bc(cacheSz, thr, ra);
cout <<
"Cache Size: " << cacheSz <<
" read Threads: " << thr <<
" read Ahead: " << ra <<
" clients: " << clients <<
" repetitions: " << reps <<
" max Blocks: " << bc.maxBlocksAvailable <<
endl;
cout <<
"Cache Size: " << cacheSz <<
" read Threads: " << thr <<
" read Ahead: " << ra <<
" clients: " << clients <<
" repetitions: " << reps <<
" max Blocks: " << bc.maxBlocksAvailable <<
endl;
// loader test
struct loadThr loader1(bc, reps);
vector<boost::thread*> v;
struct loadThr loader1(bc, reps);
vector<boost::thread*> v;
for (int i=0; i<clients;i++) {
boost::thread* th1 = new boost::thread(loader1);
v.push_back(th1);
}
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = new boost::thread(loader1);
v.push_back(th1);
}
for (int i=0; i<clients;i++) {
boost::thread* th1 = v[i];
th1->join();
delete th1;
}
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = v[i];
th1->join();
delete th1;
}
v.clear();
v.clear();
// reader test
return 0;
return 0;
} // end main

425
utils/testbc/testbc2.cpp
View File

@ -45,163 +45,175 @@ using namespace std;
using namespace logging;
using namespace primitiveprocessor;
Stats* gPMStatsPtr=NULL;
bool gPMProfOn=false;
uint32_t gSession=0;
uint32_t lastRangeListIdx=0;
const uint32_t maxLoadBlocks(1024*1024);
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
uint32_t lastRangeListIdx = 0;
const uint32_t maxLoadBlocks(1024 * 1024);
void timespec_sub(const struct timespec &tv1,
const struct timespec &tv2,
double &tm)
void timespec_sub(const struct timespec& tv1,
const struct timespec& tv2,
double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9*(tv2.tv_nsec - tv1.tv_nsec);
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
namespace primitiveprocessor
{
Logger ml;
Logger ml;
}
class BCTest {
class BCTest
{
public:
struct OidRanges {
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid=o;
hwm=h;
ranges=r;
}
}; //struct OidRanges
struct OidRanges
{
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid = o;
hwm = h;
ranges = r;
}
}; //struct OidRanges
BCTest(const int cacheSz=64*1024, int readThr=2, int readAhead=1024);
BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024);
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t>OidRangesList_t;
OidRangesList_t OidRangesList;
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t>OidRangesList_t;
OidRangesList_t OidRangesList;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
BlockRequestProcessor BRP;
BlockRequestProcessor BRP;
}; // class BCTest
BCTest::BCTest(int cacheSz, int readThr, int readAhead) :
fCacheSz(cacheSz),
fReadThr(readThr),
fReadAhead(readAhead),
BRP(fCacheSz, fReadThr, fReadAhead)
fCacheSz(cacheSz),
fReadThr(readThr),
fReadAhead(readAhead),
BRP(fCacheSz, fReadThr, fReadAhead)
{
setUp();
setUp();
} // ctor
//
void BCTest::setUp()
{
LBIDRange_v r;
HWM_t hwm;
OID_t oid=1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable=0;
int i=0;
fExtentSize=dbrm.getExtentSize();
LBIDRange_v r;
HWM_t hwm;
OID_t oid = 1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable = 0;
int i = 0;
fExtentSize = dbrm.getExtentSize();
while ( oid < 5000 )
{
int ret=0;
ret = dbrm.lookup(oid, r);
if (ret==0 && r.size()>0) {
dbrm.getHWM(oid, hwm);
maxBlocksAvailable+=(r.size()*extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
//cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm=0;
r.clear();
}
oid++;
}
while ( oid < 5000 )
{
int ret = 0;
ret = dbrm.lookup(oid, r);
//cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i=0;
if (ret == 0 && r.size() > 0)
{
dbrm.getHWM(oid, hwm);
maxBlocksAvailable += (r.size() * extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
//cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm = 0;
r.clear();
}
oid++;
}
//cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i = 0;
} // setUp()
int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
uint32_t rCount=0;
blockCacheClient bc(BRP);
uint32_t rCount = 0;
for (uint32_t i =0; i<o.ranges.size() ; i++)
{
const InlineLBIDRange r={o.ranges[i].start, o.ranges[i].size};
if (r.size>0) {
bc.check(r, 0, rCount);
//cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount+=rCount;
}
rCount=0;
} // for
for (uint32_t i = 0; i < o.ranges.size() ; i++)
{
const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size};
//cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
if (r.size > 0)
{
bc.check(r, 0, rCount);
//cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount += rCount;
}
return loadCount;
rCount = 0;
} // for
//cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
return loadCount;
} // LoadOid
int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss)
{
blockCacheClient bc(BRP);
uint8_t inBuff[8192];
int32_t readBlocks=0;
int32_t missBlocks=0;
blockCacheClient bc(BRP);
uint8_t inBuff[8192];
int32_t readBlocks = 0;
int32_t missBlocks = 0;
for(uint32_t i=0; i<v.ranges.size(); i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r={v.ranges[i].start, v.ranges[i].size};
for(int j=r.start; readBlocks<fCacheSz && j<r.start+r.size; j++)
{
FileBuffer* ptr =bc.getBlockPtr(j, 0);
if (ptr) {
readBlocks++;
memcpy(inBuff, ptr->getData(), 8192);
}
else
missBlocks++;
}
*hits+=readBlocks;
*miss+=missBlocks;
missBlocks=0;
readBlocks=0;
//cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl;
}
for (uint32_t i = 0; i < v.ranges.size(); i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size};
return *hits;
for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++)
{
FileBuffer* ptr = bc.getBlockPtr(j, 0);
if (ptr)
{
readBlocks++;
memcpy(inBuff, ptr->getData(), 8192);
}
else
missBlocks++;
}
*hits += readBlocks;
*miss += missBlocks;
missBlocks = 0;
readBlocks = 0;
//cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl;
}
return *hits;
} // ReadRange
@ -209,146 +221,151 @@ int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss)
//
void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
} // LoadLbid
// get one block out of block cache
//
void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
uint8_t d[8192]={'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
uint8_t d[8192] = {'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
} // ReadLbid
struct loadThr
{
loadThr(BCTest& bc, int reps=1) : fBC(bc), fReps(reps) {}
loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {}
void operator()()
{
uint32_t loadedBlocks=0;
uint32_t readBlocks=0;
uint32_t missBlocks=0;
uint32_t oidBlocks;
uint32_t i=0;
uint32_t rc=0;
void operator()()
{
uint32_t loadedBlocks = 0;
uint32_t readBlocks = 0;
uint32_t missBlocks = 0;
uint32_t oidBlocks;
uint32_t i = 0;
uint32_t rc = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
for(uint32_t j=0; j<fReps; j++)
for(i=0; loadedBlocks<maxLoadBlocks && i<fBC.OidRangesList.size(); i++)
{
oidBlocks=0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
/**
cout << "."
<< "-- " << i << " " << fBC.OidRangesList[i].oid
<< " h: " << fBC.OidRangesList[i].hwm
<< "/" << oidBlocks
<< endl;
**/
loadedBlocks+=oidBlocks;
readBlocks+=fBC.ReadOidRanges(fBC.OidRangesList[i], &readBlocks, &missBlocks);
clock_gettime(CLOCK_REALTIME, &tm1);
}
for (uint32_t j = 0; j < fReps; j++)
for (i = 0; loadedBlocks < maxLoadBlocks && i < fBC.OidRangesList.size(); i++)
{
oidBlocks = 0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
/**
cout << "."
<< "-- " << i << " " << fBC.OidRangesList[i].oid
<< " h: " << fBC.OidRangesList[i].hwm
<< "/" << oidBlocks
<< endl;
**/
loadedBlocks += oidBlocks;
readBlocks += fBC.ReadOidRanges(fBC.OidRangesList[i], &readBlocks, &missBlocks);
clock_gettime(CLOCK_REALTIME, &tm2);
double tm3;
timespec_sub(tm1, tm2, tm3);
lastRangeListIdx=i;
}
cout << "loadtest ld: " << loadedBlocks
<< " rd: " << readBlocks << "/" << missBlocks
<< " sz: " << fBC.fCacheSz
//<< " last: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
clock_gettime(CLOCK_REALTIME, &tm2);
double tm3;
timespec_sub(tm1, tm2, tm3);
lastRangeListIdx = i;
} // operator()
cout << "loadtest ld: " << loadedBlocks
<< " rd: " << readBlocks << "/" << missBlocks
<< " sz: " << fBC.fCacheSz
//<< " last: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
BCTest& fBC;
uint32_t fReps;
struct timespec tm1;
struct timespec tm2;
} // operator()
BCTest& fBC;
uint32_t fReps;
struct timespec tm1;
struct timespec tm2;
};
struct readThr
{
readThr(BCTest& bc, int reps=1) : fBC(bc), fReps(reps) {}
readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {}
void operator()()
{
for (uint32_t k=0; k<fReps; k++) {
}
void operator()()
{
for (uint32_t k = 0; k < fReps; k++)
{
}
} // operator()
} // operator()
BCTest& fBC;
uint32_t fReps;
BCTest& fBC;
uint32_t fReps;
};
void usage()
{
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
}
//
int main(int argc, char* argv[]) {
int main(int argc, char* argv[])
{
int cacheSz=128; // K number of blocks
int thr=1;
int ra=1024;
int clients=1;
int reps=1;
int cacheSz = 128; // K number of blocks
int thr = 1;
int ra = 1024;
int clients = 1;
int reps = 1;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz=atoi(argv[1])*1024;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz = atoi(argv[1]) * 1024;
if (argc > 2 && atoi(argv[2]) > 0)
thr=atoi(argv[2]);
if (argc > 2 && atoi(argv[2]) > 0)
thr = atoi(argv[2]);
if (argc > 3 && atoi(argv[3]) > 0)
ra=atoi(argv[3]);
if (argc > 3 && atoi(argv[3]) > 0)
ra = atoi(argv[3]);
if (argc > 4 && atoi(argv[4]) > 0)
clients=atoi(argv[4]);
if (argc > 4 && atoi(argv[4]) > 0)
clients = atoi(argv[4]);
if (argc > 5 && atoi(argv[5]) > 0)
reps=atoi(argv[5]);
if (argc > 5 && atoi(argv[5]) > 0)
reps = atoi(argv[5]);
BCTest bc(cacheSz, thr, ra);
BCTest bc(cacheSz, thr, ra);
cout <<
"Cache Size: " << cacheSz <<
" read Threads: " << thr <<
" read Ahead: " << ra <<
" clients: " << clients <<
" repetitions: " << reps <<
" max Blocks: " << bc.maxBlocksAvailable <<
endl;
cout <<
"Cache Size: " << cacheSz <<
" read Threads: " << thr <<
" read Ahead: " << ra <<
" clients: " << clients <<
" repetitions: " << reps <<
" max Blocks: " << bc.maxBlocksAvailable <<
endl;
// loader test
struct loadThr loader1(bc, reps);
vector<boost::thread*> v;
struct loadThr loader1(bc, reps);
vector<boost::thread*> v;
for (int i=0; i<clients;i++) {
boost::thread* th1 = new boost::thread(loader1);
v.push_back(th1);
}
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = new boost::thread(loader1);
v.push_back(th1);
}
for (int i=0; i<clients;i++) {
boost::thread* th1 = v[i];
th1->join();
delete th1;
}
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = v[i];
th1->join();
delete th1;
}
v.clear();
v.clear();
return 0;
return 0;
} // end main

497
utils/testbc/testbc3.cpp
View File

@ -45,164 +45,176 @@ using namespace std;
using namespace logging;
using namespace primitiveprocessor;
Stats* gPMStatsPtr=NULL;
bool gPMProfOn=false;
uint32_t gSession=0;
uint32_t lastRangeListIdx=0;
const uint32_t maxLoadBlocks(1024*1024);
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
uint32_t lastRangeListIdx = 0;
const uint32_t maxLoadBlocks(1024 * 1024);
void timespec_sub(const struct timespec &tv1,
const struct timespec &tv2,
double &tm)
void timespec_sub(const struct timespec& tv1,
const struct timespec& tv2,
double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9*(tv2.tv_nsec - tv1.tv_nsec);
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
namespace primitiveprocessor
{
Logger ml;
Logger ml;
}
class BCTest {
class BCTest
{
public:
struct OidRanges {
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid=o;
hwm=h;
ranges=r;
}
}; //struct OidRanges
struct OidRanges
{
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid = o;
hwm = h;
ranges = r;
}
}; //struct OidRanges
BCTest(const int cacheSz=64*1024, int readThr=2, int readAhead=1024);
BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024);
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t>OidRangesList_t;
OidRangesList_t OidRangesList;
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t>OidRangesList_t;
OidRangesList_t OidRangesList;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
BlockRequestProcessor BRP;
BlockRequestProcessor BRP;
}; // class BCTest
BCTest::BCTest(int cacheSz, int readThr, int readAhead) :
fCacheSz(cacheSz),
fReadThr(readThr),
fReadAhead(readAhead),
BRP(fCacheSz, fReadThr, fReadAhead)
fCacheSz(cacheSz),
fReadThr(readThr),
fReadAhead(readAhead),
BRP(fCacheSz, fReadThr, fReadAhead)
{
setUp();
setUp();
} // ctor
//
void BCTest::setUp()
{
LBIDRange_v r;
HWM_t hwm;
OID_t oid=1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable=0;
int i=0;
fExtentSize=dbrm.getExtentSize();
LBIDRange_v r;
HWM_t hwm;
OID_t oid = 1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable = 0;
int i = 0;
fExtentSize = dbrm.getExtentSize();
while ( oid < 5000 )
{
int ret=0;
ret = dbrm.lookup(oid, r);
if (ret==0 && r.size()>0) {
dbrm.getHWM(oid, hwm);
maxBlocksAvailable+=(r.size()*extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
//cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm=0;
r.clear();
}
oid++;
}
while ( oid < 5000 )
{
int ret = 0;
ret = dbrm.lookup(oid, r);
//cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i=0;
if (ret == 0 && r.size() > 0)
{
dbrm.getHWM(oid, hwm);
maxBlocksAvailable += (r.size() * extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
//cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm = 0;
r.clear();
}
oid++;
}
//cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i = 0;
} // setUp()
int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
uint32_t rCount=0;
blockCacheClient bc(BRP);
uint32_t rCount = 0;
for (uint32_t i =0; i<o.ranges.size() ; i++)
{
const InlineLBIDRange r={o.ranges[i].start, o.ranges[i].size};
if (r.size>0) {
bc.check(r, 0, rCount);
//cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount+=rCount;
}
rCount=0;
} // for
for (uint32_t i = 0; i < o.ranges.size() ; i++)
{
const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size};
//cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
if (r.size > 0)
{
bc.check(r, 0, rCount);
//cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount += rCount;
}
return loadCount;
rCount = 0;
} // for
//cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
return loadCount;
} // LoadOid
int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss)
{
blockCacheClient bc(BRP);
uint32_t readBlocks=0;
uint32_t missBlocks=0;
uint8_t inBuff[8192];
int ret;
blockCacheClient bc(BRP);
uint32_t readBlocks = 0;
uint32_t missBlocks = 0;
uint8_t inBuff[8192];
int ret;
for(uint32_t i=0; i<v.ranges.size() && i < lastRangeListIdx; i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r={v.ranges[i].start, v.ranges[i].size};
for(int j=r.start; j<r.start+r.size; j++)
{
ret=bc.read(j, 0, fb);
FileBuffer* ptr =bc.getBlockPtr(j, 0);
if (ptr) {
readBlocks++;
memcpy(inBuff, ptr->getData(), 8192);
}
else
missBlocks++;
}
*hits+=readBlocks;
*miss+=missBlocks;
for (uint32_t i = 0; i < v.ranges.size() && i < lastRangeListIdx; i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size};
//cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl;
}
for (int j = r.start; j < r.start + r.size; j++)
{
ret = bc.read(j, 0, fb);
FileBuffer* ptr = bc.getBlockPtr(j, 0);
return readBlocks;
if (ptr)
{
readBlocks++;
memcpy(inBuff, ptr->getData(), 8192);
}
else
missBlocks++;
}
*hits += readBlocks;
*miss += missBlocks;
//cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl;
}
return readBlocks;
} // ReadRange
@ -210,184 +222,193 @@ int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss)
//
void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
} // LoadLbid
// get one block out of block cache
//
void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
uint8_t d[8192]={'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
uint8_t d[8192] = {'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
} // ReadLbid
struct loadThr
{
loadThr(BCTest& bc, int reps=1) : fBC(bc), fReps(reps) {
}
loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
{
}
void operator()()
{
uint32_t loadedBlocks=0;
uint32_t oidBlocks;
uint32_t i=0;
uint32_t rc=0;
struct timespec tm1;
struct timespec tm2;
void operator()()
{
uint32_t loadedBlocks = 0;
uint32_t oidBlocks;
uint32_t i = 0;
uint32_t rc = 0;
struct timespec tm1;
struct timespec tm2;
tm1.tv_sec=0;
tm1.tv_nsec=0;
tm2.tv_sec=0;
tm2.tv_nsec=0;
double tm3=0;
clock_gettime(CLOCK_REALTIME, &tm1);
for(i=0; (loadedBlocks+(fBC.OidRangesList[i].ranges.size()*fBC.extentSize))<fBC.fCacheSz && i<fBC.OidRangesList.size(); i++)
{
oidBlocks=0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
//cout
// << "-- Load " << i << " " << fBC.OidRangesList[i].oid
// << " h: " << fBC.OidRangesList[i].hwm
// << "/" << oidBlocks << " " << loadedBlocks
// << endl;
loadedBlocks+=oidBlocks;
} // for (i...
lastRangeListIdx=i;
tm1.tv_sec = 0;
tm1.tv_nsec = 0;
tm2.tv_sec = 0;
tm2.tv_nsec = 0;
double tm3 = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
clock_gettime(CLOCK_REALTIME, &tm2);
timespec_sub(tm1, tm2, tm3);
for (i = 0; (loadedBlocks + (fBC.OidRangesList[i].ranges.size()*fBC.extentSize)) < fBC.fCacheSz && i < fBC.OidRangesList.size(); i++)
{
oidBlocks = 0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
//cout
// << "-- Load " << i << " " << fBC.OidRangesList[i].oid
// << " h: " << fBC.OidRangesList[i].hwm
// << "/" << oidBlocks << " " << loadedBlocks
// << endl;
loadedBlocks += oidBlocks;
} // for (i...
cout << "load ld: " << loadedBlocks
<< " sz: " << fBC.fCacheSz
<< " rng: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
lastRangeListIdx = i;
} // operator()
clock_gettime(CLOCK_REALTIME, &tm2);
timespec_sub(tm1, tm2, tm3);
BCTest& fBC;
uint32_t fReps;
cout << "load ld: " << loadedBlocks
<< " sz: " << fBC.fCacheSz
<< " rng: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
};
struct readThr
{
readThr(BCTest& bc, int reps=1) : fBC(bc), fReps(reps) {}
readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {}
void operator()()
{
uint32_t rc=0;
uint32_t readBlocks=0;
uint32_t hits=0;
uint32_t miss=0;
uint32_t hitsTot=0;
uint32_t missTot=0;
struct timespec tm1;
struct timespec tm2;
tm1.tv_sec=0;
tm1.tv_nsec=0;
tm2.tv_sec=0;
tm2.tv_nsec=0;
double tm3=0;
clock_gettime(CLOCK_REALTIME, &tm1);
for (uint32_t k=0; k<fReps; k++) {
for(uint32_t i=0; i<lastRangeListIdx && i<fBC.OidRangesList.size() ; i++)
{
rc = fBC.ReadOidRanges(fBC.OidRangesList[i], &hits, &miss);
//cout << "-- ReadTest " << fBC.OidRangesList[i].oid << " h: " << fBC.OidRangesList[i].hwm << "/" << rc << endl;
readBlocks+=rc;
rc=0;
}
hitsTot+=hits;
missTot+=miss;
hits=0;
miss=0;
}
void operator()()
{
uint32_t rc = 0;
uint32_t readBlocks = 0;
uint32_t hits = 0;
uint32_t miss = 0;
uint32_t hitsTot = 0;
uint32_t missTot = 0;
struct timespec tm1;
struct timespec tm2;
tm1.tv_sec = 0;
tm1.tv_nsec = 0;
tm2.tv_sec = 0;
tm2.tv_nsec = 0;
double tm3 = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
clock_gettime(CLOCK_REALTIME, &tm2);
timespec_sub(tm1, tm2, tm3);
for (uint32_t k = 0; k < fReps; k++)
{
for (uint32_t i = 0; i < lastRangeListIdx && i < fBC.OidRangesList.size() ; i++)
{
rc = fBC.ReadOidRanges(fBC.OidRangesList[i], &hits, &miss);
//cout << "-- ReadTest " << fBC.OidRangesList[i].oid << " h: " << fBC.OidRangesList[i].hwm << "/" << rc << endl;
readBlocks += rc;
rc = 0;
}
cout << "readtest rd: " << hitsTot << "/" << missTot
<< " sz: " << fBC.fCacheSz
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
hitsTot += hits;
missTot += miss;
hits = 0;
miss = 0;
}
} // operator()
clock_gettime(CLOCK_REALTIME, &tm2);
timespec_sub(tm1, tm2, tm3);
cout << "readtest rd: " << hitsTot << "/" << missTot
<< " sz: " << fBC.fCacheSz
<< " tm: " << right << setw(10) << fixed << tm3
<< endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
BCTest& fBC;
uint32_t fReps;
};
void usage()
{
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
}
//
int main(int argc, char* argv[]) {
int main(int argc, char* argv[])
{
int cacheSz=128; // K number of blocks
int thr=1;
int ra=1024;
int clients=1;
int reps=1;
int cacheSz = 128; // K number of blocks
int thr = 1;
int ra = 1024;
int clients = 1;
int reps = 1;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz=atoi(argv[1])*1024;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz = atoi(argv[1]) * 1024;
if (argc > 2 && atoi(argv[2]) > 0)
thr=atoi(argv[2]);
if (argc > 2 && atoi(argv[2]) > 0)
thr = atoi(argv[2]);
if (argc > 3 && atoi(argv[3]) > 0)
ra=atoi(argv[3]);
if (argc > 3 && atoi(argv[3]) > 0)
ra = atoi(argv[3]);
if (argc > 4 && atoi(argv[4]) > 0)
clients=atoi(argv[4]);
if (argc > 4 && atoi(argv[4]) > 0)
clients = atoi(argv[4]);
if (argc > 5 && atoi(argv[5]) > 0)
reps=atoi(argv[5]);
if (argc > 5 && atoi(argv[5]) > 0)
reps = atoi(argv[5]);
BCTest bc(cacheSz, thr, ra);
BCTest bc(cacheSz, thr, ra);
cout <<
"Cache Size: " << cacheSz <<
" read Threads: " << thr <<
" read Ahead: " << ra <<
" clients: " << clients <<
" repetitions: " << reps <<
" max Blocks: " << bc.maxBlocksAvailable <<
endl;
cout <<
"Cache Size: " << cacheSz <<
" read Threads: " << thr <<
" read Ahead: " << ra <<
" clients: " << clients <<
" repetitions: " << reps <<
" max Blocks: " << bc.maxBlocksAvailable <<
endl;
// loader cache
struct loadThr loader1(bc, 1);
struct readThr reader1(bc, reps);
vector<boost::thread*> v;
// loader cache
struct loadThr loader1(bc, 1);
struct readThr reader1(bc, reps);
vector<boost::thread*> v;
boost::thread* th1 = new boost::thread(loader1);
th1->join();
boost::thread* th1 = new boost::thread(loader1);
th1->join();
for (int i=0; i<clients;i++) {
boost::thread* rd1 = new boost::thread(reader1);
v.push_back(rd1);
}
for (int i = 0; i < clients; i++)
{
boost::thread* rd1 = new boost::thread(reader1);
v.push_back(rd1);
}
for (int i=0; i<clients;i++) {
boost::thread* rd1 = v[i];
rd1->join();
delete rd1;
}
for (int i = 0; i < clients; i++)
{
boost::thread* rd1 = v[i];
rd1->join();
delete rd1;
}
v.clear();
v.clear();
delete th1;
delete th1;
return 0;
return 0;
} // end main