database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

clang format apply

Browse Source
This commit is contained in:
Leonid Fedorov
2022-01-21 16:43:49 +00:00
parent 6b6411229f
commit 04752ec546
1376 changed files with 393460 additions and 412662 deletions
Download Patch File Download Diff File Expand all files Collapse all files

928
utils/batchloader/batchloader.cpp
View File

File diff suppressed because it is too large Load Diff

153
utils/batchloader/batchloader.h
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: we_dbrootextenttracker.h 3672 2012-03-26 12:31:27Z rdempsey $
*/
* $Id: we_dbrootextenttracker.h 3672 2012-03-26 12:31:27Z rdempsey $
*/
/** @file we_dbrootextenttracker.h
* Contains classes to track the order of placement (rotation) of extents as
@ -33,99 +33,96 @@
namespace batchloader
{
//------------------------------------------------------------------------------
/** @brief Class to find the PM id to send a batch of row
*/
//------------------------------------------------------------------------------
class BatchLoader
{
public:
public:
/** @brief BatchLoader constructor
* @param tableOid table OID of interest.
* @param sessionId cpimport use 0 as session id.
* @param PMs vector Collection of PM ids.
*/
BatchLoader(uint32_t tableOid, execplan::CalpontSystemCatalog::SCN sessionId, std::vector<uint32_t>& PMs);
/** @brief BatchLoader constructor
* @param tableOid table OID of interest.
* @param sessionId cpimport use 0 as session id.
* @param PMs vector Collection of PM ids.
/**
* @brief select the Next PM where batch data to be distributed.
* return the PM where next batch to be send.
* if an error occurs, 0 will be returned
*/
uint32_t selectNextPM();
/**
* @brief Move to previous Sequence in the array.
* This can be used when we find that we cannot use the PM right now
* and want to use it later. for example the queue is full of the
* current PM and we want to get the same PM Id when we call selectNextPM()
* next time also.
*/
void reverseSequence();
/*
* @brief After calling selectFirstPM(), if we need to keep continuing to
* the next PM in the list, we need to call this. If we just want to start
* distributing from dbroot 1 onwards, no need to call this function.
*/
void prepareForSecondPM();
struct RootExtentsBlocks
{
/** @brief the dbroot
*/
BatchLoader ( uint32_t tableOid, execplan::CalpontSystemCatalog::SCN sessionId, std::vector<uint32_t>& PMs);
/**
* @brief select the Next PM where batch data to be distributed.
* return the PM where next batch to be send.
* if an error occurs, 0 will be returned
uint32_t DBRoot;
/** @brief the number of extents
*/
uint32_t selectNextPM();
/**
* @brief Move to previous Sequence in the array.
* This can be used when we find that we cannot use the PM right now
* and want to use it later. for example the queue is full of the
* current PM and we want to get the same PM Id when we call selectNextPM()
* next time also.
uint64_t numExtents;
/** @brief the number of blocks in the last partition
*/
void reverseSequence();
uint64_t numBlocks;
};
/*
* @brief After calling selectFirstPM(), if we need to keep continuing to
* the next PM in the list, we need to call this. If we just want to start
* distributing from dbroot 1 onwards, no need to call this function.
struct PMRootInfo
{
/** @brief the module id
*/
void prepareForSecondPM();
struct RootExtentsBlocks
{
/** @brief the dbroot
*/
uint32_t DBRoot;
/** @brief the number of extents
*/
uint64_t numExtents;
/** @brief the number of blocks in the last partition
*/
uint64_t numBlocks;
};
struct PMRootInfo
{
/** @brief the module id
*/
uint32_t PMId;
/** @brief the dbroot info
*/
std::vector<RootExtentsBlocks> rootInfo;
};
private:
/** @brief Select the first PM to send the first batch of rows.
* @param startFromNextPM - if true, don't use the PMId. Instead use the next one in the sorted dmList.
* @param PMId - The PM id to send the first batch of rows if startFromNextPM is false.
* @return Returns 0 if success, else returns error code.
uint32_t PMId;
/** @brief the dbroot info
*/
void selectFirstPM ( uint32_t& PMId);
std::vector<RootExtentsBlocks> rootInfo;
};
/** @brief build the batch distribution sequence in a vector
* return void
*/
void buildBatchDistSeqVector();
private:
/** @brief Select the first PM to send the first batch of rows.
* @param startFromNextPM - if true, don't use the PMId. Instead use the next one in the sorted dmList.
* @param PMId - The PM id to send the first batch of rows if startFromNextPM is false.
* @return Returns 0 if success, else returns error code.
*/
void selectFirstPM(uint32_t& PMId);
/** @brief build the batch distribution sequence in a vector
* return void
*/
void buildBatchDistSeqVector(uint32_t StartPm);
/** @brief build the batch distribution sequence in a vector
* return void
*/
void buildBatchDistSeqVector();
typedef std::vector<uint32_t> BlIntVec;
BlIntVec fPMs;
BlIntVec fDbRoots;
BlIntVec fPmDistSeq;
uint32_t fNextIdx;
uint32_t fFirstPm;
execplan::CalpontSystemCatalog::SCN fSessionId;
uint32_t fTableOid;
oam::OamCache::PMDbrootsMap_t fPmDbrootMap;
oam::OamCache::dbRootPMMap_t fDbrootPMmap;
/** @brief build the batch distribution sequence in a vector
* return void
*/
void buildBatchDistSeqVector(uint32_t StartPm);
typedef std::vector<uint32_t> BlIntVec;
BlIntVec fPMs;
BlIntVec fDbRoots;
BlIntVec fPmDistSeq;
uint32_t fNextIdx;
uint32_t fFirstPm;
execplan::CalpontSystemCatalog::SCN fSessionId;
uint32_t fTableOid;
oam::OamCache::PMDbrootsMap_t fPmDbrootMap;
oam::OamCache::dbRootPMMap_t fDbrootPMmap;
};
} //end of namespace
} // namespace batchloader
#undef EXPORT

406
utils/cacheutils/cacheutils.cpp
View File

@ -50,123 +50,126 @@ using namespace BRM;
namespace
{
//Only one of the cacheutils fcns can run at a time
// Only one of the cacheutils fcns can run at a time
boost::mutex CacheOpsMutex;
//This global is updated only w/ atomic ops
// This global is updated only w/ atomic ops
volatile uint32_t MultiReturnCode;
int32_t extractRespCode(const ByteStream& bs)
{
if (bs.length() < (sizeof(ISMPacketHeader) + sizeof(int32_t)))
return 1;
if (bs.length() < (sizeof(ISMPacketHeader) + sizeof(int32_t)))
return 1;
const uint8_t* bytePtr = bs.buf();
const ISMPacketHeader* hdrp = reinterpret_cast<const ISMPacketHeader*>(bytePtr);
const uint8_t* bytePtr = bs.buf();
const ISMPacketHeader* hdrp = reinterpret_cast<const ISMPacketHeader*>(bytePtr);
if (hdrp->Command != CACHE_OP_RESULTS)
return 1;
if (hdrp->Command != CACHE_OP_RESULTS)
return 1;
const int32_t* resp = reinterpret_cast<const int32_t*>(bytePtr + sizeof(ISMPacketHeader));
return *resp;
const int32_t* resp = reinterpret_cast<const int32_t*>(bytePtr + sizeof(ISMPacketHeader));
return *resp;
}
class CacheOpThread
{
public:
CacheOpThread(const string& svr, const ByteStream& outBs) : fServerName(svr), fOutBs(outBs) {}
~CacheOpThread() {}
void operator()()
public:
CacheOpThread(const string& svr, const ByteStream& outBs) : fServerName(svr), fOutBs(outBs)
{
}
~CacheOpThread()
{
}
void operator()()
{
struct timespec ts = {10, 0};
int32_t rc = 0;
scoped_ptr<MessageQueueClient> cl(new MessageQueueClient(fServerName));
try
{
struct timespec ts = { 10, 0 };
int32_t rc = 0;
scoped_ptr<MessageQueueClient> cl(new MessageQueueClient(fServerName));
try
{
cl->write(fOutBs);
rc = extractRespCode(cl->read(&ts));
}
catch (...)
{
rc = 1;
}
if (rc != 0)
atomicops::atomicCAS<uint32_t>(&MultiReturnCode, 0, 1);
cl->write(fOutBs);
rc = extractRespCode(cl->read(&ts));
}
catch (...)
{
rc = 1;
}
private:
//CacheOpThread(const CacheOpThread& rhs);
//CacheOpThread& operator=(const CacheOpThread& rhs);
if (rc != 0)
atomicops::atomicCAS<uint32_t>(&MultiReturnCode, 0, 1);
}
string fServerName;
ByteStream fOutBs;
private:
// CacheOpThread(const CacheOpThread& rhs);
// CacheOpThread& operator=(const CacheOpThread& rhs);
string fServerName;
ByteStream fOutBs;
};
int sendToAll(const ByteStream& outBs)
{
//Not thread-safe: external synchronization is needed!
// Not thread-safe: external synchronization is needed!
// added code here to flush any running primprocs that may be active
// TODO: we really only need to flush each unique PrimProc, but we can't tell from the
// config file which those are, so use the same logic as joblist::DistributedEngineComm
Config* cf = Config::makeConfig();
// added code here to flush any running primprocs that may be active
// TODO: we really only need to flush each unique PrimProc, but we can't tell from the
// config file which those are, so use the same logic as joblist::DistributedEngineComm
Config* cf = Config::makeConfig();
const string section = "PrimitiveServers";
int cnt = static_cast<int>(Config::fromText(cf->getConfig(section, "Count")));
const string section = "PrimitiveServers";
int cnt = static_cast<int>(Config::fromText(cf->getConfig(section, "Count")));
if (cnt <= 0) cnt = 1;
if (cnt <= 0)
cnt = 1;
thread_group tg;
int rc = 0;
MultiReturnCode = 0;
thread_group tg;
int rc = 0;
MultiReturnCode = 0;
for (int i = 0; i < cnt; i++)
{
ostringstream oss;
oss << "PMS" << (i + 1);
tg.create_thread(CacheOpThread(oss.str(), outBs));
}
for (int i = 0; i < cnt; i++)
{
ostringstream oss;
oss << "PMS" << (i + 1);
tg.create_thread(CacheOpThread(oss.str(), outBs));
}
tg.join_all();
tg.join_all();
if (MultiReturnCode != 0)
rc = -1;
if (MultiReturnCode != 0)
rc = -1;
return rc;
return rc;
}
}
} // namespace
namespace cacheutils
{
/**
*
*/
int flushPrimProcCache()
{
boost::mutex::scoped_lock lk(CacheOpsMutex);
boost::mutex::scoped_lock lk(CacheOpsMutex);
try
{
const int msgsize = sizeof(ISMPacketHeader);
uint8_t msgbuf[msgsize];
memset(msgbuf, 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(&msgbuf[0]);
hdrp->Command = CACHE_FLUSH;
try
{
const int msgsize = sizeof(ISMPacketHeader);
uint8_t msgbuf[msgsize];
memset(msgbuf, 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(&msgbuf[0]);
hdrp->Command = CACHE_FLUSH;
ByteStream bs(msgbuf, msgsize);
int rc = sendToAll(bs);
return rc;
}
catch (...)
{
}
ByteStream bs(msgbuf, msgsize);
int rc = sendToAll(bs);
return rc;
}
catch (...)
{
}
return -1;
return -1;
}
/**
@ -174,180 +177,181 @@ int flushPrimProcCache()
*/
int flushPrimProcBlocks(const BRM::BlockList_t& list)
{
if (list.empty()) return 0;
if (list.empty())
return 0;
boost::mutex::scoped_lock lk(CacheOpsMutex);
boost::mutex::scoped_lock lk(CacheOpsMutex);
#if defined(__LP64__) || defined(_WIN64)
if (list.size() > numeric_limits<uint32_t>::max()) return -1;
if (list.size() > numeric_limits<uint32_t>::max())
return -1;
#endif
try
{
const size_t msgsize = sizeof(ISMPacketHeader) + sizeof(uint32_t) + sizeof(LbidAtVer) * list.size();
scoped_array<uint8_t> msgbuf(new uint8_t[msgsize]);
memset(msgbuf.get(), 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(msgbuf.get());
hdrp->Command = CACHE_CLEAN_VSS;
uint32_t* cntp = reinterpret_cast<uint32_t*>(msgbuf.get() + sizeof(ISMPacketHeader));
*cntp = static_cast<uint32_t>(list.size());
LbidAtVer* itemp = reinterpret_cast<LbidAtVer*>(msgbuf.get() + sizeof(ISMPacketHeader) + sizeof(uint32_t));
BlockList_t::const_iterator iter = list.begin();
BlockList_t::const_iterator end = list.end();
try
{
const size_t msgsize = sizeof(ISMPacketHeader) + sizeof(uint32_t) + sizeof(LbidAtVer) * list.size();
scoped_array<uint8_t> msgbuf(new uint8_t[msgsize]);
memset(msgbuf.get(), 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(msgbuf.get());
hdrp->Command = CACHE_CLEAN_VSS;
uint32_t* cntp = reinterpret_cast<uint32_t*>(msgbuf.get() + sizeof(ISMPacketHeader));
*cntp = static_cast<uint32_t>(list.size());
LbidAtVer* itemp =
reinterpret_cast<LbidAtVer*>(msgbuf.get() + sizeof(ISMPacketHeader) + sizeof(uint32_t));
BlockList_t::const_iterator iter = list.begin();
BlockList_t::const_iterator end = list.end();
while (iter != end)
{
itemp->LBID = static_cast<uint64_t>(iter->first);
itemp->Ver = static_cast<uint32_t>(iter->second);
++itemp;
++iter;
}
ByteStream bs(msgbuf.get(), msgsize);
int rc = sendToAll(bs);
return rc;
}
catch (...)
while (iter != end)
{
itemp->LBID = static_cast<uint64_t>(iter->first);
itemp->Ver = static_cast<uint32_t>(iter->second);
++itemp;
++iter;
}
return -1;
ByteStream bs(msgbuf.get(), msgsize);
int rc = sendToAll(bs);
return rc;
}
catch (...)
{
}
return -1;
}
int flushPrimProcAllverBlocks(const vector<LBID_t>& list)
{
if (list.empty()) return 0;
if (list.empty())
return 0;
ByteStream bs(sizeof(ISMPacketHeader) + sizeof(uint32_t) + (sizeof(LBID_t) * list.size()));
ISMPacketHeader* hdr;
int rc;
ByteStream bs(sizeof(ISMPacketHeader) + sizeof(uint32_t) + (sizeof(LBID_t) * list.size()));
ISMPacketHeader* hdr;
int rc;
hdr = (ISMPacketHeader*) bs.getInputPtr();
hdr->Command = FLUSH_ALL_VERSION;
bs.advanceInputPtr(sizeof(ISMPacketHeader));
bs << (uint32_t) list.size();
bs.append((uint8_t*) &list[0], sizeof(LBID_t) * list.size());
hdr = (ISMPacketHeader*)bs.getInputPtr();
hdr->Command = FLUSH_ALL_VERSION;
bs.advanceInputPtr(sizeof(ISMPacketHeader));
bs << (uint32_t)list.size();
bs.append((uint8_t*)&list[0], sizeof(LBID_t) * list.size());
try
{
boost::mutex::scoped_lock lk(CacheOpsMutex);
rc = sendToAll(bs);
return rc;
}
catch (...)
{
}
try
{
boost::mutex::scoped_lock lk(CacheOpsMutex);
rc = sendToAll(bs);
return rc;
}
catch (...)
{
}
return -1;
return -1;
}
int flushOIDsFromCache(const vector<BRM::OID_t>& oids)
{
/* Message format:
* ISMPacketHeader
* uint32_t - OID count
* uint32_t * - OID array
*/
/* Message format:
* ISMPacketHeader
* uint32_t - OID count
* uint32_t * - OID array
*/
boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t());
boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t());
ByteStream bs;
ISMPacketHeader ism;
uint32_t i;
ByteStream bs;
ISMPacketHeader ism;
uint32_t i;
void *ismp = static_cast<void*>(&ism);
memset(ismp, 0, sizeof(ISMPacketHeader));
ism.Command = CACHE_FLUSH_BY_OID;
bs.load((uint8_t*) &ism, sizeof(ISMPacketHeader));
bs << (uint32_t) oids.size();
void* ismp = static_cast<void*>(&ism);
memset(ismp, 0, sizeof(ISMPacketHeader));
ism.Command = CACHE_FLUSH_BY_OID;
bs.load((uint8_t*)&ism, sizeof(ISMPacketHeader));
bs << (uint32_t)oids.size();
for (i = 0; i < oids.size(); i++)
bs << (uint32_t) oids[i];
for (i = 0; i < oids.size(); i++)
bs << (uint32_t)oids[i];
lk.lock();
return sendToAll(bs);
lk.lock();
return sendToAll(bs);
}
int flushPartition(const std::vector<BRM::OID_t>& oids, set<BRM::LogicalPartition>& partitionNums)
{
/* Message format:
* ISMPacketHeader
* uint32_t - partition count
* LogicalPartition * - partitionNum
* uint32_t - OID count
* uint32_t * - OID array
*/
/* Message format:
* ISMPacketHeader
* uint32_t - partition count
* LogicalPartition * - partitionNum
* uint32_t - OID count
* uint32_t * - OID array
*/
boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t());
boost::mutex::scoped_lock lk(CacheOpsMutex, boost::defer_lock_t());
ByteStream bs;
ISMPacketHeader ism;
ByteStream bs;
ISMPacketHeader ism;
void *ismp = static_cast<void*>(&ism);
memset(ismp, 0, sizeof(ISMPacketHeader));
ism.Command = CACHE_FLUSH_PARTITION;
bs.load((uint8_t*) &ism, sizeof(ISMPacketHeader));
serializeSet<BRM::LogicalPartition>(bs, partitionNums);
serializeInlineVector<BRM::OID_t>(bs, oids);
void* ismp = static_cast<void*>(&ism);
memset(ismp, 0, sizeof(ISMPacketHeader));
ism.Command = CACHE_FLUSH_PARTITION;
bs.load((uint8_t*)&ism, sizeof(ISMPacketHeader));
serializeSet<BRM::LogicalPartition>(bs, partitionNums);
serializeInlineVector<BRM::OID_t>(bs, oids);
lk.lock();
return sendToAll(bs);
lk.lock();
return sendToAll(bs);
}
int dropPrimProcFdCache()
{
const int msgsize = sizeof(ISMPacketHeader);
uint8_t msgbuf[msgsize];
memset(msgbuf, 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(&msgbuf[0]);
hdrp->Command = CACHE_DROP_FDS;
ByteStream bs(msgbuf, msgsize);
const int msgsize = sizeof(ISMPacketHeader);
uint8_t msgbuf[msgsize];
memset(msgbuf, 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(&msgbuf[0]);
hdrp->Command = CACHE_DROP_FDS;
ByteStream bs(msgbuf, msgsize);
try
{
boost::mutex::scoped_lock lk(CacheOpsMutex);
int rc = sendToAll(bs);
return rc;
}
catch (...)
{
}
try
{
boost::mutex::scoped_lock lk(CacheOpsMutex);
int rc = sendToAll(bs);
return rc;
}
catch (...)
{
}
return -1;
return -1;
}
int purgePrimProcFdCache(const std::vector<BRM::FileInfo> files, const int pmId)
{
const int msgsize = sizeof(ISMPacketHeader);
uint8_t msgbuf[msgsize];
memset(msgbuf, 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(&msgbuf[0]);
hdrp->Command = CACHE_PURGE_FDS;
ByteStream bs(msgbuf, msgsize);
serializeInlineVector<FileInfo>(bs, files);
int32_t rc = 0;
const int msgsize = sizeof(ISMPacketHeader);
uint8_t msgbuf[msgsize];
memset(msgbuf, 0, sizeof(ISMPacketHeader));
ISMPacketHeader* hdrp = reinterpret_cast<ISMPacketHeader*>(&msgbuf[0]);
hdrp->Command = CACHE_PURGE_FDS;
ByteStream bs(msgbuf, msgsize);
serializeInlineVector<FileInfo>(bs, files);
int32_t rc = 0;
try
{
struct timespec ts = { 10, 0 };
ostringstream oss;
oss << "PMS" << pmId;
scoped_ptr<MessageQueueClient> cl(new MessageQueueClient(oss.str()));
cl->write(bs);
rc = extractRespCode(cl->read(&ts));
}
catch (...)
{
rc = -1;
}
try
{
struct timespec ts = {10, 0};
ostringstream oss;
oss << "PMS" << pmId;
scoped_ptr<MessageQueueClient> cl(new MessageQueueClient(oss.str()));
cl->write(bs);
rc = extractRespCode(cl->read(&ts));
}
catch (...)
{
rc = -1;
}
return rc;
}
return rc;
}
} // namespace cacheutils
// vim:ts=4 sw=4:

4
utils/cacheutils/cacheutils.h
View File

@ -71,7 +71,5 @@ int dropPrimProcFdCache();
*/
int purgePrimProcFdCache(const std::vector<BRM::FileInfo> files, const int pmId);
}
} // namespace cacheutils
// vim:ts=4 sw=4:

360
utils/cloudio/SMComm.cpp
View File

@ -23,258 +23,258 @@ using namespace messageqcpp;
namespace
{
idbdatafile::SMComm *instance = NULL;
idbdatafile::SMComm* instance = NULL;
boost::mutex m;
};
}; // namespace
namespace idbdatafile
{
SMComm * SMComm::get()
SMComm* SMComm::get()
{
if (instance)
return instance;
boost::mutex::scoped_lock sl(m);
if (instance)
return instance;
instance = new SMComm();
if (instance)
return instance;
boost::mutex::scoped_lock sl(m);
if (instance)
return instance;
instance = new SMComm();
return instance;
}
// timesavers
#define common_exit(bs1, bs2, retCode) \
{ \
int l_errno = errno; \
buffers.returnByteStream(bs1); \
buffers.returnByteStream(bs2); \
errno = l_errno; \
return retCode; \
}
{ \
int l_errno = errno; \
buffers.returnByteStream(bs1); \
buffers.returnByteStream(bs2); \
errno = l_errno; \
return retCode; \
}
// bs1 is the bytestream ptr with the command to SMComm.
// bs2 is the bytestream pointer with the response from SMComm.
// retCode is the var to store the return code in from the msg.
// returns with the output pointer at the fcn-specific data
#define check_for_error(bs1, bs2, retCode) \
{ \
int l_errno; \
*bs2 >> retCode; \
if (retCode < 0) \
{ \
*bs2 >> l_errno; \
errno = l_errno; \
common_exit(bs1, bs2, retCode); \
} \
else \
errno = 0; \
}
{ \
int l_errno; \
*bs2 >> retCode; \
if (retCode < 0) \
{ \
*bs2 >> l_errno; \
errno = l_errno; \
common_exit(bs1, bs2, retCode); \
} \
else \
errno = 0; \
}
SMComm::SMComm()
{
char buf[4096];
cwd = ::getcwd(buf, 4096);
char buf[4096];
cwd = ::getcwd(buf, 4096);
}
SMComm::~SMComm()
{
}
string SMComm::getAbsFilename(const string &filename)
string SMComm::getAbsFilename(const string& filename)
{
if (filename[0] == '/')
return filename;
else
return cwd + '/' + filename;
if (filename[0] == '/')
return filename;
else
return cwd + '/' + filename;
}
int SMComm::open(const string &filename, const int mode, struct stat *statbuf)
int SMComm::open(const string& filename, const int mode, struct stat* statbuf)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t) storagemanager::OPEN << mode << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
memcpy(statbuf, response->buf(), sizeof(*statbuf));
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t)storagemanager::OPEN << mode << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
memcpy(statbuf, response->buf(), sizeof(*statbuf));
common_exit(command, response, err);
}
ssize_t SMComm::pread(const string &filename, void *buf, const size_t count, const off_t offset)
ssize_t SMComm::pread(const string& filename, void* buf, const size_t count, const off_t offset)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t) storagemanager::READ << count << offset << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
memcpy(buf, response->buf(), err);
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t)storagemanager::READ << count << offset << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
memcpy(buf, response->buf(), err);
common_exit(command, response, err);
}
ssize_t SMComm::pwrite(const string &filename, const void *buf, const size_t count, const off_t offset)
ssize_t SMComm::pwrite(const string& filename, const void* buf, const size_t count, const off_t offset)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t) storagemanager::WRITE << count << offset << absfilename;
command->needAtLeast(count);
uint8_t *cmdBuf = command->getInputPtr();
memcpy(cmdBuf, buf, count);
command->advanceInputPtr(count);
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t)storagemanager::WRITE << count << offset << absfilename;
command->needAtLeast(count);
uint8_t* cmdBuf = command->getInputPtr();
memcpy(cmdBuf, buf, count);
command->advanceInputPtr(count);
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
common_exit(command, response, err);
}
ssize_t SMComm::append(const string &filename, const void *buf, const size_t count)
ssize_t SMComm::append(const string& filename, const void* buf, const size_t count)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t) storagemanager::APPEND << count << absfilename;
command->needAtLeast(count);
uint8_t *cmdBuf = command->getInputPtr();
memcpy(cmdBuf, buf, count);
command->advanceInputPtr(count);
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
*command << (uint8_t)storagemanager::APPEND << count << absfilename;
command->needAtLeast(count);
uint8_t* cmdBuf = command->getInputPtr();
memcpy(cmdBuf, buf, count);
command->advanceInputPtr(count);
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
common_exit(command, response, err);
}
int SMComm::unlink(const string &filename)
int SMComm::unlink(const string& filename)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t) storagemanager::UNLINK << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t)storagemanager::UNLINK << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
common_exit(command, response, err);
}
int SMComm::stat(const string &filename, struct stat *statbuf)
int SMComm::stat(const string& filename, struct stat* statbuf)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t) storagemanager::STAT << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
memcpy(statbuf, response->buf(), sizeof(*statbuf));
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t)storagemanager::STAT << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
memcpy(statbuf, response->buf(), sizeof(*statbuf));
common_exit(command, response, err);
}
int SMComm::truncate(const string &filename, const off64_t length)
int SMComm::truncate(const string& filename, const off64_t length)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t) storagemanager::TRUNCATE << length << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename(getAbsFilename(filename));
*command << (uint8_t)storagemanager::TRUNCATE << length << absfilename;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
common_exit(command, response, err);
}
int SMComm::listDirectory(const string &path, list<string> *entries)
int SMComm::listDirectory(const string& path, list<string>* entries)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string abspath(getAbsFilename(path));
*command << (uint8_t) storagemanager::LIST_DIRECTORY << abspath;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
uint32_t numElements;
string stmp;
entries->clear();
*response >> numElements;
while (numElements > 0) {
*response >> stmp;
entries->push_back(stmp);
numElements--;
}
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string abspath(getAbsFilename(path));
*command << (uint8_t)storagemanager::LIST_DIRECTORY << abspath;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
uint32_t numElements;
string stmp;
entries->clear();
*response >> numElements;
while (numElements > 0)
{
*response >> stmp;
entries->push_back(stmp);
numElements--;
}
common_exit(command, response, err);
}
int SMComm::ping()
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
*command << (uint8_t) storagemanager::PING;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
*command << (uint8_t)storagemanager::PING;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
common_exit(command, response, err);
}
int SMComm::sync()
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
*command << (uint8_t) storagemanager::SYNC;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
*command << (uint8_t)storagemanager::SYNC;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
common_exit(command, response, err);
}
int SMComm::copyFile(const string &file1, const string &file2)
int SMComm::copyFile(const string& file1, const string& file2)
{
ByteStream *command = buffers.getByteStream();
ByteStream *response = buffers.getByteStream();
ssize_t err;
string absfilename1(getAbsFilename(file1));
string absfilename2(getAbsFilename(file2));
*command << (uint8_t) storagemanager::COPY << absfilename1 << absfilename2;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
ByteStream* command = buffers.getByteStream();
ByteStream* response = buffers.getByteStream();
ssize_t err;
string absfilename1(getAbsFilename(file1));
string absfilename2(getAbsFilename(file2));
*command << (uint8_t)storagemanager::COPY << absfilename1 << absfilename2;
err = sockets.send_recv(*command, response);
if (err)
common_exit(command, response, err);
check_for_error(command, response, err);
common_exit(command, response, err);
}
}
} // namespace idbdatafile

92
utils/cloudio/SMComm.h
View File

@ -23,56 +23,54 @@
#include "bytestream.h"
#include "bytestreampool.h"
namespace idbdatafile
namespace idbdatafile
{
class SMComm : public boost::noncopyable
{
public:
// This is a singleton. Get it with get()
static SMComm *get();
/* Open currently returns a stat struct so SMDataFile can set its initial position, otherwise
behaves how you'd think. */
int open(const std::string &filename, const int mode, struct stat *statbuf);
ssize_t pread(const std::string &filename, void *buf, const size_t count, const off_t offset);
ssize_t pwrite(const std::string &filename, const void *buf, const size_t count, const off_t offset);
/* append exists for cases where the file is open in append mode. A normal write won't work
because the file position/size may be out of date if there are multiple writers. */
ssize_t append(const std::string &filename, const void *buf, const size_t count);
int unlink(const std::string &filename);
int stat(const std::string &filename, struct stat *statbuf);
// added this one because it should be trivial to implement in SM, and prevents a large
// operation in SMDataFile.
int truncate(const std::string &filename, const off64_t length);
int listDirectory(const std::string &path, std::list<std::string> *entries);
// health indicator. 0 = processes are talking to each other and SM has read/write access to
// the specified S3 bucket. Need to define specific error codes.
int ping();
int sync();
public:
// This is a singleton. Get it with get()
static SMComm* get();
int copyFile(const std::string &file1, const std::string &file2);
virtual ~SMComm();
private:
SMComm();
std::string getAbsFilename(const std::string &filename);
SocketPool sockets;
messageqcpp::ByteStreamPool buffers;
std::string cwd;
/* Open currently returns a stat struct so SMDataFile can set its initial position, otherwise
behaves how you'd think. */
int open(const std::string& filename, const int mode, struct stat* statbuf);
ssize_t pread(const std::string& filename, void* buf, const size_t count, const off_t offset);
ssize_t pwrite(const std::string& filename, const void* buf, const size_t count, const off_t offset);
/* append exists for cases where the file is open in append mode. A normal write won't work
because the file position/size may be out of date if there are multiple writers. */
ssize_t append(const std::string& filename, const void* buf, const size_t count);
int unlink(const std::string& filename);
int stat(const std::string& filename, struct stat* statbuf);
// added this one because it should be trivial to implement in SM, and prevents a large
// operation in SMDataFile.
int truncate(const std::string& filename, const off64_t length);
int listDirectory(const std::string& path, std::list<std::string>* entries);
// health indicator. 0 = processes are talking to each other and SM has read/write access to
// the specified S3 bucket. Need to define specific error codes.
int ping();
int sync();
int copyFile(const std::string& file1, const std::string& file2);
virtual ~SMComm();
private:
SMComm();
std::string getAbsFilename(const std::string& filename);
SocketPool sockets;
messageqcpp::ByteStreamPool buffers;
std::string cwd;
};
}
} // namespace idbdatafile

140
utils/cloudio/SMDataFile.cpp
View File

@ -14,7 +14,7 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <fcntl.h>
#include "SMDataFile.h"
@ -23,131 +23,123 @@ using namespace std;
namespace idbdatafile
{
SMDataFile::~SMDataFile()
{
}
SMDataFile::SMDataFile(const char *name, int _openmode, const struct stat &_stat) :
IDBDataFile(name)
SMDataFile::SMDataFile(const char* name, int _openmode, const struct stat& _stat) : IDBDataFile(name)
{
openmode = _openmode;
// the 'a' file open mode is the only one that starts at EOF
if ((openmode & O_APPEND) && !(openmode & O_RDWR))
position = _stat.st_size;
else
position = 0;
comm = SMComm::get();
openmode = _openmode;
// the 'a' file open mode is the only one that starts at EOF
if ((openmode & O_APPEND) && !(openmode & O_RDWR))
position = _stat.st_size;
else
position = 0;
comm = SMComm::get();
}
ssize_t SMDataFile::pread(void *buf, off64_t offset, size_t count)
ssize_t SMDataFile::pread(void* buf, off64_t offset, size_t count)
{
return comm->pread(name(), buf, count, offset);
return comm->pread(name(), buf, count, offset);
}
ssize_t SMDataFile::read(void *buf, size_t count)
ssize_t SMDataFile::read(void* buf, size_t count)
{
ssize_t ret = comm->pread(name(), buf, count, position);
if (ret < 0)
return ret;
position += ret;
ssize_t ret = comm->pread(name(), buf, count, position);
if (ret < 0)
return ret;
position += ret;
return ret;
}
ssize_t SMDataFile::write(const void *buf, size_t count)
ssize_t SMDataFile::write(const void* buf, size_t count)
{
if (openmode & O_APPEND)
return comm->append(name(), buf, count);
ssize_t ret = comm->pwrite(name(), buf, count, position);
if (ret < 0)
return ret;
position += ret;
if (openmode & O_APPEND)
return comm->append(name(), buf, count);
ssize_t ret = comm->pwrite(name(), buf, count, position);
if (ret < 0)
return ret;
position += ret;
return ret;
}
int SMDataFile::seek(off64_t offset, int whence)
{
switch (whence) {
case SEEK_SET:
position = offset;
break;
case SEEK_CUR:
position += offset;
break;
case SEEK_END:
{
struct stat _stat;
int err = comm->stat(name(), &_stat);
if (err)
return err;
position = _stat.st_size + offset;
break;
}
default:
errno = EINVAL;
return -1;
switch (whence)
{
case SEEK_SET: position = offset; break;
case SEEK_CUR: position += offset; break;
case SEEK_END:
{
struct stat _stat;
int err = comm->stat(name(), &_stat);
if (err)
return err;
position = _stat.st_size + offset;
break;
}
return 0;
default: errno = EINVAL; return -1;
}
return 0;
}
int SMDataFile::truncate(off64_t length)
{
return comm->truncate(name(), length);
return comm->truncate(name(), length);
}
int SMDataFile::fallocate(int mode, off64_t offset, off64_t length)
{
idbassert_s(mode == 0, "SMDataFile::fallocate() does not support mode != 0 right now.");
return comm->truncate(name(), offset + length);
idbassert_s(mode == 0, "SMDataFile::fallocate() does not support mode != 0 right now.");
return comm->truncate(name(), offset + length);
}
off64_t SMDataFile::size()
{
struct stat _stat;
int err = comm->stat(name(), &_stat);
if (err)
return err;
return _stat.st_size;
struct stat _stat;
int err = comm->stat(name(), &_stat);
if (err)
return err;
return _stat.st_size;
}
off64_t SMDataFile::tell()
{
return position;
return position;
}
int SMDataFile::flush()
{
return 0; // writes are synchronous b/c of replication. If we allow asynchronous replication,
// then we need to implement a flush() cmd in SMComm.
return 0; // writes are synchronous b/c of replication. If we allow asynchronous replication,
// then we need to implement a flush() cmd in SMComm.
}
time_t SMDataFile::mtime()
{
struct stat _stat;
int err = comm->stat(name(), &_stat);
if (err)
return (time_t) err;
return _stat.st_mtime;
struct stat _stat;
int err = comm->stat(name(), &_stat);
if (err)
return (time_t)err;
return _stat.st_mtime;
}
int SMDataFile::close()
{
return 0;
return 0;
}
// constructor used for testing
SMDataFile::SMDataFile(const char *fname, int _openmode, size_t fake_size)
: IDBDataFile(fname)
SMDataFile::SMDataFile(const char* fname, int _openmode, size_t fake_size) : IDBDataFile(fname)
{
openmode = _openmode;
// the 'a' file open mode is the only one that starts at EOF
if ((openmode & O_APPEND) && !(openmode & O_RDWR))
position = fake_size;
else
position = 0;
comm = SMComm::get();
openmode = _openmode;
// the 'a' file open mode is the only one that starts at EOF
if ((openmode & O_APPEND) && !(openmode & O_RDWR))
position = fake_size;
else
position = 0;
comm = SMComm::get();
}
}
} // namespace idbdatafile

58
utils/cloudio/SMDataFile.h
View File

@ -14,7 +14,7 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <string>
@ -22,38 +22,36 @@
#include "IDBDataFile.h"
#include "SMComm.h"
namespace idbdatafile
namespace idbdatafile
{
class SMDataFile : public IDBDataFile
{
public:
virtual ~SMDataFile();
public:
virtual ~SMDataFile();
ssize_t pread(void* ptr, off64_t offset, size_t count);
ssize_t read(void* ptr, size_t count);
ssize_t write(const void* ptr, size_t count);
int seek(off64_t offset, int whence);
int truncate(off64_t length);
int fallocate(int mode, off64_t offset, off64_t length);
off64_t size();
off64_t tell();
int flush();
time_t mtime();
int close();
// for testing only
SMDataFile(const char *fname, int openmode, size_t fake_size);
private:
SMDataFile();
SMDataFile(const char *fname, int openmode, const struct stat &);
off64_t position;
int openmode;
SMComm *comm;
friend class SMFileFactory;
ssize_t pread(void* ptr, off64_t offset, size_t count);
ssize_t read(void* ptr, size_t count);
ssize_t write(const void* ptr, size_t count);
int seek(off64_t offset, int whence);
int truncate(off64_t length);
int fallocate(int mode, off64_t offset, off64_t length);
off64_t size();
off64_t tell();
int flush();
time_t mtime();
int close();
// for testing only
SMDataFile(const char* fname, int openmode, size_t fake_size);
private:
SMDataFile();
SMDataFile(const char* fname, int openmode, const struct stat&);
off64_t position;
int openmode;
SMComm* comm;
friend class SMFileFactory;
};
}
} // namespace idbdatafile

153
utils/cloudio/SMFileFactory.cpp
View File

@ -14,7 +14,7 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <sys/stat.h>
#include <fcntl.h>
#include "SMFileFactory.h"
@ -25,83 +25,82 @@
using namespace std;
namespace idbdatafile {
IDBDataFile* SMFileFactory::open(const char *filename, const char *mode, unsigned opts, unsigned colWidth)
namespace idbdatafile
{
// TODO, test whether this breaks anything.
//if (opts & IDBDataFile::USE_TMPFILE)
// return new BufferedFile(filename, mode, opts);
bool _read = false;
bool _write = false;
bool create = false;
bool truncate = false;
bool append = false;
IDBDataFile* SMFileFactory::open(const char* filename, const char* mode, unsigned opts, unsigned colWidth)
{
// TODO, test whether this breaks anything.
// if (opts & IDBDataFile::USE_TMPFILE)
// return new BufferedFile(filename, mode, opts);
// strip 'b' chars from mode
char newmode[8] = {'\0'}; // there'd better not be 7 chars in the mode string
int i = 0;
for (const char *c = mode; *c != '\0' && i < 8; c++)
if (*c != 'b')
newmode[i++] = *c;
if (i == 8) {
errno = EINVAL;
return NULL;
}
// parse the new mode string
if (newmode[0] == 'r')
{
_read = true;
if (newmode[1] == '+')
_write = true;
}
else if (newmode[0] == 'w')
{
_write = true;
truncate = true;
create = true;
if (newmode[1] == '+')
_read = true;
}
else if (newmode[0] == 'a')
{
_write = true;
create = true;
append = true;
if (newmode[1] == '+')
_read = true;
}
else
{
errno = EINVAL;
return NULL;
}
// turn newmode into posix flags
uint posix_flags = 0;
if (_read && _write)
posix_flags |= O_RDWR;
else if (_read)
posix_flags |= O_RDONLY;
else if (_write)
posix_flags |= O_WRONLY;
posix_flags |= (create ? O_CREAT : 0);
posix_flags |= (truncate ? O_TRUNC : 0);
posix_flags |= (append ? O_APPEND : 0);
SMComm *comm = SMComm::get();
struct stat _stat;
int err = comm->open(filename, posix_flags, &_stat);
if (err)
return NULL;
SMDataFile *ret = new SMDataFile(filename, posix_flags, _stat);
return ret;
bool _read = false;
bool _write = false;
bool create = false;
bool truncate = false;
bool append = false;
// strip 'b' chars from mode
char newmode[8] = {'\0'}; // there'd better not be 7 chars in the mode string
int i = 0;
for (const char* c = mode; *c != '\0' && i < 8; c++)
if (*c != 'b')
newmode[i++] = *c;
if (i == 8)
{
errno = EINVAL;
return NULL;
}
// parse the new mode string
if (newmode[0] == 'r')
{
_read = true;
if (newmode[1] == '+')
_write = true;
}
else if (newmode[0] == 'w')
{
_write = true;
truncate = true;
create = true;
if (newmode[1] == '+')
_read = true;
}
else if (newmode[0] == 'a')
{
_write = true;
create = true;
append = true;
if (newmode[1] == '+')
_read = true;
}
else
{
errno = EINVAL;
return NULL;
}
// turn newmode into posix flags
uint posix_flags = 0;
if (_read && _write)
posix_flags |= O_RDWR;
else if (_read)
posix_flags |= O_RDONLY;
else if (_write)
posix_flags |= O_WRONLY;
posix_flags |= (create ? O_CREAT : 0);
posix_flags |= (truncate ? O_TRUNC : 0);
posix_flags |= (append ? O_APPEND : 0);
SMComm* comm = SMComm::get();
struct stat _stat;
int err = comm->open(filename, posix_flags, &_stat);
if (err)
return NULL;
SMDataFile* ret = new SMDataFile(filename, posix_flags, _stat);
return ret;
}
}
} // namespace idbdatafile

7
utils/cloudio/SMFileFactory.h
View File

@ -22,11 +22,10 @@
namespace idbdatafile
{
class SMFileFactory : public FileFactoryBase
{
public:
IDBDataFile * open(const char* fname, const char* mode, unsigned opts, unsigned colWidth);
public:
IDBDataFile* open(const char* fname, const char* mode, unsigned opts, unsigned colWidth);
};
}
} // namespace idbdatafile

113
utils/cloudio/SMFileSystem.cpp
View File

@ -24,95 +24,94 @@ using namespace std;
namespace idbdatafile
{
SMFileSystem::SMFileSystem() : IDBFileSystem(IDBFileSystem::CLOUD)
{
SMComm::get(); // get SMComm running
SMComm::get(); // get SMComm running
}
SMFileSystem::~SMFileSystem()
{
}
int SMFileSystem::mkdir(const char *path)
int SMFileSystem::mkdir(const char* path)
{
return 0;
return 0;
}
off64_t SMFileSystem::size(const char *filename) const
off64_t SMFileSystem::size(const char* filename) const
{
struct stat _stat;
SMComm *smComm = SMComm::get();
int err = smComm->stat(filename, &_stat);
if (err)
return err;
return _stat.st_size;
}
struct stat _stat;
off64_t SMFileSystem::compressedSize(const char *filename) const
{
// Yikes, punting on this one.
throw NotImplementedYet(__func__);
}
int SMFileSystem::remove(const char *filename)
{
SMComm *comm = SMComm::get();
return comm->unlink(filename);
}
int SMFileSystem::rename(const char *oldFile, const char *newFile)
{
int err = copyFile(oldFile, newFile);
if (err)
return err;
err = this->remove(oldFile);
SMComm* smComm = SMComm::get();
int err = smComm->stat(filename, &_stat);
if (err)
return err;
return _stat.st_size;
}
bool SMFileSystem::exists(const char *filename) const
off64_t SMFileSystem::compressedSize(const char* filename) const
{
struct stat _stat;
SMComm *comm = SMComm::get();
int err = comm->stat(filename, &_stat);
return (err == 0);
// Yikes, punting on this one.
throw NotImplementedYet(__func__);
}
int SMFileSystem::remove(const char* filename)
{
SMComm* comm = SMComm::get();
return comm->unlink(filename);
}
int SMFileSystem::rename(const char* oldFile, const char* newFile)
{
int err = copyFile(oldFile, newFile);
if (err)
return err;
err = this->remove(oldFile);
return err;
}
bool SMFileSystem::exists(const char* filename) const
{
struct stat _stat;
SMComm* comm = SMComm::get();
int err = comm->stat(filename, &_stat);
return (err == 0);
}
int SMFileSystem::listDirectory(const char* pathname, std::list<std::string>& contents) const
{
SMComm *comm = SMComm::get();
return comm->listDirectory(pathname, &contents);
SMComm* comm = SMComm::get();
return comm->listDirectory(pathname, &contents);
}
bool SMFileSystem::isDir(const char *path) const
bool SMFileSystem::isDir(const char* path) const
{
SMComm *comm = SMComm::get();
struct stat _stat;
int err = comm->stat(path, &_stat);
if (err != 0)
return false; // reasonable to throw here? todo, look at what the other classes do.
return (_stat.st_mode & S_IFDIR);
SMComm* comm = SMComm::get();
struct stat _stat;
int err = comm->stat(path, &_stat);
if (err != 0)
return false; // reasonable to throw here? todo, look at what the other classes do.
return (_stat.st_mode & S_IFDIR);
}
int SMFileSystem::copyFile(const char *src, const char *dest) const
int SMFileSystem::copyFile(const char* src, const char* dest) const
{
SMComm *comm = SMComm::get();
return comm->copyFile(src, dest);
SMComm* comm = SMComm::get();
return comm->copyFile(src, dest);
}
bool SMFileSystem::filesystemIsUp() const
{
SMComm *comm = SMComm::get();
return (comm->ping() == 0);
SMComm* comm = SMComm::get();
return (comm->ping() == 0);
}
bool SMFileSystem::filesystemSync() const
{
SMComm *comm = SMComm::get();
return (comm->sync() == 0);
}
SMComm* comm = SMComm::get();
return (comm->sync() == 0);
}
} // namespace idbdatafile

36
utils/cloudio/SMFileSystem.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <list>
@ -25,27 +24,24 @@
namespace idbdatafile
{
class SMFileSystem : public IDBFileSystem, boost::noncopyable
{
public:
SMFileSystem();
virtual ~SMFileSystem();
// why are some of these const and some not const in IDBFileSystem?
int mkdir(const char* pathname);
off64_t size(const char* path) const;
off64_t compressedSize(const char* path) const;
int remove(const char* pathname);
int rename(const char* oldpath, const char* newpath);
bool exists(const char* pathname) const;
int listDirectory(const char* pathname, std::list<std::string>& contents) const;
bool isDir(const char* pathname) const;
int copyFile(const char* srcPath, const char* destPath) const;
bool filesystemIsUp() const;
bool filesystemSync() const;
public:
SMFileSystem();
virtual ~SMFileSystem();
// why are some of these const and some not const in IDBFileSystem?
int mkdir(const char* pathname);
off64_t size(const char* path) const;
off64_t compressedSize(const char* path) const;
int remove(const char* pathname);
int rename(const char* oldpath, const char* newpath);
bool exists(const char* pathname) const;
int listDirectory(const char* pathname, std::list<std::string>& contents) const;
bool isDir(const char* pathname) const;
int copyFile(const char* srcPath, const char* destPath) const;
bool filesystemIsUp() const;
bool filesystemSync() const;
};
}
} // namespace idbdatafile

401
utils/cloudio/SocketPool.cpp
View File

@ -27,248 +27,249 @@ using namespace std;
namespace
{
void log(logging::LOG_TYPE whichLogFile, const string& msg)
{
logging::Logger logger(12); //12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
logging::Logger logger(12); // 12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
}
}
} // namespace
namespace idbdatafile
{
SocketPool::SocketPool()
{
config::Config *config = config::Config::makeConfig(); // the most 'config' ever put into a single line of code?
string stmp;
int64_t itmp = 0;
try
config::Config* config =
config::Config::makeConfig(); // the most 'config' ever put into a single line of code?
string stmp;
int64_t itmp = 0;
try
{
stmp = config->getConfig("StorageManager", "MaxSockets");
itmp = strtol(stmp.c_str(), NULL, 10);
if (itmp > 500 || itmp < 1)
{
stmp = config->getConfig("StorageManager", "MaxSockets");
itmp = strtol(stmp.c_str(), NULL, 10);
if (itmp > 500 || itmp < 1)
{
string errmsg = "SocketPool(): Got a bad value '" + stmp + "' for StorageManager/MaxSockets. Range is 1-500.";
log(logging::LOG_TYPE_CRITICAL, errmsg);
throw runtime_error(errmsg);
}
maxSockets = itmp;
}
catch (exception &e)
{
ostringstream os;
os << "SocketPool(): Using default of " << defaultSockets << ".";
log(logging::LOG_TYPE_CRITICAL, os.str());
maxSockets = defaultSockets;
string errmsg =
"SocketPool(): Got a bad value '" + stmp + "' for StorageManager/MaxSockets. Range is 1-500.";
log(logging::LOG_TYPE_CRITICAL, errmsg);
throw runtime_error(errmsg);
}
maxSockets = itmp;
}
catch (exception& e)
{
ostringstream os;
os << "SocketPool(): Using default of " << defaultSockets << ".";
log(logging::LOG_TYPE_CRITICAL, os.str());
maxSockets = defaultSockets;
}
}
SocketPool::~SocketPool()
{
boost::mutex::scoped_lock lock(mutex);
boost::mutex::scoped_lock lock(mutex);
for (uint i = 0; i < allSockets.size(); i++)
::close(allSockets[i]);
for (uint i = 0; i < allSockets.size(); i++)
::close(allSockets[i]);
}
#define sm_check_error \
if (err < 0) \
{ \
char _smbuf[80]; \
int l_errno = errno; \
log(logging::LOG_TYPE_ERROR, string("SocketPool: got a network error: ") + strerror_r(l_errno, _smbuf, 80)); \
remoteClosed(sock); \
return -1; \
}
int SocketPool::send_recv(messageqcpp::ByteStream &in, messageqcpp::ByteStream *out)
#define sm_check_error \
if (err < 0) \
{ \
char _smbuf[80]; \
int l_errno = errno; \
log(logging::LOG_TYPE_ERROR, \
string("SocketPool: got a network error: ") + strerror_r(l_errno, _smbuf, 80)); \
remoteClosed(sock); \
return -1; \
}
int SocketPool::send_recv(messageqcpp::ByteStream& in, messageqcpp::ByteStream* out)
{
uint count = 0;
uint length = in.length();
int sock = -1;
const uint8_t *inbuf = in.buf();
ssize_t err = 0;
uint count = 0;
uint length = in.length();
int sock = -1;
const uint8_t* inbuf = in.buf();
ssize_t err = 0;
retry:
int retries = 0;
while (sock < 0)
int retries = 0;
while (sock < 0)
{
sock = getSocket();
if (sock < 0)
{
sock = getSocket();
if (sock < 0)
{
if (++retries < 10)
{
//log(logging::LOG_TYPE_ERROR, "SocketPool::send_recv(): retrying in 5 sec...");
sleep(1);
}
else
{
errno = ECONNREFUSED;
return -1;
}
}
}
storagemanager::sm_msg_header hdr;
hdr.type = storagemanager::SM_MSG_START;
hdr.payloadLen = length;
hdr.flags = 0;
//cout << "SP sending msg on sock " << sock << " with length = " << length << endl;
err = ::write(sock, &hdr, sizeof(hdr));
if (err < 0 && errno == EPIPE)
{
log(logging::LOG_TYPE_WARNING, "SocketPool: remote connection is closed, getting a new one");
remoteClosed(sock);
sock = -1;
goto retry;
if (++retries < 10)
{
// log(logging::LOG_TYPE_ERROR, "SocketPool::send_recv(): retrying in 5 sec...");
sleep(1);
}
else
{
errno = ECONNREFUSED;
return -1;
}
}
}
storagemanager::sm_msg_header hdr;
hdr.type = storagemanager::SM_MSG_START;
hdr.payloadLen = length;
hdr.flags = 0;
// cout << "SP sending msg on sock " << sock << " with length = " << length << endl;
err = ::write(sock, &hdr, sizeof(hdr));
if (err < 0 && errno == EPIPE)
{
log(logging::LOG_TYPE_WARNING, "SocketPool: remote connection is closed, getting a new one");
remoteClosed(sock);
sock = -1;
goto retry;
}
sm_check_error;
while (count < length)
{
err = ::write(sock, &inbuf[count], length - count);
sm_check_error;
while (count < length)
count += err;
in.advance(err);
}
// cout << "SP sent msg with length = " << length << endl;
out->restart();
uint8_t* outbuf;
uint8_t window[8192];
uint remainingBytes = 0;
uint i;
storagemanager::sm_msg_header* resp = NULL;
while (1)
{
// cout << "SP receiving msg on sock " << sock << endl;
// here remainingBytes means the # of bytes from the previous message
err = ::read(sock, &window[remainingBytes], 8192 - remainingBytes);
sm_check_error;
if (err == 0)
{
err = ::write(sock, &inbuf[count], length-count);
sm_check_error;
count += err;
in.advance(err);
remoteClosed(sock);
// TODO, a retry loop
return -1;
}
//cout << "SP sent msg with length = " << length << endl;
out->restart();
uint8_t *outbuf;
uint8_t window[8192];
uint remainingBytes = 0;
uint i;
storagemanager::sm_msg_header *resp = NULL;
while (1)
uint endOfData = remainingBytes + err; // for clarity
// scan for the 8-byte header. If it is fragmented, move the fragment to the front of the buffer
// for the next iteration to handle.
if (endOfData < storagemanager::SM_HEADER_LEN)
{
//cout << "SP receiving msg on sock " << sock << endl;
// here remainingBytes means the # of bytes from the previous message
err = ::read(sock, &window[remainingBytes], 8192 - remainingBytes);
sm_check_error;
if (err == 0)
{
remoteClosed(sock);
// TODO, a retry loop
return -1;
}
uint endOfData = remainingBytes + err; // for clarity
// scan for the 8-byte header. If it is fragmented, move the fragment to the front of the buffer
// for the next iteration to handle.
if (endOfData < storagemanager::SM_HEADER_LEN)
{
remainingBytes = endOfData;
continue;
}
for (i = 0; i <= endOfData - storagemanager::SM_HEADER_LEN; i++)
{
if (*((uint *) &window[i]) == storagemanager::SM_MSG_START)
{
resp = (storagemanager::sm_msg_header *) &window[i];
break;
}
}
if (resp == NULL) // didn't find the header yet
{
remainingBytes = endOfData - i;
memmove(window, &window[i], remainingBytes);
}
else
{
// i == first byte of the header here
// copy the payload fragment we got into the output bytestream
uint startOfPayload = i + storagemanager::SM_HEADER_LEN; // for clarity
out->needAtLeast(resp->payloadLen);
outbuf = out->getInputPtr();
if (resp->payloadLen < endOfData - startOfPayload)
cout << "SocketPool: warning! Probably got a bad length field! payload length = " << resp->payloadLen <<
" endOfData = " << endOfData << " startOfPayload = " << startOfPayload << endl;
memcpy(outbuf, &window[startOfPayload], endOfData - startOfPayload);
remainingBytes = resp->payloadLen - (endOfData - startOfPayload); // remainingBytes is now the # of bytes left to read
out->advanceInputPtr(endOfData - startOfPayload);
break; // done looking for the header, can fill the output buffer directly now.
}
remainingBytes = endOfData;
continue;
}
// read the rest of the payload directly into the output bytestream
while (remainingBytes > 0)
for (i = 0; i <= endOfData - storagemanager::SM_HEADER_LEN; i++)
{
err = ::read(sock, &outbuf[resp->payloadLen - remainingBytes], remainingBytes);
sm_check_error;
remainingBytes -= err;
out->advanceInputPtr(err);
if (*((uint*)&window[i]) == storagemanager::SM_MSG_START)
{
resp = (storagemanager::sm_msg_header*)&window[i];
break;
}
}
returnSocket(sock);
return 0;
if (resp == NULL) // didn't find the header yet
{
remainingBytes = endOfData - i;
memmove(window, &window[i], remainingBytes);
}
else
{
// i == first byte of the header here
// copy the payload fragment we got into the output bytestream
uint startOfPayload = i + storagemanager::SM_HEADER_LEN; // for clarity
out->needAtLeast(resp->payloadLen);
outbuf = out->getInputPtr();
if (resp->payloadLen < endOfData - startOfPayload)
cout << "SocketPool: warning! Probably got a bad length field! payload length = "
<< resp->payloadLen << " endOfData = " << endOfData << " startOfPayload = " << startOfPayload
<< endl;
memcpy(outbuf, &window[startOfPayload], endOfData - startOfPayload);
remainingBytes = resp->payloadLen -
(endOfData - startOfPayload); // remainingBytes is now the # of bytes left to read
out->advanceInputPtr(endOfData - startOfPayload);
break; // done looking for the header, can fill the output buffer directly now.
}
}
// read the rest of the payload directly into the output bytestream
while (remainingBytes > 0)
{
err = ::read(sock, &outbuf[resp->payloadLen - remainingBytes], remainingBytes);
sm_check_error;
remainingBytes -= err;
out->advanceInputPtr(err);
}
returnSocket(sock);
return 0;
}
int SocketPool::getSocket()
{
boost::mutex::scoped_lock lock(mutex);
int clientSocket;
if (freeSockets.size() == 0 && allSockets.size() < maxSockets)
boost::mutex::scoped_lock lock(mutex);
int clientSocket;
if (freeSockets.size() == 0 && allSockets.size() < maxSockets)
{
// make a new connection
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null...
clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr));
if (err >= 0)
allSockets.push_back(clientSocket);
else
{
// make a new connection
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null...
clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr));
if (err >= 0)
allSockets.push_back(clientSocket);
else
{
int saved_errno = errno;
ostringstream os;
char buf[80];
os << "SocketPool::getSocket() failed to connect; got '" << strerror_r(saved_errno, buf, 80) << "'";
cout << os.str() << endl;
log(logging::LOG_TYPE_ERROR, os.str());
close(clientSocket);
errno = saved_errno;
return -1;
}
return clientSocket;
int saved_errno = errno;
ostringstream os;
char buf[80];
os << "SocketPool::getSocket() failed to connect; got '" << strerror_r(saved_errno, buf, 80) << "'";
cout << os.str() << endl;
log(logging::LOG_TYPE_ERROR, os.str());
close(clientSocket);
errno = saved_errno;
return -1;
}
// wait for a socket to become free
while (freeSockets.size() == 0)
socketAvailable.wait(lock);
clientSocket = freeSockets.front();
freeSockets.pop_front();
return clientSocket;
}
// wait for a socket to become free
while (freeSockets.size() == 0)
socketAvailable.wait(lock);
clientSocket = freeSockets.front();
freeSockets.pop_front();
return clientSocket;
}
void SocketPool::returnSocket(const int sock)
{
boost::mutex::scoped_lock lock(mutex);
//cout << "returning socket " << sock << endl;
freeSockets.push_back(sock);
socketAvailable.notify_one();
boost::mutex::scoped_lock lock(mutex);
// cout << "returning socket " << sock << endl;
freeSockets.push_back(sock);
socketAvailable.notify_one();
}
void SocketPool::remoteClosed(const int sock)
{
boost::mutex::scoped_lock lock(mutex);
//cout << "closing socket " << sock << endl;
::close(sock);
for (vector<int>::iterator i = allSockets.begin(); i != allSockets.end(); ++i)
if (*i == sock)
{
allSockets.erase(i, i+1);
break;
}
boost::mutex::scoped_lock lock(mutex);
// cout << "closing socket " << sock << endl;
::close(sock);
for (vector<int>::iterator i = allSockets.begin(); i != allSockets.end(); ++i)
if (*i == sock)
{
allSockets.erase(i, i + 1);
break;
}
}
}
} // namespace idbdatafile

44
utils/cloudio/SocketPool.h
View File

@ -25,33 +25,29 @@
namespace idbdatafile
{
/* This should be renamed; it's more than a pool, it also does the low-level communication. TBD. */
class SocketPool : public boost::noncopyable
{
public:
SocketPool();
// the dtor will immediately close all sockets
virtual ~SocketPool();
// 0 = success, -1 = failure. Should this throw instead?
int send_recv(messageqcpp::ByteStream &to_send, messageqcpp::ByteStream *to_recv);
public:
SocketPool();
private:
int getSocket();
void returnSocket(const int sock);
void remoteClosed(const int sock);
std::vector<int> allSockets;
std::deque<int> freeSockets;
boost::mutex mutex;
boost::condition_variable socketAvailable;
uint maxSockets;
static const uint defaultSockets = 20;
// the dtor will immediately close all sockets
virtual ~SocketPool();
// 0 = success, -1 = failure. Should this throw instead?
int send_recv(messageqcpp::ByteStream& to_send, messageqcpp::ByteStream* to_recv);
private:
int getSocket();
void returnSocket(const int sock);
void remoteClosed(const int sock);
std::vector<int> allSockets;
std::deque<int> freeSockets;
boost::mutex mutex;
boost::condition_variable socketAvailable;
uint maxSockets;
static const uint defaultSockets = 20;
};
}
} // namespace idbdatafile

14
utils/cloudio/cloud_plugin.cpp
View File

@ -22,18 +22,18 @@
using namespace idbdatafile;
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wreturn-type-c-linkage"
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wreturn-type-c-linkage"
#endif
extern "C"
{
FileFactoryEnt plugin_instance()
{
return FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem());
}
FileFactoryEnt plugin_instance()
{
return FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem());
}
}
#ifdef __clang__
#pragma clang diagnostic pop
#pragma clang diagnostic pop
#endif

416
utils/cloudio/component_test.cpp
View File

@ -14,9 +14,7 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
@ -44,218 +42,224 @@ using namespace std;
volatile bool die = false;
int errCode = 0;
/* Right now this will just accept one connection & send a fixed error response */
void error_server_thread()
{
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]);
int server_socket = ::socket(AF_UNIX, SOCK_STREAM, 0);
assert(server_socket >= 0);
int err = ::bind(server_socket, (struct sockaddr *) &addr, sizeof(addr));
assert(err == 0);
err = ::listen(server_socket, 1);
assert(err == 0);
socklen_t addrlen;
struct sockaddr_un client_addr;
memset(&client_addr, 0, sizeof(client_addr));
int client_socket = ::accept(server_socket, (struct sockaddr *) &client_addr, &addrlen);
assert(client_socket >= 0);
//cout << "server thread got a connection" << endl;
uint8_t buf[4096];
uint32_t response[4] = { storagemanager::SM_MSG_START, 8, (uint32_t ) -1, EINVAL };
uint remainingBytes = 0;
while (!die)
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]);
int server_socket = ::socket(AF_UNIX, SOCK_STREAM, 0);
assert(server_socket >= 0);
int err = ::bind(server_socket, (struct sockaddr*)&addr, sizeof(addr));
assert(err == 0);
err = ::listen(server_socket, 1);
assert(err == 0);
socklen_t addrlen;
struct sockaddr_un client_addr;
memset(&client_addr, 0, sizeof(client_addr));
int client_socket = ::accept(server_socket, (struct sockaddr*)&client_addr, &addrlen);
assert(client_socket >= 0);
// cout << "server thread got a connection" << endl;
uint8_t buf[4096];
uint32_t response[4] = {storagemanager::SM_MSG_START, 8, (uint32_t)-1, EINVAL};
uint remainingBytes = 0;
while (!die)
{
/* This just scans for SM_MSG_START, and as it finds them it sends a generic error
response. */
err = ::recv(client_socket, &buf[remainingBytes], 4096 - remainingBytes, MSG_DONTWAIT);
if (err < 0)
{
/* This just scans for SM_MSG_START, and as it finds them it sends a generic error
response. */
err = ::recv(client_socket, &buf[remainingBytes], 4096 - remainingBytes, MSG_DONTWAIT);
if (err < 0)
{
if (errno == EAGAIN || errno == EWOULDBLOCK)
{
//cout << "server looping..." << endl;
sleep(1); // who cares
continue;
}
else {
//char errbuf[80];
//cout << "server thread got an error: " << strerror_r(errno, errbuf, 80) << endl;
close(client_socket);
errCode = -1;
die = true;
return;
}
}
//cout << "server thread got some data" << endl;
uint endOfData = remainingBytes + err;
uint i;
if (endOfData < 4)
{
remainingBytes = endOfData;
continue;
}
for (i = 0; i <= endOfData - 4; i++)
{
if (*((uint32_t *) &buf[i]) == storagemanager::SM_MSG_START) {
//cout << "server thread found a msg start magic" << endl;
err = ::send(client_socket, response, 16, 0);
assert(err > 0);
}
}
memmove(buf, &buf[i], endOfData - i); // should be the trailing 3 bytes of the data
remainingBytes = endOfData - i;
if (errno == EAGAIN || errno == EWOULDBLOCK)
{
// cout << "server looping..." << endl;
sleep(1); // who cares
continue;
}
else
{
// char errbuf[80];
// cout << "server thread got an error: " << strerror_r(errno, errbuf, 80) << endl;
close(client_socket);
errCode = -1;
die = true;
return;
}
}
}
// cout << "server thread got some data" << endl;
uint endOfData = remainingBytes + err;
uint i;
if (endOfData < 4)
{
remainingBytes = endOfData;
continue;
}
for (i = 0; i <= endOfData - 4; i++)
{
if (*((uint32_t*)&buf[i]) == storagemanager::SM_MSG_START)
{
// cout << "server thread found a msg start magic" << endl;
err = ::send(client_socket, response, 16, 0);
assert(err > 0);
}
}
memmove(buf, &buf[i], endOfData - i); // should be the trailing 3 bytes of the data
remainingBytes = endOfData - i;
}
}
int test1()
{
/* Start a server thread that returns generic errors */
boost::thread server_thread(error_server_thread);
/* Instantiate each of SM subclasses, call each function, and verify the expected error response */
int err;
SMFileFactory factory;
cout << "open" << endl;
IDBDataFile *file = factory.open("dummy", "r", 0, 0);
assert(file == NULL && errno == EINVAL && !die);
SMFileSystem filesystem;
cout << "compressedSize" << endl;
bool gotException = false;
try {
filesystem.compressedSize("dummy");
}
catch (NotImplementedYet &) {
gotException = true;
}
assert(gotException && !die);
cout << "copyFile" << endl;
try {
filesystem.copyFile("dummy1", "dummy2");
}
catch (NotImplementedYet &) {
gotException = true;
}
assert(gotException && !die);
cout << "rename" << endl;
try {
filesystem.rename("dummy1", "dummy2");
}
catch (NotImplementedYet &) {
gotException = true;
}
assert(gotException && !die);
cout << "exists" << endl;
err = filesystem.exists("dummy");
assert(!err && errno == EINVAL);
cout << "filesystemisup" << endl;
err = filesystem.filesystemIsUp();
assert(!err && errno == EINVAL && !die);
cout << "isdir" << endl;
err = filesystem.isDir("dummy");
assert(!err && errno == EINVAL && !die);
cout << "listdirectory" << endl;
list<string> filenames;
err = filesystem.listDirectory("dummy", filenames);
assert(err == -1 && errno == EINVAL && filenames.empty() && !die);
cout << "remove" << endl;
err = filesystem.remove("dummy");
assert(err == -1 && errno == EINVAL && !die);
cout << "size" << endl;
err = filesystem.size("dummy");
assert(err == -1 && errno == EINVAL && !die);
cout << "mkdir" << endl;
err = filesystem.mkdir("dummy");
assert(err == 0 && !die);
cout << "datafile constructor" << endl;
SMDataFile f("dummy", O_RDONLY, 12345);
f = SMDataFile("dummy2", O_WRONLY, 123456);
f = SMDataFile("dummy2", O_RDWR | O_APPEND, 1234567);
cout << "pread" << endl;
uint8_t buf[1024];
err = f.pread(buf, 0, 1024);
assert(err == -1 && errno == EINVAL && !die);
cout << "read" << endl;
err = f.read(buf, 1024);
assert(err == -1 && errno == EINVAL && !die);
cout << "write" << endl;
err = f.write(buf, 1024);
assert(err == -1 && errno == EINVAL && !die);
cout << "seek" << endl;
err = f.seek(1234, SEEK_SET);
assert(err == 0 && !die);
err = f.seek(1234, SEEK_CUR);
assert(err == 0 && !die);
err = f.seek(1234, SEEK_END);
assert(err == -1 && errno == EINVAL && !die);
cout << "truncate" << endl;
err = f.truncate(1234);
assert(err == -1 && errno == EINVAL && !die);
cout << "size" << endl;
err = f.size();
assert(err == -1 && errno == EINVAL && !die);
cout << "tell" << endl;
err = f.tell();
assert(err == 2468);
cout << "flush" << endl;
err = f.flush();
assert(err == 0);
cout << "mtime" << endl;
err = f.mtime();
assert(err == -1 && errno == EINVAL && !die);
cout << "close" << endl;
err = f.close();
assert(err == 0);
// done, return errCode
die = true;
cout << "done, waiting for server thread to stop" << endl;
server_thread.join();
return errCode;
/* Start a server thread that returns generic errors */
boost::thread server_thread(error_server_thread);
/* Instantiate each of SM subclasses, call each function, and verify the expected error response */
int err;
SMFileFactory factory;
cout << "open" << endl;
IDBDataFile* file = factory.open("dummy", "r", 0, 0);
assert(file == NULL && errno == EINVAL && !die);
SMFileSystem filesystem;
cout << "compressedSize" << endl;
bool gotException = false;
try
{
filesystem.compressedSize("dummy");
}
catch (NotImplementedYet&)
{
gotException = true;
}
assert(gotException && !die);
cout << "copyFile" << endl;
try
{
filesystem.copyFile("dummy1", "dummy2");
}
catch (NotImplementedYet&)
{
gotException = true;
}
assert(gotException && !die);
cout << "rename" << endl;
try
{
filesystem.rename("dummy1", "dummy2");
}
catch (NotImplementedYet&)
{
gotException = true;
}
assert(gotException && !die);
cout << "exists" << endl;
err = filesystem.exists("dummy");
assert(!err && errno == EINVAL);
cout << "filesystemisup" << endl;
err = filesystem.filesystemIsUp();
assert(!err && errno == EINVAL && !die);
cout << "isdir" << endl;
err = filesystem.isDir("dummy");
assert(!err && errno == EINVAL && !die);
cout << "listdirectory" << endl;
list<string> filenames;
err = filesystem.listDirectory("dummy", filenames);
assert(err == -1 && errno == EINVAL && filenames.empty() && !die);
cout << "remove" << endl;
err = filesystem.remove("dummy");
assert(err == -1 && errno == EINVAL && !die);
cout << "size" << endl;
err = filesystem.size("dummy");
assert(err == -1 && errno == EINVAL && !die);
cout << "mkdir" << endl;
err = filesystem.mkdir("dummy");
assert(err == 0 && !die);
cout << "datafile constructor" << endl;
SMDataFile f("dummy", O_RDONLY, 12345);
f = SMDataFile("dummy2", O_WRONLY, 123456);
f = SMDataFile("dummy2", O_RDWR | O_APPEND, 1234567);
cout << "pread" << endl;
uint8_t buf[1024];
err = f.pread(buf, 0, 1024);
assert(err == -1 && errno == EINVAL && !die);
cout << "read" << endl;
err = f.read(buf, 1024);
assert(err == -1 && errno == EINVAL && !die);
cout << "write" << endl;
err = f.write(buf, 1024);
assert(err == -1 && errno == EINVAL && !die);
cout << "seek" << endl;
err = f.seek(1234, SEEK_SET);
assert(err == 0 && !die);
err = f.seek(1234, SEEK_CUR);
assert(err == 0 && !die);
err = f.seek(1234, SEEK_END);
assert(err == -1 && errno == EINVAL && !die);
cout << "truncate" << endl;
err = f.truncate(1234);
assert(err == -1 && errno == EINVAL && !die);
cout << "size" << endl;
err = f.size();
assert(err == -1 && errno == EINVAL && !die);
cout << "tell" << endl;
err = f.tell();
assert(err == 2468);
cout << "flush" << endl;
err = f.flush();
assert(err == 0);
cout << "mtime" << endl;
err = f.mtime();
assert(err == -1 && errno == EINVAL && !die);
cout << "close" << endl;
err = f.close();
assert(err == 0);
// done, return errCode
die = true;
cout << "done, waiting for server thread to stop" << endl;
server_thread.join();
return errCode;
}
int main()
{
int ret;
ret = test1();
return ret;
int ret;
ret = test1();
return ret;
}

335
utils/cloudio/end_to_end_test.cpp Executable file → Normal file
View File

@ -14,9 +14,7 @@
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
@ -47,216 +45,217 @@ int errCode = 0;
void printResultPASS(int result)
{
if (result == 0)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
if (result == 0)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
}
void printResultFAIL(int result)
{
if (result == 0)
cout << " FAIL" << endl;
else
cout << " PASS" << endl;
if (result == 0)
cout << " FAIL" << endl;
else
cout << " PASS" << endl;
}
void printResultPASS(bool result)
{
if (result)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
if (result)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
}
void printResultFAIL(bool result)
{
if (result)
cout << " FAIL" << endl;
else
cout << " PASS" << endl;
if (result)
cout << " FAIL" << endl;
else
cout << " PASS" << endl;
}
void printResult(int result, int expected)
{
if (result == expected)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
if (result == expected)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
}
int testAll()
{
/* Instantiate each of SM subclasses, call each function, and verify the expected error response */
int err;
SMFileFactory factory;
list<string> filenames;
SMFileSystem filesystem;
cout << "compressedSize --- ";
bool gotException = false;
try {
filesystem.compressedSize("./testData/dummy");
}
catch (NotImplementedYet &) {
gotException = true;
}
assert(gotException && !die);
cout << " N/A" << endl;
/* Instantiate each of SM subclasses, call each function, and verify the expected error response */
int err;
SMFileFactory factory;
list<string> filenames;
SMFileSystem filesystem;
cout << "copyFile dummy1->dummy2 --- ";
err = filesystem.copyFile("./testData/dummy1", "./testData/dummy2");
printResultFAIL(err);
cout << "compressedSize --- ";
bool gotException = false;
try
{
filesystem.compressedSize("./testData/dummy");
}
catch (NotImplementedYet&)
{
gotException = true;
}
assert(gotException && !die);
cout << " N/A" << endl;
cout << "copyFile end_to_end_test_file.start->readTest --- ";
err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/readTest");
printResultPASS(err);
cout << "copyFile dummy1->dummy2 --- ";
err = filesystem.copyFile("./testData/dummy1", "./testData/dummy2");
printResultFAIL(err);
cout << "copyFile end_to_end_test_file.start->dummy --- ";
err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/dummy");
printResultPASS(err);
cout << "rename dummy->dummyrename --- ";
err = filesystem.rename("./testData/dummy", "./testData/dummyrename");
printResultPASS(err);
cout << "copyFile end_to_end_test_file.start->readTest --- ";
err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/readTest");
printResultPASS(err);
cout << "exists dummyrename --- ";
bool exists = filesystem.exists("./testData/dummyrename");
printResultPASS(exists);
cout << "exists DNE --- ";
bool doesNotExist = filesystem.exists("./testData/DNE");
printResultFAIL(doesNotExist);
cout << "filesystemisup --- ";
bool fsIsUp = filesystem.filesystemIsUp();
printResultPASS(fsIsUp);
cout << "isdir testData --- ";
bool isDir = filesystem.isDir("testData");
printResultPASS(isDir);
cout << "copyFile end_to_end_test_file.start->dummy --- ";
err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/dummy");
printResultPASS(err);
cout << "isdir readTest --- ";
bool isNotDir = filesystem.isDir("./testData/readTest");
printResultFAIL(isNotDir);
cout << "rename dummy->dummyrename --- ";
err = filesystem.rename("./testData/dummy", "./testData/dummyrename");
printResultPASS(err);
cout << "isdir DNE --- ";
isNotDir = filesystem.isDir("./testData/DNE");
printResultFAIL(isNotDir);
cout << "exists dummyrename --- ";
bool exists = filesystem.exists("./testData/dummyrename");
printResultPASS(exists);
cout << "exists DNE --- ";
bool doesNotExist = filesystem.exists("./testData/DNE");
printResultFAIL(doesNotExist);
cout << "listdirectory testData --- ";
err = filesystem.listDirectory("testData", filenames);
printResultPASS(err);
cout << "listdirectory dummyrename --- ";
err = filesystem.listDirectory("./testData/dummyrename", filenames);
printResultFAIL(err);
cout << "filesystemisup --- ";
bool fsIsUp = filesystem.filesystemIsUp();
printResultPASS(fsIsUp);
cout << "listdirectory DNE --- ";
err = filesystem.listDirectory("./testData/DNE", filenames);
printResultFAIL(err);
cout << "isdir testData --- ";
bool isDir = filesystem.isDir("testData");
printResultPASS(isDir);
cout << "remove dummyrename --- ";
err = filesystem.remove("./testData/dummyrename");
printResultPASS(err);
cout << "isdir readTest --- ";
bool isNotDir = filesystem.isDir("./testData/readTest");
printResultFAIL(isNotDir);
cout << "size readTest --- ";
err = filesystem.size("./testData/readTest");
printResult(err,10940);
cout << "isdir DNE --- ";
isNotDir = filesystem.isDir("./testData/DNE");
printResultFAIL(isNotDir);
cout << "size DNE --- ";
err = filesystem.size("./testData/DNE");
printResultFAIL(err);
cout << "listdirectory testData --- ";
err = filesystem.listDirectory("testData", filenames);
printResultPASS(err);
cout << "open readTest r --- ";
IDBDataFile *fileR = factory.open("./testData/readTest", "r", 0, 0);
if (fileR)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
cout << "listdirectory dummyrename --- ";
err = filesystem.listDirectory("./testData/dummyrename", filenames);
printResultFAIL(err);
cout << "open readTest a --- ";
IDBDataFile *fileW = factory.open("./testData/readTest", "a", 0, 0);
if (fileW)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
cout << "listdirectory DNE --- ";
err = filesystem.listDirectory("./testData/DNE", filenames);
printResultFAIL(err);
cout << "open DNE --- ";
IDBDataFile *file2 = factory.open("./testData/DNE", "r", 0, 0);
if (file2)
cout << " FAIL" << endl;
else
cout << " PASS" << endl;
cout << "remove dummyrename --- ";
err = filesystem.remove("./testData/dummyrename");
printResultPASS(err);
cout << "pread --- ";
uint8_t buf[2048];
err = fileR->pread(buf, 0, 1024);
printResult(err,1024);
cout << "size readTest --- ";
err = filesystem.size("./testData/readTest");
printResult(err, 10940);
cout << "read --- ";
err = fileR->read(buf, 1024);
printResult(err,1024);
cout << "size DNE --- ";
err = filesystem.size("./testData/DNE");
printResultFAIL(err);
int newSize = 10940 + 1024;
cout << "write --- ";
err = fileW->write(buf, 1024);
printResult(err,1024);
cout << "size "<< newSize << " --- ";
err = fileW->size();
cout << err;
printResult(err,newSize);
cout << "open readTest r --- ";
IDBDataFile* fileR = factory.open("./testData/readTest", "r", 0, 0);
if (fileR)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
cout << "seek SEEK_SET --- ";
err = fileR->seek(1234, SEEK_SET);
printResultPASS(err);
cout << "tell 1234 --- ";
err = fileR->tell();
cout << err;
printResult(err,1234);
cout << "seek SEEK_CUR --- ";
err = fileR->seek(1234, SEEK_CUR);
printResultPASS(err);
cout << "tell 2468 --- ";
err = fileR->tell();
cout << err;
printResult(err,2468);
cout << "seek SEEK_END --- ";
err = fileR->seek(1234, SEEK_END);
printResultPASS(err);
cout << "tell 12174 --- ";
err = fileR->tell();
cout << err;
printResult(err,13198);
cout << "open readTest a --- ";
IDBDataFile* fileW = factory.open("./testData/readTest", "a", 0, 0);
if (fileW)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
cout << "truncate 1234 --- ";
err = fileR->truncate(1234);
printResultPASS(err);
cout << "open DNE --- ";
IDBDataFile* file2 = factory.open("./testData/DNE", "r", 0, 0);
if (file2)
cout << " FAIL" << endl;
else
cout << " PASS" << endl;
cout << "size 1234 --- ";
err = fileR->size();
printResult(err,1234);
cout << "pread --- ";
uint8_t buf[2048];
err = fileR->pread(buf, 0, 1024);
printResult(err, 1024);
cout << "mtime --- ";
err = fileR->mtime();
if (err > 0)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
cout << "read --- ";
err = fileR->read(buf, 1024);
printResult(err, 1024);
cout << "remove readTest --- ";
err = filesystem.remove("./testData/readTest");
printResultPASS(err);
int newSize = 10940 + 1024;
cout << "write --- ";
err = fileW->write(buf, 1024);
printResult(err, 1024);
cout << "size " << newSize << " --- ";
err = fileW->size();
cout << err;
printResult(err, newSize);
cout << "done." << endl;
return errCode;
cout << "seek SEEK_SET --- ";
err = fileR->seek(1234, SEEK_SET);
printResultPASS(err);
cout << "tell 1234 --- ";
err = fileR->tell();
cout << err;
printResult(err, 1234);
cout << "seek SEEK_CUR --- ";
err = fileR->seek(1234, SEEK_CUR);
printResultPASS(err);
cout << "tell 2468 --- ";
err = fileR->tell();
cout << err;
printResult(err, 2468);
cout << "seek SEEK_END --- ";
err = fileR->seek(1234, SEEK_END);
printResultPASS(err);
cout << "tell 12174 --- ";
err = fileR->tell();
cout << err;
printResult(err, 13198);
cout << "truncate 1234 --- ";
err = fileR->truncate(1234);
printResultPASS(err);
cout << "size 1234 --- ";
err = fileR->size();
printResult(err, 1234);
cout << "mtime --- ";
err = fileR->mtime();
if (err > 0)
cout << " PASS" << endl;
else
cout << " FAIL" << endl;
cout << "remove readTest --- ";
err = filesystem.remove("./testData/readTest");
printResultPASS(err);
cout << "done." << endl;
return errCode;
}
int main()
{
int ret;
ret = testAll();
//ret = testTest();
return ret;
int ret;
ret = testAll();
// ret = testTest();
return ret;
}

13
utils/cloudio/sm_exceptions.h
View File

@ -21,17 +21,14 @@
namespace idbdatafile
{
class NotImplementedYet : public std::logic_error
{
public:
NotImplementedYet(const std::string &s);
public:
NotImplementedYet(const std::string& s);
};
NotImplementedYet::NotImplementedYet(const std::string &s) :
std::logic_error(s)
NotImplementedYet::NotImplementedYet(const std::string& s) : std::logic_error(s)
{
}
}
} // namespace idbdatafile

155
utils/common/MonitorProcMem.cpp
View File

@ -45,10 +45,9 @@ using namespace logging;
namespace utils
{
uint64_t MonitorProcMem::fMemTotal = 1;
uint64_t MonitorProcMem::fMemFree = 0;
int MonitorProcMem::fMemPctCheck = 0;
int MonitorProcMem::fMemPctCheck = 0;
//------------------------------------------------------------------------------
// Thread entry point function, that drives a thread to periodically (every
@ -57,30 +56,30 @@ int MonitorProcMem::fMemPctCheck = 0;
//------------------------------------------------------------------------------
void MonitorProcMem::operator()() const
{
while (1)
while (1)
{
if (fMaxPct > 0)
{
if (fMaxPct > 0)
{
size_t pct = rss() * 100 / fMemTotal;
size_t pct = rss() * 100 / fMemTotal;
if (pct > fMaxPct)
{
cerr << "PrimProc: Too much memory allocated!" << endl;
if (pct > fMaxPct)
{
cerr << "PrimProc: Too much memory allocated!" << endl;
LoggingID logid(fSubsystemID);
logging::Message msg(logging::M0045);
logging::Message::Args args;
msg.format(args);
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
exit(1);
}
}
fMemFree = cg.getFreeMemory();
//calculateFreeMem();
pause_();
LoggingID logid(fSubsystemID);
logging::Message msg(logging::M0045);
logging::Message::Args args;
msg.format(args);
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
exit(1);
}
}
fMemFree = cg.getFreeMemory();
// calculateFreeMem();
pause_();
}
}
//------------------------------------------------------------------------------
@ -89,7 +88,7 @@ void MonitorProcMem::operator()() const
//------------------------------------------------------------------------------
size_t MonitorProcMem::memTotal() const
{
return cg.getTotalMemory();
return cg.getTotalMemory();
}
//------------------------------------------------------------------------------
@ -98,83 +97,81 @@ size_t MonitorProcMem::memTotal() const
//------------------------------------------------------------------------------
size_t MonitorProcMem::rss() const
{
uint64_t rss;
uint64_t rss;
#if defined(_MSC_VER)
HANDLE hProcess;
PROCESS_MEMORY_COUNTERS pmc;
hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |
PROCESS_VM_READ,
FALSE, fPid );
HANDLE hProcess;
PROCESS_MEMORY_COUNTERS pmc;
hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, fPid);
if (NULL == hProcess)
return 0;
if (NULL == hProcess)
return 0;
if ( GetProcessMemoryInfo( hProcess, &pmc, sizeof(pmc)) )
rss = pmc.WorkingSetSize;
else
rss = 0;
if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc)))
rss = pmc.WorkingSetSize;
else
rss = 0;
CloseHandle( hProcess );
CloseHandle(hProcess);
#elif defined(__FreeBSD__)
ostringstream cmd;
cmd << "ps -a -o rss -p " << getpid() << " | tail +2";
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.str().c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
rss = atoi(input) * 1024LL;
ostringstream cmd;
cmd << "ps -a -o rss -p " << getpid() << " | tail +2";
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.str().c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
rss = atoi(input) * 1024LL;
#else
ostringstream pstat;
pstat << "/proc/" << fPid << "/statm";
ifstream in(pstat.str().c_str());
size_t x;
ostringstream pstat;
pstat << "/proc/" << fPid << "/statm";
ifstream in(pstat.str().c_str());
size_t x;
in >> x;
in >> rss;
in >> x;
in >> rss;
//rss is now in pages, convert to bytes
rss *= fPageSize;
// rss is now in pages, convert to bytes
rss *= fPageSize;
#endif
return static_cast<size_t>(rss);
return static_cast<size_t>(rss);
}
//------------------------------------------------------------------------------
// Stays in "sleep" state till fSleepSec seconds have elapsed.
//------------------------------------------------------------------------------
void MonitorProcMem::pause_( ) const
void MonitorProcMem::pause_() const
{
struct timespec req;
struct timespec rem;
struct timespec req;
struct timespec rem;
req.tv_sec = fSleepSec;
req.tv_nsec = 0;
req.tv_sec = fSleepSec;
req.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
while (1)
{
while (1)
{
#ifdef _MSC_VER
Sleep(req.tv_sec * 1000);
Sleep(req.tv_sec * 1000);
#else
if (nanosleep(&req, &rem) != 0)
{
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
continue;
}
}
if (nanosleep(&req, &rem) != 0)
{
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
continue;
}
}
#endif
break;
}
break;
}
}
//------------------------------------------------------------------------------
@ -182,7 +179,7 @@ void MonitorProcMem::pause_( ) const
//------------------------------------------------------------------------------
unsigned MonitorProcMem::memUsedPct()
{
return ((100 * (fMemTotal - fMemFree)) / fMemTotal);
return ((100 * (fMemTotal - fMemFree)) / fMemTotal);
}
//------------------------------------------------------------------------------
@ -190,8 +187,8 @@ unsigned MonitorProcMem::memUsedPct()
//------------------------------------------------------------------------------
bool MonitorProcMem::isMemAvailable(size_t memRequest)
{
int memAvailPct = ((100 * (fMemFree - memRequest)) / fMemTotal);
return (memAvailPct < fMemPctCheck);
int memAvailPct = ((100 * (fMemFree - memRequest)) / fMemTotal);
return (memAvailPct < fMemPctCheck);
}
} // end of namespace
} // namespace utils

133
utils/common/MonitorProcMem.h
View File

@ -35,7 +35,6 @@
namespace utils
{
/** @brief A class to monitor a process's memory usage.
*
* This class will monitor and terminate the process if
@ -43,86 +42,78 @@ namespace utils
*/
class MonitorProcMem
{
public:
/** @brief MonitorProcMem constructor
*
* @param maxPct (in) maximum allowable memory usage
* @param memChk (in) monitor total system physical memory usage
* @param msgLog (in) message logger to log msg to
* @param sec (in) number of seconds between memory checks
*/
explicit MonitorProcMem(size_t maxPct,
size_t memChk,
uint32_t subsystemID,
unsigned sec = 1) :
fPid ( getpid() ),
fMaxPct ( maxPct ),
fSleepSec ( sec ),
fSubsystemID ( subsystemID ),
fPageSize ( getpagesize() )
{
fMemPctCheck = memChk;
fMemTotal = memTotal();
}
public:
/** @brief MonitorProcMem constructor
*
* @param maxPct (in) maximum allowable memory usage
* @param memChk (in) monitor total system physical memory usage
* @param msgLog (in) message logger to log msg to
* @param sec (in) number of seconds between memory checks
*/
explicit MonitorProcMem(size_t maxPct, size_t memChk, uint32_t subsystemID, unsigned sec = 1)
: fPid(getpid()), fMaxPct(maxPct), fSleepSec(sec), fSubsystemID(subsystemID), fPageSize(getpagesize())
{
fMemPctCheck = memChk;
fMemTotal = memTotal();
}
virtual ~MonitorProcMem() {};
virtual ~MonitorProcMem(){};
/** @brief Thread entry point
*
* Entry point for this thread that monitors memory usage.
*/
virtual void operator()() const;
/** @brief Thread entry point
*
* Entry point for this thread that monitors memory usage.
*/
virtual void operator()() const;
/* Accessor to return the free memory
*
*/
size_t getFreeMem() const
{
return fMemFree;
}
/* Accessor to return the free memory
*
*/
size_t getFreeMem() const
{
return fMemFree;
}
/* return the % of total memory used
*
*/
static unsigned memUsedPct();
/* return the % of total memory used
*
*/
static unsigned memUsedPct();
/* return true if memory usage is below configured limit
*
*/
static bool isMemAvailable(size_t memRequest = 0);
/* return true if memory usage is below configured limit
*
*/
static bool isMemAvailable(size_t memRequest = 0);
protected:
//Defaults are okay
//MonitorProcMem (const MonitorProcMem& rhs);
//MonitorProcMem& operator=(const MonitorProcMem& rhs);
protected:
// Defaults are okay
// MonitorProcMem (const MonitorProcMem& rhs);
// MonitorProcMem& operator=(const MonitorProcMem& rhs);
/** return the current process RSS size in MB
*
*/
size_t rss() const;
/** return the current process RSS size in MB
*
*/
size_t rss() const;
/* return the system RAM size in MB
*
*/
size_t memTotal() const;
/* return the system RAM size in MB
*
*/
size_t memTotal() const;
/* pause for fSleepSec seconds between memory usage checks
*
*/
void pause_() const;
/* pause for fSleepSec seconds between memory usage checks
*
*/
void pause_() const;
pid_t fPid; // process pid
size_t fMaxPct; // max allowable % memory use
unsigned fSleepSec; // sleep interval in seconds
uint32_t fSubsystemID; // Subsystem ID for Logger
int fPageSize; // page size for this host (in bytes)
pid_t fPid; // process pid
size_t fMaxPct; // max allowable % memory use
unsigned fSleepSec; // sleep interval in seconds
uint32_t fSubsystemID; // Subsystem ID for Logger
int fPageSize; // page size for this host (in bytes)
// @bug4507, monitor % of total used system memory
static uint64_t fMemTotal;
static uint64_t fMemFree;
static int fMemPctCheck;
mutable CGroupConfigurator cg;
// @bug4507, monitor % of total used system memory
static uint64_t fMemTotal;
static uint64_t fMemFree;
static int fMemPctCheck;
mutable CGroupConfigurator cg;
};
}
} // namespace utils

518
utils/common/any.hpp Executable file → Normal file
View File

@ -18,290 +18,294 @@ namespace static_any
{
namespace anyimpl
{
struct empty_any
{
};
struct empty_any
{
};
struct base_any_policy
{
virtual void static_delete(void** x) = 0;
virtual void copy_from_value(void const* src, void** dest) = 0;
virtual void clone(void* const* src, void** dest) = 0;
virtual void move(void* const* src, void** dest) = 0;
virtual void* get_value(void** src) = 0;
virtual size_t get_size() = 0;
protected:
~base_any_policy() = default;
};
struct base_any_policy
{
virtual void static_delete(void** x) = 0;
virtual void copy_from_value(void const* src, void** dest) = 0;
virtual void clone(void* const* src, void** dest) = 0;
virtual void move(void* const* src, void** dest) = 0;
virtual void* get_value(void** src) = 0;
virtual size_t get_size() = 0;
//inline base_any_policy::~base_any_policy() throw () {}
protected:
~base_any_policy() = default;
};
template<typename T>
struct typed_base_any_policy : base_any_policy
{
virtual size_t get_size()
{
return sizeof(T);
}
protected:
~typed_base_any_policy() = default;
};
// inline base_any_policy::~base_any_policy() throw () {}
template<typename T>
struct small_any_policy : typed_base_any_policy<T>
{
virtual ~small_any_policy() = default;
virtual void static_delete(void** x)
{
*x = 0;
}
virtual void copy_from_value(void const* src, void** dest)
{
new(dest) T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = *src;
}
virtual void move(void* const* src, void** dest)
{
*dest = *src;
}
virtual void* get_value(void** src)
{
return reinterpret_cast<void*>(src);
}
};
template <typename T>
struct typed_base_any_policy : base_any_policy
{
virtual size_t get_size()
{
return sizeof(T);
}
template<typename T>
struct big_any_policy : typed_base_any_policy<T>
{
virtual ~big_any_policy() = default;
virtual void static_delete(void** x)
{
if (*x)
delete(*reinterpret_cast<T**>(x));
*x = NULL;
}
virtual void copy_from_value(void const* src, void** dest)
{
*dest = new T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = new T(**reinterpret_cast<T* const*>(src));
}
virtual void move(void* const* src, void** dest)
{
(*reinterpret_cast<T**>(dest))->~T();
**reinterpret_cast<T**>(dest) = **reinterpret_cast<T* const*>(src);
}
virtual void* get_value(void** src)
{
return *src;
}
};
protected:
~typed_base_any_policy() = default;
};
template<typename T>
struct choose_policy
{
typedef big_any_policy<T> type;
};
template <typename T>
struct small_any_policy : typed_base_any_policy<T>
{
virtual ~small_any_policy() = default;
virtual void static_delete(void** x)
{
*x = 0;
}
virtual void copy_from_value(void const* src, void** dest)
{
new (dest) T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = *src;
}
virtual void move(void* const* src, void** dest)
{
*dest = *src;
}
virtual void* get_value(void** src)
{
return reinterpret_cast<void*>(src);
}
};
template<typename T>
struct choose_policy<T*>
{
typedef small_any_policy<T*> type;
};
template <typename T>
struct big_any_policy : typed_base_any_policy<T>
{
virtual ~big_any_policy() = default;
virtual void static_delete(void** x)
{
if (*x)
delete (*reinterpret_cast<T**>(x));
*x = NULL;
}
virtual void copy_from_value(void const* src, void** dest)
{
*dest = new T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = new T(**reinterpret_cast<T* const*>(src));
}
virtual void move(void* const* src, void** dest)
{
(*reinterpret_cast<T**>(dest))->~T();
**reinterpret_cast<T**>(dest) = **reinterpret_cast<T* const*>(src);
}
virtual void* get_value(void** src)
{
return *src;
}
};
struct any;
template <typename T>
struct choose_policy
{
typedef big_any_policy<T> type;
};
/// Choosing the policy for an any type is illegal, but should never happen.
/// This is designed to throw a compiler error.
template<>
struct choose_policy<any>
{
typedef void type;
};
template <typename T>
struct choose_policy<T*>
{
typedef small_any_policy<T*> type;
};
/// Specializations for big types.
#define BIG_POLICY(TYPE) template<> struct \
choose_policy<TYPE> { typedef big_any_policy<TYPE> type; };
struct any;
BIG_POLICY(int128_t);
/// Specializations for small types.
#define SMALL_POLICY(TYPE) template<> struct \
choose_policy<TYPE> { typedef small_any_policy<TYPE> type; };
/// Choosing the policy for an any type is illegal, but should never happen.
/// This is designed to throw a compiler error.
template <>
struct choose_policy<any>
{
typedef void type;
};
SMALL_POLICY(char);
SMALL_POLICY(signed char);
SMALL_POLICY(unsigned char);
SMALL_POLICY(signed short);
SMALL_POLICY(unsigned short);
SMALL_POLICY(signed int);
SMALL_POLICY(unsigned int);
SMALL_POLICY(signed long);
SMALL_POLICY(unsigned long);
SMALL_POLICY(signed long long);
SMALL_POLICY(unsigned long long);
SMALL_POLICY(float);
SMALL_POLICY(double);
SMALL_POLICY(bool);
/// Specializations for big types.
#define BIG_POLICY(TYPE) \
template <> \
struct choose_policy<TYPE> \
{ \
typedef big_any_policy<TYPE> type; \
};
BIG_POLICY(int128_t);
/// Specializations for small types.
#define SMALL_POLICY(TYPE) \
template <> \
struct choose_policy<TYPE> \
{ \
typedef small_any_policy<TYPE> type; \
};
SMALL_POLICY(char);
SMALL_POLICY(signed char);
SMALL_POLICY(unsigned char);
SMALL_POLICY(signed short);
SMALL_POLICY(unsigned short);
SMALL_POLICY(signed int);
SMALL_POLICY(unsigned int);
SMALL_POLICY(signed long);
SMALL_POLICY(unsigned long);
SMALL_POLICY(signed long long);
SMALL_POLICY(unsigned long long);
SMALL_POLICY(float);
SMALL_POLICY(double);
SMALL_POLICY(bool);
#undef SMALL_POLICY
/// This function will return a different policy for each type.
template<typename T>
base_any_policy* get_policy()
{
static typename choose_policy<T>::type policy;
return &policy;
};
}
/// This function will return a different policy for each type.
template <typename T>
base_any_policy* get_policy()
{
static typename choose_policy<T>::type policy;
return &policy;
};
} // namespace anyimpl
class any
{
private:
// fields
anyimpl::base_any_policy* policy;
void* object;
private:
// fields
anyimpl::base_any_policy* policy;
void* object;
public:
/// Initializing constructor.
template <typename T>
any(const T& x)
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
public:
/// Initializing constructor.
template <typename T>
any(const T& x) : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Empty constructor.
any() : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
}
/// Special initializing constructor for string literals.
any(const char* x) : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Copy constructor.
any(const any& x) : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Destructor.
~any()
{
policy->static_delete(&object);
}
/// Assignment function from another any.
any& assign(const any& x)
{
reset();
policy = x.policy;
policy->clone(&x.object, &object);
return *this;
}
/// Assignment function.
template <typename T>
any& assign(const T& x)
{
reset();
policy = anyimpl::get_policy<T>();
policy->copy_from_value(&x, &object);
return *this;
}
/// Assignment operator.
template <typename T>
any& operator=(const T& x)
{
return assign(x);
}
/// Assignment operator, specialed for literal strings.
/// They have types like const char [6] which don't work as expected.
any& operator=(const char* x)
{
return assign(x);
}
/// Less than operator for sorting
bool operator<(const any& x) const
{
if (policy == x.policy)
{
assign(x);
}
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1), x.policy->get_value(&p2), policy->get_size()) < 0 ? 1 : 0;
}
return 0;
}
/// Empty constructor.
any()
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
/// equal operator
bool operator==(const any& x) const
{
if (policy == x.policy)
{
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1), x.policy->get_value(&p2), policy->get_size()) == 0 ? 1 : 0;
}
return 0;
}
/// Special initializing constructor for string literals.
any(const char* x)
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Utility functions
uint8_t getHash() const
{
void* p1 = const_cast<void*>(object);
return *(uint64_t*)policy->get_value(&p1) % 4048;
}
any& swap(any& x)
{
std::swap(policy, x.policy);
std::swap(object, x.object);
return *this;
}
/// Copy constructor.
any(const any& x)
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Cast operator. You can only cast to the original type.
template <typename T>
T& cast()
{
if (policy != anyimpl::get_policy<T>())
throw std::runtime_error("static_any: type mismatch in cast");
T* r = reinterpret_cast<T*>(policy->get_value(&object));
return *r;
}
/// Destructor.
~any()
{
policy->static_delete(&object);
}
/// Returns true if the any contains no value.
bool empty() const
{
return policy == anyimpl::get_policy<anyimpl::empty_any>();
}
/// Assignment function from another any.
any& assign(const any& x)
{
reset();
policy = x.policy;
policy->clone(&x.object, &object);
return *this;
}
/// Frees any allocated memory, and sets the value to NULL.
void reset()
{
policy->static_delete(&object);
policy = anyimpl::get_policy<anyimpl::empty_any>();
}
/// Assignment function.
template <typename T>
any& assign(const T& x)
{
reset();
policy = anyimpl::get_policy<T>();
policy->copy_from_value(&x, &object);
return *this;
}
/// Assignment operator.
template<typename T>
any& operator=(const T& x) {
return assign(x);
}
/// Assignment operator, specialed for literal strings.
/// They have types like const char [6] which don't work as expected.
any& operator=(const char* x) {
return assign(x);
}
/// Less than operator for sorting
bool operator<(const any& x) const
{
if (policy == x.policy)
{
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1),
x.policy->get_value(&p2),
policy->get_size()) < 0 ? 1 : 0;
}
return 0;
}
/// equal operator
bool operator==(const any& x) const
{
if (policy == x.policy)
{
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1),
x.policy->get_value(&p2),
policy->get_size()) == 0 ? 1 : 0;
}
return 0;
}
/// Utility functions
uint8_t getHash() const
{
void* p1 = const_cast<void*>(object);
return *(uint64_t*)policy->get_value(&p1) % 4048;
}
any& swap(any& x)
{
std::swap(policy, x.policy);
std::swap(object, x.object);
return *this;
}
/// Cast operator. You can only cast to the original type.
template<typename T>
T& cast()
{
if (policy != anyimpl::get_policy<T>())
throw std::runtime_error("static_any: type mismatch in cast");
T* r = reinterpret_cast<T*>(policy->get_value(&object));
return *r;
}
/// Returns true if the any contains no value.
bool empty() const
{
return policy == anyimpl::get_policy<anyimpl::empty_any>();
}
/// Frees any allocated memory, and sets the value to NULL.
void reset()
{
policy->static_delete(&object);
policy = anyimpl::get_policy<anyimpl::empty_any>();
}
/// Returns true if the two types are the same.
bool compatible(const any& x) const
{
return policy == x.policy;
}
/// Returns true if the two types are the same.
bool compatible(const any& x) const
{
return policy == x.policy;
}
};
}
} // namespace static_any

123
utils/common/atomicops.h
View File

@ -34,108 +34,97 @@ Boost has something in interprocess::ipcdetail, but it doesn't have 64-bit API's
namespace atomicops
{
//Returns the resulting, incremented value
// Returns the resulting, incremented value
template <typename T>
inline T atomicInc(volatile T* mem)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
return InterlockedIncrement(reinterpret_cast<volatile LONG*>(mem));
switch (sizeof(T))
{
case 4:
default: return InterlockedIncrement(reinterpret_cast<volatile LONG*>(mem));
case 8:
return InterlockedIncrement64(reinterpret_cast<volatile LONGLONG*>(mem));
}
case 8: return InterlockedIncrement64(reinterpret_cast<volatile LONGLONG*>(mem));
}
#else
return __sync_add_and_fetch(mem, 1);
return __sync_add_and_fetch(mem, 1);
#endif
}
//decrements, but returns the pre-decrement value
// decrements, but returns the pre-decrement value
template <typename T>
inline T atomicDec(volatile T* mem)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
return InterlockedDecrement(reinterpret_cast<volatile LONG*>(mem)) + 1;
switch (sizeof(T))
{
case 4:
default: return InterlockedDecrement(reinterpret_cast<volatile LONG*>(mem)) + 1;
case 8:
return InterlockedDecrement64(reinterpret_cast<volatile LONGLONG*>(mem)) + 1;
}
case 8: return InterlockedDecrement64(reinterpret_cast<volatile LONGLONG*>(mem)) + 1;
}
#else
return __sync_fetch_and_add(mem, -1);
return __sync_fetch_and_add(mem, -1);
#endif
}
//Returns the resulting value (but doesn't need to yet)
// Returns the resulting value (but doesn't need to yet)
template <typename T>
inline T atomicAdd(volatile T* mem, T val)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), val);
break;
switch (sizeof(T))
{
case 4:
default: InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), val); break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), val);
break;
}
case 8: InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), val); break;
}
return *mem;
return *mem;
#else
return __sync_add_and_fetch(mem, val);
return __sync_add_and_fetch(mem, val);
#endif
}
//Returns the resulting value
// Returns the resulting value
template <typename T>
inline T atomicSub(volatile T* mem, T val)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), -(static_cast<LONG>(val)));
break;
switch (sizeof(T))
{
case 4:
default: InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), -(static_cast<LONG>(val))); break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), -(static_cast<LONGLONG>(val)));
break;
}
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), -(static_cast<LONGLONG>(val)));
break;
}
return *mem;
return *mem;
#else
return __sync_sub_and_fetch(mem, val);
return __sync_sub_and_fetch(mem, val);
#endif
}
//Implements a memory barrier
// Implements a memory barrier
inline void atomicMb()
{
#ifdef _MSC_VER
MemoryBarrier();
MemoryBarrier();
#else
__sync_synchronize();
__sync_synchronize();
#endif
}
//Returns true iff the CAS took place, that is
// Returns true iff the CAS took place, that is
// if (*mem == comp) {
// *mem = swap;
// return true;
@ -147,32 +136,32 @@ inline bool atomicCAS(volatile T* mem, T comp, T swap)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
//The function returns the initial value of the mem parameter
return (InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(mem), swap, comp) == comp);
switch (sizeof(T))
{
case 4:
default:
// The function returns the initial value of the mem parameter
return (InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(mem), swap, comp) == comp);
case 8:
return (InterlockedCompareExchange64(reinterpret_cast<volatile LONGLONG*>(mem), swap, comp) == comp);
}
case 8:
return (InterlockedCompareExchange64(reinterpret_cast<volatile LONGLONG*>(mem), swap, comp) == comp);
}
#else
//If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is successful and swap was written.
return __sync_bool_compare_and_swap(mem, comp, swap);
// If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is
// successful and swap was written.
return __sync_bool_compare_and_swap(mem, comp, swap);
#endif
}
//Implements a scheduler yield
// Implements a scheduler yield
inline void atomicYield()
{
#ifdef _MSC_VER
SwitchToThread();
SwitchToThread();
#else
sched_yield();
sched_yield();
#endif
}
}
} // namespace atomicops

6
utils/common/branchpred.h
View File

@ -26,8 +26,6 @@
#endif
#ifndef LIKELY
#define LIKELY(x) __builtin_expect((x),1)
#define UNLIKELY(x) __builtin_expect((x),0)
#define LIKELY(x) __builtin_expect((x), 1)
#define UNLIKELY(x) __builtin_expect((x), 0)
#endif

681
utils/common/cgroupconfigurator.cpp
View File

@ -30,58 +30,61 @@
using namespace boost;
using namespace std;
// minor space-savers
#define RETURN_NO_GROUP(err) do { \
if (!printedWarning) { \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! The group " << cGroupName << " does not exist."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
#define RETURN_NO_GROUP(err) \
do \
{ \
if (!printedWarning) \
{ \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! The group " << cGroupName << " does not exist."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
#define RETURN_READ_ERROR(err) do { \
if (!printedWarning) { \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! Could not read the file " << filename << "."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
#define RETURN_READ_ERROR(err) \
do \
{ \
if (!printedWarning) \
{ \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! Could not read the file " << filename << "."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
namespace
{
void log(logging::LOG_TYPE whichLogFile, const string& msg)
{
logging::Logger logger(12); //12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
logging::Logger logger(12); // 12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
}
}
} // namespace
namespace utils
{
CGroupConfigurator::CGroupConfigurator()
{
config = config::Config::makeConfig();
config = config::Config::makeConfig();
cGroupName = config->getConfig("SystemConfig", "CGroup");
cGroupName = config->getConfig("SystemConfig", "CGroup");
if (cGroupName.empty())
cGroupDefined = false;
else
cGroupDefined = true;
if (cGroupName.empty())
cGroupDefined = false;
else
cGroupDefined = true;
totalMemory = 0;
totalSwap = 0;
printedWarning = false;
totalMemory = 0;
totalSwap = 0;
printedWarning = false;
}
CGroupConfigurator::~CGroupConfigurator()
@ -90,445 +93,441 @@ CGroupConfigurator::~CGroupConfigurator()
uint32_t CGroupConfigurator::getNumCoresFromCGroup()
{
ostringstream filename_os;
filename_os << "/sys/fs/cgroup/cpuset/" << cGroupName << "/cpus";
string filename = filename_os.str();
ostringstream filename_os;
filename_os << "/sys/fs/cgroup/cpuset/" << cGroupName << "/cpus";
string filename = filename_os.str();
ifstream in(filename.c_str());
string cpusString;
uint32_t cpus = 0;
ifstream in(filename.c_str());
string cpusString;
uint32_t cpus = 0;
if (!in)
RETURN_NO_GROUP(0);
if (!in)
RETURN_NO_GROUP(0);
try
try
{
// Need to parse & count how many CPUs we have access to
in >> cpusString;
}
catch (...)
{
RETURN_READ_ERROR(0);
}
// the file has comma-deliminted CPU ranges like "0-7,9,11-12".
size_t first = 0, last;
bool lastRange = false;
while (!lastRange)
{
size_t dash;
string oneRange;
last = cpusString.find(',', first);
if (last == string::npos)
{
// Need to parse & count how many CPUs we have access to
in >> cpusString;
lastRange = true;
oneRange = cpusString.substr(first);
}
catch (...)
else
oneRange = cpusString.substr(first, last - first - 1);
if ((dash = oneRange.find('-')) == string::npos) // single-cpu range
cpus++;
else
{
RETURN_READ_ERROR(0);
const char* data = oneRange.c_str();
uint32_t firstCPU = strtol(data, NULL, 10);
uint32_t lastCPU = strtol(&data[dash + 1], NULL, 10);
cpus += lastCPU - firstCPU + 1;
}
// the file has comma-deliminted CPU ranges like "0-7,9,11-12".
size_t first = 0, last;
bool lastRange = false;
first = last + 1;
}
while (!lastRange)
{
size_t dash;
string oneRange;
last = cpusString.find(',', first);
if (last == string::npos)
{
lastRange = true;
oneRange = cpusString.substr(first);
}
else
oneRange = cpusString.substr(first, last - first - 1);
if ((dash = oneRange.find('-')) == string::npos) // single-cpu range
cpus++;
else
{
const char* data = oneRange.c_str();
uint32_t firstCPU = strtol(data, NULL, 10);
uint32_t lastCPU = strtol(&data[dash + 1], NULL, 10);
cpus += lastCPU - firstCPU + 1;
}
first = last + 1;
}
//cout << "found " << cpus << " CPUS in the string " << cpusString << endl;
return cpus;
// cout << "found " << cpus << " CPUS in the string " << cpusString << endl;
return cpus;
}
uint32_t CGroupConfigurator::getNumCoresFromProc()
{
#ifdef _MSC_VER
SYSTEM_INFO siSysInfo;
GetSystemInfo(&siSysInfo);
return siSysInfo.dwNumberOfProcessors;
SYSTEM_INFO siSysInfo;
GetSystemInfo(&siSysInfo);
return siSysInfo.dwNumberOfProcessors;
#else
uint32_t nc = sysconf(_SC_NPROCESSORS_ONLN);
uint32_t nc = sysconf(_SC_NPROCESSORS_ONLN);
return nc;
return nc;
#endif
}
uint32_t CGroupConfigurator::getNumCores()
{
/*
Detect if InfiniDB is in a C-Group
- get the group ID
If not, get the number of cores from /proc
*/
uint32_t ret;
/*
Detect if InfiniDB is in a C-Group
- get the group ID
If not, get the number of cores from /proc
*/
uint32_t ret;
if (!cGroupDefined)
ret = getNumCoresFromProc();
else
{
ret = getNumCoresFromCGroup();
if (!cGroupDefined)
ret = getNumCoresFromProc();
else
{
ret = getNumCoresFromCGroup();
if (ret == 0)
ret = getNumCoresFromProc();
}
if (ret == 0)
ret = getNumCoresFromProc();
}
//cout << "There are " << ret << " cores available" << endl;
return ret;
// cout << "There are " << ret << " cores available" << endl;
return ret;
}
uint64_t CGroupConfigurator::getTotalMemory()
{
uint64_t ret;
uint64_t ret;
if (totalMemory != 0)
return totalMemory;
if (!cGroupDefined)
ret = getTotalMemoryFromProc();
else
{
ret = getTotalMemoryFromCGroup();
if (ret == 0)
ret = getTotalMemoryFromProc();
}
//cout << "Total mem available is " << ret << endl;
totalMemory = ret;
if (totalMemory != 0)
return totalMemory;
if (!cGroupDefined)
ret = getTotalMemoryFromProc();
else
{
ret = getTotalMemoryFromCGroup();
if (ret == 0)
ret = getTotalMemoryFromProc();
}
// cout << "Total mem available is " << ret << endl;
totalMemory = ret;
return totalMemory;
}
uint64_t CGroupConfigurator::getTotalMemoryFromProc()
{
size_t memTot;
size_t memTot;
#if defined(_MSC_VER)
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat) == 0)
//FIXME: Assume 2GB?
memTot = 2 * 1024 * 1024;
else
{
if (GlobalMemoryStatusEx(&memStat) == 0)
// FIXME: Assume 2GB?
memTot = 2 * 1024 * 1024;
else
{
#ifndef _WIN64
memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys);
memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys);
#endif
//We now have the total phys mem in bytes
memTot = memStat.ullTotalPhys / 1024;
}
// We now have the total phys mem in bytes
memTot = memStat.ullTotalPhys / 1024;
}
#elif defined(__FreeBSD__)
string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'");
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
memTot = atoi(input);
string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'");
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
memTot = atoi(input);
#else
ifstream in("/proc/meminfo");
string x;
ifstream in("/proc/meminfo");
string x;
in >> x;
in >> memTot;
in >> x;
in >> memTot;
#endif
//memTot is now in KB, convert to bytes
memTot *= 1024;
// memTot is now in KB, convert to bytes
memTot *= 1024;
return memTot;
return memTot;
}
uint64_t CGroupConfigurator::getTotalMemoryFromCGroup()
{
ifstream in;
uint64_t ret;
ostringstream os;
string filename;
ifstream in;
uint64_t ret;
ostringstream os;
string filename;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.limit_in_bytes";
filename = os.str();
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.limit_in_bytes";
filename = os.str();
in.open(filename.c_str());
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(0);
if (!in)
RETURN_NO_GROUP(0);
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(0);
}
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(0);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getFreeMemory()
{
uint64_t ret;
uint64_t ret;
if (!cGroupDefined)
ret = getFreeMemoryFromProc();
if (!cGroupDefined)
ret = getFreeMemoryFromProc();
else
{
uint64_t usage = getMemUsageFromCGroup();
if (usage == 0)
ret = getFreeMemoryFromProc();
else
{
uint64_t usage = getMemUsageFromCGroup();
ret = getTotalMemory() - usage;
}
if (usage == 0)
ret = getFreeMemoryFromProc();
else
ret = getTotalMemory() - usage;
}
//cout << "free memory = " << ret << endl;
return ret;
// cout << "free memory = " << ret << endl;
return ret;
}
uint64_t CGroupConfigurator::getMemUsageFromCGroup()
{
uint64_t ret = 0;
bool found = false;
char oneline[80];
uint64_t ret = 0;
bool found = false;
char oneline[80];
if (memUsageFilename.empty())
if (memUsageFilename.empty())
{
ostringstream filename;
filename << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
memUsageFilename = filename.str();
}
ifstream in(memUsageFilename.c_str());
string& filename = memUsageFilename;
if (!in)
RETURN_NO_GROUP(0);
try
{
while (in && !found)
{
ostringstream filename;
filename << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
memUsageFilename = filename.str();
in.getline(oneline, 80);
if (strncmp(oneline, "rss", 2) == 0)
{
ret = atoll(&oneline[3]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(0);
}
ifstream in(memUsageFilename.c_str());
string& filename = memUsageFilename;
if (!in)
RETURN_NO_GROUP(0);
try
{
while (in && !found)
{
in.getline(oneline, 80);
if (strncmp(oneline, "rss", 2) == 0)
{
ret = atoll(&oneline[3]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(0);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getFreeMemoryFromProc()
{
uint64_t memFree = 0;
uint64_t buffers = 0;
uint64_t cached = 0;
uint64_t memTotal = 0;
uint64_t memAvailable = 0;
uint64_t memFree = 0;
uint64_t buffers = 0;
uint64_t cached = 0;
uint64_t memTotal = 0;
uint64_t memAvailable = 0;
#if defined(_MSC_VER)
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat))
{
memAvailable = memStat.ullAvailPhys;
if (GlobalMemoryStatusEx(&memStat))
{
memAvailable = memStat.ullAvailPhys;
#ifndef _WIN64
uint64_t tmp = getTotalMemoryFromProc();
uint64_t tmp = getTotalMemoryFromProc();
if (memFree > tmp)
memAvailable = tmp;
if (memFree > tmp)
memAvailable = tmp;
#endif
}
}
#elif defined(__FreeBSD__)
// FreeBSD is not supported, no optimization.
memAvailable = 0;
// FreeBSD is not supported, no optimization.
memAvailable = 0;
#else
ifstream in("/proc/meminfo");
string x;
ifstream in("/proc/meminfo");
string x;
in >> x; // MemTotal:
in >> memTotal;
in >> x; // kB
in >> x; // MemTotal:
in >> memTotal;
in >> x; // kB
in >> x; // MemFree:
in >> memFree;
in >> x; // kB
in >> x; // MemFree:
in >> memFree;
in >> x; // kB
//check if available or buffers is passed
in >> x;
// check if available or buffers is passed
in >> x;
if ( x == "MemAvailable:")
{
in >> memAvailable; // MemAvailable
}
else
{
// centos 6 and older OSs
in >> buffers;
in >> x; // kB
if (x == "MemAvailable:")
{
in >> memAvailable; // MemAvailable
}
else
{
// centos 6 and older OSs
in >> buffers;
in >> x; // kB
in >> x; // Cached:
in >> cached;
in >> x; // Cached:
in >> cached;
memAvailable = memFree + buffers + cached;
}
memAvailable = memFree + buffers + cached;
}
#endif
// amount available for application
memAvailable *= 1024;
return memAvailable;
// amount available for application
memAvailable *= 1024;
return memAvailable;
}
uint64_t CGroupConfigurator::getTotalSwapSpace()
{
int64_t ret;
int64_t ret;
if (totalSwap != 0)
return totalSwap;
if (totalSwap != 0)
return totalSwap;
if (!cGroupDefined)
ret = getTotalSwapFromSysinfo();
if (!cGroupDefined)
ret = getTotalSwapFromSysinfo();
else
{
ret = getTotalMemAndSwapFromCGroup();
// if no limit is set in the cgroup, the file contains maxint64. Use sysinfo in that case.
if (ret == -1 || ret == numeric_limits<int64_t>::max())
ret = getTotalSwapFromSysinfo();
else
{
ret = getTotalMemAndSwapFromCGroup();
ret -= getTotalMemory();
}
// if no limit is set in the cgroup, the file contains maxint64. Use sysinfo in that case.
if (ret == -1 || ret == numeric_limits<int64_t>::max())
ret = getTotalSwapFromSysinfo();
else
ret -= getTotalMemory();
}
//cout << "total swap=" << ret << endl;
totalSwap = ret;
return ret;
// cout << "total swap=" << ret << endl;
totalSwap = ret;
return ret;
}
uint64_t CGroupConfigurator::getTotalSwapFromSysinfo()
{
struct sysinfo si;
struct sysinfo si;
sysinfo(&si);
return si.totalswap;
sysinfo(&si);
return si.totalswap;
}
int64_t CGroupConfigurator::getTotalMemAndSwapFromCGroup()
{
int64_t ret;
ifstream in;
string filename;
ostringstream os;
int64_t ret;
ifstream in;
string filename;
ostringstream os;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.memsw.limit_in_bytes";
filename = os.str();
in.open(filename.c_str());
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.memsw.limit_in_bytes";
filename = os.str();
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
if (!in)
RETURN_NO_GROUP(-1);
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getSwapInUse()
{
int64_t ret;
int64_t ret;
if (!cGroupDefined)
ret = getSwapInUseFromSysinfo();
else
{
ret = getSwapInUseFromCGroup();
if (!cGroupDefined)
ret = getSwapInUseFromSysinfo();
else
{
ret = getSwapInUseFromCGroup();
if (ret == -1)
ret = getSwapInUseFromSysinfo();
}
if (ret == -1)
ret = getSwapInUseFromSysinfo();
}
//cout << "current swap in use=" << ret << endl;
return ret;
// cout << "current swap in use=" << ret << endl;
return ret;
}
int64_t CGroupConfigurator::getSwapInUseFromCGroup()
{
int64_t ret = -1;
ifstream in;
bool found = false;
char oneline[80];
int64_t ret = -1;
ifstream in;
bool found = false;
char oneline[80];
if (usedSwapFilename.empty())
if (usedSwapFilename.empty())
{
ostringstream os;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
usedSwapFilename = os.str();
}
string& filename = usedSwapFilename;
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
try
{
while (in && !found)
{
ostringstream os;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
usedSwapFilename = os.str();
in.getline(oneline, 80);
if (strncmp(oneline, "swap", 4) == 0)
{
ret = atoll(&oneline[5]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
string& filename = usedSwapFilename;
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
try
{
while (in && !found)
{
in.getline(oneline, 80);
if (strncmp(oneline, "swap", 4) == 0)
{
ret = atoll(&oneline[5]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getSwapInUseFromSysinfo()
{
struct sysinfo si;
struct sysinfo si;
sysinfo(&si);
return si.totalswap - si.freeswap;
sysinfo(&si);
return si.totalswap - si.freeswap;
}
}
} // namespace utils

67
utils/common/cgroupconfigurator.h
View File

@ -25,50 +25,47 @@
namespace utils
{
/* This class wraps a few methods for getting configuration variables that potentially
come from a cgroup. Might move it to utils/configcpp, and/or change the name. */
class CGroupConfigurator
{
public:
CGroupConfigurator();
virtual ~CGroupConfigurator();
public:
CGroupConfigurator();
virtual ~CGroupConfigurator();
uint32_t getNumCores();
uint64_t getTotalMemory();
uint64_t getFreeMemory();
uint64_t getTotalSwapSpace();
uint64_t getSwapInUse();
uint32_t getNumCores();
uint64_t getTotalMemory();
uint64_t getFreeMemory();
uint64_t getTotalSwapSpace();
uint64_t getSwapInUse();
bool usingCGroup()
{
return cGroupDefined;
}
bool usingCGroup()
{
return cGroupDefined;
}
private:
uint32_t getNumCoresFromProc();
uint32_t getNumCoresFromCGroup();
uint64_t getTotalMemoryFromProc();
uint64_t getTotalMemoryFromCGroup();
uint64_t getFreeMemoryFromProc();
uint64_t getMemUsageFromCGroup();
uint64_t getTotalSwapFromSysinfo();
int64_t getTotalMemAndSwapFromCGroup();
uint64_t getSwapInUseFromSysinfo();
int64_t getSwapInUseFromCGroup();
private:
uint32_t getNumCoresFromProc();
uint32_t getNumCoresFromCGroup();
uint64_t getTotalMemoryFromProc();
uint64_t getTotalMemoryFromCGroup();
uint64_t getFreeMemoryFromProc();
uint64_t getMemUsageFromCGroup();
uint64_t getTotalSwapFromSysinfo();
int64_t getTotalMemAndSwapFromCGroup();
uint64_t getSwapInUseFromSysinfo();
int64_t getSwapInUseFromCGroup();
std::string memUsageFilename;
std::string usedSwapFilename;
std::string cGroupName;
bool cGroupDefined;
config::Config* config;
uint64_t totalMemory;
uint64_t totalSwap;
bool printedWarning;
std::string memUsageFilename;
std::string usedSwapFilename;
std::string cGroupName;
bool cGroupDefined;
config::Config* config;
uint64_t totalMemory;
uint64_t totalSwap;
bool printedWarning;
};
}
} // namespace utils

45
utils/common/checks.h
View File

@ -19,31 +19,34 @@
#include <type_traits>
#include "mcs_int128.h"
namespace utils {
namespace utils
{
template <typename T>
typename std::enable_if<std::is_unsigned<T>::value || datatypes::is_uint128_t<T>::value, bool>::type
is_nonnegative(T)
{
return true;
};
template <typename T>
typename std::enable_if<std::is_unsigned<T>::value ||
datatypes::is_uint128_t<T>::value,
bool>::type
is_nonnegative(T) { return true; };
typename std::enable_if<std::is_signed<T>::value || datatypes::is_int128_t<T>::value, bool>::type
is_nonnegative(T v)
{
return v >= 0;
};
template <typename T>
typename std::enable_if<std::is_signed<T>::value ||
datatypes::is_int128_t<T>::value,
bool>::type
is_nonnegative(T v) { return v >= 0; };
typename std::enable_if<std::is_unsigned<T>::value || datatypes::is_uint128_t<T>::value, bool>::type
is_negative(T)
{
return false;
};
template <typename T>
typename std::enable_if<std::is_unsigned<T>::value ||
datatypes::is_uint128_t<T>::value,
bool>::type
is_negative(T) { return false; };
template <typename T>
typename std::enable_if<std::is_signed<T>::value ||
datatypes::is_int128_t<T>::value,
bool>::type
is_negative(T v) { return v < 0; };
} // namespace utils
typename std::enable_if<std::is_signed<T>::value || datatypes::is_int128_t<T>::value, bool>::type is_negative(
T v)
{
return v < 0;
};
} // namespace utils

235
utils/common/collation.h
View File

@ -19,18 +19,18 @@
#if defined(PREFER_MY_CONFIG_H)
#if !defined(MY_CONFIG_H)
#error my_config.h was not included (but PREFER_MY_CONFIG_H was set)
#endif
#if !defined(MY_CONFIG_H)
#error my_config.h was not included (but PREFER_MY_CONFIG_H was set)
#endif
#include "mcsconfig_conflicting_defs_remember.h"
#include "mcsconfig_conflicting_defs_undef.h"
#include "mcsconfig_conflicting_defs_remember.h"
#include "mcsconfig_conflicting_defs_undef.h"
#else
#if defined(MY_CONFIG_H)
#error my_config.h was included before mcsconfig.h (and PREFER_MY_CONFIG_H was not set)
#endif
#endif //PREFER_MY_CONFIG_H
#if defined(MY_CONFIG_H)
#error my_config.h was included before mcsconfig.h (and PREFER_MY_CONFIG_H was not set)
#endif
#endif // PREFER_MY_CONFIG_H
#include "mcsconfig.h"
@ -48,11 +48,11 @@
#endif
#ifndef TRUE
#define TRUE (1)
#define TRUE (1)
#endif
#ifndef DBUG_ASSERT
#define DBUG_ASSERT(x) idbassert(x)
#define DBUG_ASSERT(x) idbassert(x)
#define DBUG_ASSERT_TEMPORARILY_DEFINED
#endif
@ -72,21 +72,21 @@ typedef char my_bool;
typedef unsigned char uchar;
#if defined(__GNUC__) && !defined(_lint)
typedef char pchar; /* Mixed prototypes can take char */
typedef char puchar; /* Mixed prototypes can take char */
typedef char pbool; /* Mixed prototypes can take char */
typedef short pshort; /* Mixed prototypes can take short int */
typedef float pfloat; /* Mixed prototypes can take float */
typedef char pchar; /* Mixed prototypes can take char */
typedef char puchar; /* Mixed prototypes can take char */
typedef char pbool; /* Mixed prototypes can take char */
typedef short pshort; /* Mixed prototypes can take short int */
typedef float pfloat; /* Mixed prototypes can take float */
#else
typedef int pchar; /* Mixed prototypes can't take char */
typedef uint puchar; /* Mixed prototypes can't take char */
typedef int pbool; /* Mixed prototypes can't take char */
typedef int pshort; /* Mixed prototypes can't take short int */
typedef double pfloat; /* Mixed prototypes can't take float */
typedef int pchar; /* Mixed prototypes can't take char */
typedef uint puchar; /* Mixed prototypes can't take char */
typedef int pbool; /* Mixed prototypes can't take char */
typedef int pshort; /* Mixed prototypes can't take short int */
typedef double pfloat; /* Mixed prototypes can't take float */
#endif
typedef const struct charset_info_st CHARSET_INFO;
extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO *default_charset_info;
extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO* default_charset_info;
#include "m_ctype.h"
@ -97,127 +97,114 @@ extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO *default_charset_info;
#undef DBUG_ASSERT
#endif
#if defined(PREFER_MY_CONFIG_H)
#include "mcsconfig_conflicting_defs_restore.h"
#include "mcsconfig_conflicting_defs_restore.h"
#endif
namespace datatypes
{
class MariaDBHasher
{
ulong mPart1;
ulong mPart2;
public:
MariaDBHasher()
:mPart1(1), mPart2(4)
{ }
MariaDBHasher & add(CHARSET_INFO * cs, const char *str, size_t length)
{
cs->hash_sort((const uchar *) str, length, &mPart1, &mPart2);
return *this;
}
MariaDBHasher & add(CHARSET_INFO *cs, const utils::ConstString &str)
{
return add(cs, str.str(), str.length());
}
uint32_t finalize() const
{
return (uint32_t)mPart1;
}
};
ulong mPart1;
ulong mPart2;
public:
MariaDBHasher() : mPart1(1), mPart2(4)
{
}
MariaDBHasher& add(CHARSET_INFO* cs, const char* str, size_t length)
{
cs->hash_sort((const uchar*)str, length, &mPart1, &mPart2);
return *this;
}
MariaDBHasher& add(CHARSET_INFO* cs, const utils::ConstString& str)
{
return add(cs, str.str(), str.length());
}
uint32_t finalize() const
{
return (uint32_t)mPart1;
}
};
// A reference to MariaDB CHARSET_INFO.
class Charset
{
protected:
const struct charset_info_st * mCharset;
public:
Charset(CHARSET_INFO & cs) :mCharset(&cs) { }
Charset(CHARSET_INFO *cs)
:mCharset(cs ? cs : &my_charset_bin)
{ }
Charset(uint32_t charsetNumber);
CHARSET_INFO & getCharset() const { return *mCharset; }
uint32_t hash(const char *data, uint64_t len) const
{
return MariaDBHasher().add(mCharset, data, len).finalize();
}
bool eq(const std::string & str1, const std::string & str2) const
{
return mCharset->strnncollsp(str1.data(), str1.length(),
str2.data(), str2.length()) == 0;
}
int strnncollsp(const utils::ConstString &str1,
const utils::ConstString &str2) const
{
return mCharset->strnncollsp(str1.str(), str1.length(),
str2.str(), str2.length());
}
int strnncollsp(const char *str1, size_t length1,
const char *str2, size_t length2) const
{
return mCharset->strnncollsp(str1, length1, str2, length2);
}
int strnncollsp(const unsigned char *str1, size_t length1,
const unsigned char *str2, size_t length2) const
{
return mCharset->strnncollsp((const char *) str1, length1,
(const char *) str2, length2);
}
bool test_if_important_data(const char *str, const char *end) const
{
if (mCharset->state & MY_CS_NOPAD)
return str < end;
return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end;
protected:
const struct charset_info_st* mCharset;
}
bool like(bool neg,
const utils::ConstString &subject,
const utils::ConstString &pattern) const
{
bool res= !mCharset->wildcmp(subject.str(), subject.end(),
pattern.str(), pattern.end(),
'\\','_','%');
return neg ? !res : res;
}
public:
Charset(CHARSET_INFO& cs) : mCharset(&cs)
{
}
Charset(CHARSET_INFO* cs) : mCharset(cs ? cs : &my_charset_bin)
{
}
Charset(uint32_t charsetNumber);
CHARSET_INFO& getCharset() const
{
return *mCharset;
}
uint32_t hash(const char* data, uint64_t len) const
{
return MariaDBHasher().add(mCharset, data, len).finalize();
}
bool eq(const std::string& str1, const std::string& str2) const
{
return mCharset->strnncollsp(str1.data(), str1.length(), str2.data(), str2.length()) == 0;
}
int strnncollsp(const utils::ConstString& str1, const utils::ConstString& str2) const
{
return mCharset->strnncollsp(str1.str(), str1.length(), str2.str(), str2.length());
}
int strnncollsp(const char* str1, size_t length1, const char* str2, size_t length2) const
{
return mCharset->strnncollsp(str1, length1, str2, length2);
}
int strnncollsp(const unsigned char* str1, size_t length1, const unsigned char* str2, size_t length2) const
{
return mCharset->strnncollsp((const char*)str1, length1, (const char*)str2, length2);
}
bool test_if_important_data(const char* str, const char* end) const
{
if (mCharset->state & MY_CS_NOPAD)
return str < end;
return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end;
}
bool like(bool neg, const utils::ConstString& subject, const utils::ConstString& pattern) const
{
bool res = !mCharset->wildcmp(subject.str(), subject.end(), pattern.str(), pattern.end(), '\\', '_', '%');
return neg ? !res : res;
}
};
class CollationAwareHasher: public Charset
class CollationAwareHasher : public Charset
{
public:
CollationAwareHasher(const Charset &cs)
:Charset(cs)
{ }
inline uint32_t operator()(const std::string& s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const char* data, uint64_t len) const
{
return Charset::hash(data, len);
}
public:
CollationAwareHasher(const Charset& cs) : Charset(cs)
{
}
inline uint32_t operator()(const std::string& s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const char* data, uint64_t len) const
{
return Charset::hash(data, len);
}
};
class CollationAwareComparator: public Charset
class CollationAwareComparator : public Charset
{
public:
CollationAwareComparator(const Charset &cs)
:Charset(cs)
{ }
bool operator()(const std::string & str1, const std::string & str2) const
{
return Charset::eq(str1, str2);
}
public:
CollationAwareComparator(const Charset& cs) : Charset(cs)
{
}
bool operator()(const std::string& str1, const std::string& str2) const
{
return Charset::eq(str1, str2);
}
};
} // end of namespace datatypes
} // end of namespace datatypes

121
utils/common/columnwidth.h
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2020 MariaDB Corporation
/* Copyright (C) 2020 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -21,70 +21,61 @@
namespace utils
{
const uint8_t MAXLEGACYWIDTH = 8ULL;
const uint8_t MAXCOLUMNWIDTH = 16ULL;
inline bool isWide(uint8_t width)
{
return width > MAXLEGACYWIDTH;
}
inline bool isNarrow(uint8_t width)
{
return width <= MAXLEGACYWIDTH;
}
/** @brief Map a DECIMAL precision to data width in bytes */
inline uint8_t widthByPrecision(unsigned p)
{
if (LIKELY(p > 18 && p < 39))
return 16;
switch (p)
{
case 1:
case 2:
return 1;
case 3:
case 4:
return 2;
case 5:
case 6:
case 7:
case 8:
case 9:
return 4;
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
case 16:
case 17:
case 18:
return 8;
default:
return 16;
}
}
inline uint8_t precisionByWidth(unsigned w)
{
switch(w)
{
case 16:
return 38;
// In case we will support decimals that spans 32 bytes.
default:
return 65;
}
}
const uint8_t MAXLEGACYWIDTH = 8ULL;
const uint8_t MAXCOLUMNWIDTH = 16ULL;
inline bool isWide(uint8_t width)
{
return width > MAXLEGACYWIDTH;
}
inline bool isNarrow(uint8_t width)
{
return width <= MAXLEGACYWIDTH;
}
/** @brief Map a DECIMAL precision to data width in bytes */
inline uint8_t widthByPrecision(unsigned p)
{
if (LIKELY(p > 18 && p < 39))
return 16;
switch (p)
{
case 1:
case 2: return 1;
case 3:
case 4: return 2;
case 5:
case 6:
case 7:
case 8:
case 9: return 4;
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
case 16:
case 17:
case 18: return 8;
default: return 16;
}
}
inline uint8_t precisionByWidth(unsigned w)
{
switch (w)
{
case 16: return 38;
// In case we will support decimals that spans 32 bytes.
default: return 65;
}
}
} // namespace utils

50
utils/common/conststring.h
View File

@ -15,31 +15,38 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <string>
#include <string.h>
namespace utils
{
class ConstString
{
protected:
const char *mStr;
protected:
const char* mStr;
size_t mLength;
public:
ConstString(const char *str, size_t length)
:mStr(str), mLength(length)
{ }
explicit ConstString(const std::string &str)
:mStr(str.data()), mLength(str.length())
{ }
const char *str() const { return mStr; }
const char *end() const { return mStr + mLength; }
size_t length() const { return mLength; }
public:
ConstString(const char* str, size_t length) : mStr(str), mLength(length)
{
}
explicit ConstString(const std::string& str) : mStr(str.data()), mLength(str.length())
{
}
const char* str() const
{
return mStr;
}
const char* end() const
{
return mStr + mLength;
}
size_t length() const
{
return mLength;
}
std::string toString() const
{
return std::string(mStr, mLength);
@ -48,18 +55,17 @@ public:
{
return mLength == 1 && mStr[0] == ch;
}
bool eq(const ConstString &rhs) const
bool eq(const ConstString& rhs) const
{
return mLength == rhs.mLength && !memcmp(mStr, rhs.mStr, mLength);
}
ConstString & rtrimZero()
ConstString& rtrimZero()
{
for ( ; mLength && mStr[mLength - 1] == '\0'; mLength--)
{ }
for (; mLength && mStr[mLength - 1] == '\0'; mLength--)
{
}
return *this;
}
};
} // namespace utils
} // namespace utils

44
utils/common/crashtrace.cpp
View File

@ -27,26 +27,26 @@
void fatalHandler(int sig)
{
char filename[128];
void* addrs[128];
snprintf(filename, 128, "%s/trace/%s.%d.log", MCSLOGDIR, program_invocation_short_name, getpid());
FILE* logfile = fopen(filename, "w");
char s[30];
struct tm tim;
time_t now;
now = time(NULL);
tim = *(localtime(&now));
strftime(s, 30, "%F %T", &tim);
fprintf(logfile, "Date/time: %s\n", s);
fprintf(logfile, "Signal: %d\n\n", sig);
fflush(logfile);
int fd = fileno(logfile);
int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0]));
backtrace_symbols_fd(addrs, count, fd);
fclose(logfile);
struct sigaction sigact;
memset(&sigact, 0, sizeof(sigact));
sigact.sa_handler = SIG_DFL;
sigaction(sig, &sigact, NULL);
raise(sig);
char filename[128];
void* addrs[128];
snprintf(filename, 128, "%s/trace/%s.%d.log", MCSLOGDIR, program_invocation_short_name, getpid());
FILE* logfile = fopen(filename, "w");
char s[30];
struct tm tim;
time_t now;
now = time(NULL);
tim = *(localtime(&now));
strftime(s, 30, "%F %T", &tim);
fprintf(logfile, "Date/time: %s\n", s);
fprintf(logfile, "Signal: %d\n\n", sig);
fflush(logfile);
int fd = fileno(logfile);
int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0]));
backtrace_symbols_fd(addrs, count, fd);
fclose(logfile);
struct sigaction sigact;
memset(&sigact, 0, sizeof(sigact));
sigact.sa_handler = SIG_DFL;
sigaction(sig, &sigact, NULL);
raise(sig);
}

92
utils/common/fixedallocator.cpp
View File

@ -16,11 +16,11 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
//This is one of the first files we compile, check the compiler...
// This is one of the first files we compile, check the compiler...
#if defined(__GNUC__)
#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
#error "This is a very old GCC, and it's probably not going to work."
@ -42,80 +42,78 @@ using namespace boost;
namespace utils
{
FixedAllocator::FixedAllocator(const FixedAllocator& f)
{
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
capacityRemaining = 0;
currentlyStored = 0;
useLock = f.useLock;
lock = false;
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
capacityRemaining = 0;
currentlyStored = 0;
useLock = f.useLock;
lock = false;
}
FixedAllocator& FixedAllocator::operator=(const FixedAllocator& f)
{
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
useLock = f.useLock;
lock = false;
deallocateAll();
return *this;
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
useLock = f.useLock;
lock = false;
deallocateAll();
return *this;
}
void FixedAllocator::setUseLock(bool useIt)
{
useLock = useIt;
useLock = useIt;
}
void FixedAllocator::setAllocSize(uint allocSize)
{
elementSize = allocSize;
elementSize = allocSize;
}
void FixedAllocator::newBlock()
{
shared_array<uint8_t> next;
shared_array<uint8_t> next;
capacityRemaining = elementCount * elementSize;
capacityRemaining = elementCount * elementSize;
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[elementCount * elementSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
{
currentlyStored = 0;
nextAlloc = mem.front().get();
}
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[elementCount * elementSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
{
currentlyStored = 0;
nextAlloc = mem.front().get();
}
}
void FixedAllocator::truncateBy(uint32_t amt)
{
if (useLock)
getSpinlock(lock);
nextAlloc -= amt;
capacityRemaining += amt;
currentlyStored -= amt;
if (useLock)
releaseSpinlock(lock);
if (useLock)
getSpinlock(lock);
nextAlloc -= amt;
capacityRemaining += amt;
currentlyStored -= amt;
if (useLock)
releaseSpinlock(lock);
}
void FixedAllocator::deallocateAll()
{
mem.clear();
currentlyStored = 0;
capacityRemaining = 0;
mem.clear();
currentlyStored = 0;
capacityRemaining = 0;
}
uint64_t FixedAllocator::getMemUsage() const
{
return (mem.size() * elementCount * elementSize);
return (mem.size() * elementCount * elementSize);
}
}
} // namespace utils

152
utils/common/fixedallocator.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/* This allocator is for frequent small allocations that all get deallocated at once.
It allocates large blocks of memory from the system and distributes 'allocsize'
@ -48,93 +48,101 @@
namespace utils
{
class FixedAllocator
{
public:
EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS = (4096 * 4); // should be a multiple of pagesize
public:
EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS = (4096 * 4); // should be a multiple of pagesize
EXPORT FixedAllocator() :
capacityRemaining(0),
elementCount(DEFAULT_NUM_ELEMENTS),
elementSize(0),
currentlyStored(0),
tmpSpace(false),
nextAlloc(0),
useLock(false),
lock(false) {}
EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false,
unsigned long numElements = DEFAULT_NUM_ELEMENTS) :
capacityRemaining(0),
elementCount(numElements),
elementSize(allocSize),
currentlyStored(0),
tmpSpace(isTmpSpace),
nextAlloc(0),
useLock(false),
lock(false) {}
EXPORT FixedAllocator(const FixedAllocator&);
EXPORT FixedAllocator& operator=(const FixedAllocator&);
virtual ~FixedAllocator() {}
EXPORT FixedAllocator()
: capacityRemaining(0)
, elementCount(DEFAULT_NUM_ELEMENTS)
, elementSize(0)
, currentlyStored(0)
, tmpSpace(false)
, nextAlloc(0)
, useLock(false)
, lock(false)
{
}
EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false,
unsigned long numElements = DEFAULT_NUM_ELEMENTS)
: capacityRemaining(0)
, elementCount(numElements)
, elementSize(allocSize)
, currentlyStored(0)
, tmpSpace(isTmpSpace)
, nextAlloc(0)
, useLock(false)
, lock(false)
{
}
EXPORT FixedAllocator(const FixedAllocator&);
EXPORT FixedAllocator& operator=(const FixedAllocator&);
virtual ~FixedAllocator()
{
}
EXPORT void* allocate();
EXPORT void* allocate(uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead)
EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution
void deallocate() { } // does nothing
EXPORT void deallocateAll(); // drops all memory in use
EXPORT uint64_t getMemUsage() const;
void setUseLock(bool);
void setAllocSize(uint);
EXPORT void* allocate();
EXPORT void* allocate(
uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead)
EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution
void deallocate()
{
} // does nothing
EXPORT void deallocateAll(); // drops all memory in use
EXPORT uint64_t getMemUsage() const;
void setUseLock(bool);
void setAllocSize(uint);
private:
void newBlock();
private:
void newBlock();
std::vector<boost::shared_array<uint8_t> > mem;
unsigned long capacityRemaining;
uint64_t elementCount;
unsigned long elementSize;
uint64_t currentlyStored;
bool tmpSpace;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
std::vector<boost::shared_array<uint8_t> > mem;
unsigned long capacityRemaining;
uint64_t elementCount;
unsigned long elementSize;
uint64_t currentlyStored;
bool tmpSpace;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
};
inline void* FixedAllocator::allocate()
{
void* ret;
void* ret;
if (useLock)
getSpinlock(lock);
if (capacityRemaining < elementSize)
newBlock();
if (useLock)
getSpinlock(lock);
if (capacityRemaining < elementSize)
newBlock();
ret = nextAlloc;
nextAlloc += elementSize;
capacityRemaining -= elementSize;
currentlyStored += elementSize;
if (useLock)
releaseSpinlock(lock);
return ret;
ret = nextAlloc;
nextAlloc += elementSize;
capacityRemaining -= elementSize;
currentlyStored += elementSize;
if (useLock)
releaseSpinlock(lock);
return ret;
}
inline void* FixedAllocator::allocate(uint32_t len)
{
void* ret;
void* ret;
if (useLock)
getSpinlock(lock);
if (capacityRemaining < len)
newBlock();
if (useLock)
getSpinlock(lock);
if (capacityRemaining < len)
newBlock();
ret = nextAlloc;
nextAlloc += len;
capacityRemaining -= len;
currentlyStored += len;
if (useLock)
releaseSpinlock(lock);
return ret;
ret = nextAlloc;
nextAlloc += len;
capacityRemaining -= len;
currentlyStored += len;
if (useLock)
releaseSpinlock(lock);
return ret;
}
#undef EXPORT
} // namespace
} // namespace utils

243
utils/common/genericparser.h
View File

@ -15,32 +15,29 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "conststring.h"
namespace genericparser
{
using utils::ConstString;
class Tokenizer
{
protected:
const char *mStr;
const char *mEnd;
public:
explicit Tokenizer(const char *str, size_t length)
:mStr(str), mEnd(str + length)
{ }
protected:
const char* mStr;
const char* mEnd;
public:
explicit Tokenizer(const char* str, size_t length) : mStr(str), mEnd(str + length)
{
}
size_t length() const
{
return mEnd - mStr;
}
const char *ptr() const
const char* ptr() const
{
return mStr;
}
@ -65,18 +62,20 @@ public:
{
if (!isSpace())
return ConstString(nullptr, 0);
const char *start = mStr;
for ( ; isSpace() ; mStr++)
{ }
const char* start = mStr;
for (; isSpace(); mStr++)
{
}
return ConstString(start, mStr - start);
}
ConstString tokenDigits()
{
if (!isDigit())
return ConstString(nullptr, 0);
const char *start = mStr;
for ( ; isDigit() ; mStr++)
{ }
const char* start = mStr;
for (; isDigit(); mStr++)
{
}
return ConstString(start, mStr - start);
}
ConstString tokenChar(char chr)
@ -93,19 +92,19 @@ public:
}
};
class Parser: public Tokenizer
class Parser : public Tokenizer
{
protected:
protected:
bool mSyntaxError;
public:
explicit Parser(const char *str, size_t length)
:Tokenizer(str, length), mSyntaxError(false)
{ }
explicit Parser(const std::string & str)
:Parser(str.data(), str.length())
{ }
Parser & skipLeadingSpaces()
public:
explicit Parser(const char* str, size_t length) : Tokenizer(str, length), mSyntaxError(false)
{
}
explicit Parser(const std::string& str) : Parser(str.data(), str.length())
{
}
Parser& skipLeadingSpaces()
{
tokenSpaces();
return *this;
@ -119,7 +118,7 @@ public:
mSyntaxError = true;
return false;
}
const char *tokStart() const
const char* tokStart() const
{
return mStr;
}
@ -128,16 +127,14 @@ public:
return ConstString(mStr, 0);
}
// A helper class template to set the parser syntax error
// if A returned isNull() after parsing.
template<class A>
class SetSyntaxErrorOnNull :public A
template <class A>
class SetSyntaxErrorOnNull : public A
{
public:
SetSyntaxErrorOnNull(Parser *p)
:A(p)
public:
SetSyntaxErrorOnNull(Parser* p) : A(p)
{
if (A::isNull())
p->setSyntaxError();
@ -146,15 +143,14 @@ public:
// A helper class template for a rule in the form: <res> := [ <a> ]
template<class A>
class Opt: public A
template <class A>
class Opt : public A
{
public:
explicit Opt(const A &rhs)
:A(rhs)
{ }
explicit Opt(Parser *p)
:A(p)
public:
explicit Opt(const A& rhs) : A(rhs)
{
}
explicit Opt(Parser* p) : A(p)
{
if (A::isNull() && !p->syntaxError())
A::operator=(A::empty(p));
@ -169,7 +165,6 @@ public:
// M - mandatory - this part is required during parse time
// O - optional - this part is optional during parse time
// A helper class template for a rule in the form: <res> := <a> <b>
// i.e. both parts are mandatory at parse time
// The value of <a> is not important, and is created
@ -178,53 +173,52 @@ public:
// Example:
// <period> <unsigned integer>
template<class A, class B>
class UD2MM: public B
template <class A, class B>
class UD2MM : public B
{
public:
explicit UD2MM(Parser *p)
:B(A(p).isNull() ? B() :SetSyntaxErrorOnNull<B>(p))
{ }
explicit UD2MM(const B & b)
:B(b)
{ }
explicit UD2MM()
:B()
{ }
bool isNull() const { return B::isNull(); }
public:
explicit UD2MM(Parser* p) : B(A(p).isNull() ? B() : SetSyntaxErrorOnNull<B>(p))
{
}
explicit UD2MM(const B& b) : B(b)
{
}
explicit UD2MM() : B()
{
}
bool isNull() const
{
return B::isNull();
}
};
// A helper class template for a rule in the form: <res> := <a> <b>
// i.e. both parts are mandatory at parse time.
template<class A, class B>
class DD2MM: public A,
public B
template <class A, class B>
class DD2MM : public A, public B
{
public:
public:
// Sets syntax error if <a> was not followed by <b>
explicit DD2MM(Parser *p)
:A(p),
B(A::isNull() ? B() : SetSyntaxErrorOnNull<B>(p))
{ }
explicit DD2MM(const A & a, const B &b)
:A(b), B(b)
{ }
explicit DD2MM(Parser* p) : A(p), B(A::isNull() ? B() : SetSyntaxErrorOnNull<B>(p))
{
}
explicit DD2MM(const A& a, const B& b) : A(b), B(b)
{
}
};
// A helper class template for a rule in the form: <res> := <a> [ <b> ]
// i.e. <a> is mandatory, <b> is optional at parse time.
template<class A, class B>
class DD2MO: public A,
public B
template <class A, class B>
class DD2MO : public A, public B
{
public:
explicit DD2MO(Parser *p)
:A(p),
B(A::isNull() ? B() : B(p))
{ }
explicit DD2MO(const A &a, const B &b)
:A(a), B(b)
{ }
public:
explicit DD2MO(Parser* p) : A(p), B(A::isNull() ? B() : B(p))
{
}
explicit DD2MO(const A& a, const B& b) : A(a), B(b)
{
}
};
// A helper class template for a rule in the form: <res> := <a> [ <b> ]
@ -232,73 +226,68 @@ public:
// The value of <a> is not important and is not included
// into the target class, e.g.
// <period> [ <unsigned integer> ]
template<class A, class B>
class UD2MO: public B
template <class A, class B>
class UD2MO : public B
{
public:
explicit UD2MO(Parser *p)
:B(A(p).isNull() ? B() : B(p))
{ }
explicit UD2MO(const B &b)
:B(b)
{ }
explicit UD2MO()
:B()
{ }
public:
explicit UD2MO(Parser* p) : B(A(p).isNull() ? B() : B(p))
{
}
explicit UD2MO(const B& b) : B(b)
{
}
explicit UD2MO() : B()
{
}
};
// A helper class template for a rule in the form: <res> := <a> [ <b> ]
// i.e. <a> is mandatory, <b> is optional at parse time.
// The result class derives from "A".
// The result class puts "B" as a member.
template<class A, class B>
class DM2MO: public A
template <class A, class B>
class DM2MO : public A
{
protected:
protected:
B mB;
public:
explicit DM2MO(Parser *p)
:A(p),
mB(A::isNull() ? B() : B(p))
{ }
};
public:
explicit DM2MO(Parser* p) : A(p), mB(A::isNull() ? B() : B(p))
{
}
};
// A helper class template for a rule in the form: <res> := [ <a> ] <b>
// i.e. <a> is optional, <b> is mandatory at parse time.
template<class A,class B>
class DD2OM: public Opt<A>,
public B
template <class A, class B>
class DD2OM : public Opt<A>, public B
{
public:
explicit DD2OM(Parser *p)
:Opt<A>(p), B(p)
public:
explicit DD2OM(Parser* p) : Opt<A>(p), B(p)
{
if (B::isNull())
p->setSyntaxError();
}
explicit DD2OM()
:Opt<A>(A())
{ }
explicit DD2OM(const A & a, const B & b)
:Opt<A>(a), B(b)
{ }
explicit DD2OM() : Opt<A>(A())
{
}
explicit DD2OM(const A& a, const B& b) : Opt<A>(a), B(b)
{
}
};
// A helper class template for a rule in the form: <res> := a | b
template<class Container, class A, class B>
class Choice2: public Container
template <class Container, class A, class B>
class Choice2 : public Container
{
public:
explicit Choice2(const A & a)
:Container(a)
{ }
explicit Choice2(const B & b)
:Container(b)
{ }
explicit Choice2(Parser *p)
:Container(A(p))
public:
explicit Choice2(const A& a) : Container(a)
{
}
explicit Choice2(const B& b) : Container(b)
{
}
explicit Choice2(Parser* p) : Container(A(p))
{
if (Container::isNull() && !p->syntaxError())
*this = Choice2(B(p));
@ -306,6 +295,4 @@ public:
};
};
} // namespace genericparser
} // namespace genericparser

585
utils/common/hasher.h
View File

@ -38,331 +38,331 @@ namespace utils
inline uint32_t rotl32(uint32_t x, int8_t r)
{
return (x << r) | (x >> (32 - r));
return (x << r) | (x >> (32 - r));
}
inline uint32_t fmix(uint32_t h)
{
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
return h;
}
inline uint64_t fmix(uint64_t k)
{
k ^= k >> 33;
k *= 0xff51afd7ed558ccdULL;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53ULL;
k ^= k >> 33;
k ^= k >> 33;
k *= 0xff51afd7ed558ccdULL;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53ULL;
k ^= k >> 33;
return k;
return k;
}
inline uint64_t rotl64(uint64_t x, int8_t r)
{
return (x << r) | (x >> (64 - r));
return (x << r) | (x >> (64 - r));
}
class Hasher
{
public:
inline uint32_t operator()(const std::string& s) const
public:
inline uint32_t operator()(const std::string& s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const char* data, uint64_t len) const
{
const int nblocks = len / 4;
uint32_t h1 = 0;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
return operator()(s.data(), s.length());
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
inline uint32_t operator()(const char* data, uint64_t len) const
//----------
// tail
const uint8_t* tail = (const uint8_t*)(data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
const int nblocks = len / 4;
case 3:
k1 ^= tail[2] << 16;
/* fall through */
case 2:
k1 ^= tail[1] << 8;
/* fall through */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
uint32_t h1 = 0;
//----------
// finalization
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
h1 ^= len;
//----------
// body
h1 = fmix(h1);
const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*) (data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
case 3:
k1 ^= tail[2] << 16;
/* fall through */
case 2:
k1 ^= tail[1] << 8;
/* fall through */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h1 = fmix(h1);
return h1;
}
return h1;
}
};
class Hasher_r
{
public:
inline uint32_t operator()(const char* data, uint64_t len, uint32_t seed) const
public:
inline uint32_t operator()(const char* data, uint64_t len, uint32_t seed) const
{
const int nblocks = len / 4;
uint32_t h1 = seed;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
const int nblocks = len / 4;
uint32_t k1 = blocks[i];
uint32_t h1 = seed;
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*) (data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
case 3:
k1 ^= tail[2] << 16;
/* FALLTHRU */
case 2:
k1 ^= tail[1] << 8;
/* FALLTHRU */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
return h1;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
inline uint32_t finalize(uint32_t seed, uint32_t len) const
//----------
// tail
const uint8_t* tail = (const uint8_t*)(data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
seed ^= len;
seed = fmix(seed);
return seed;
}
case 3:
k1 ^= tail[2] << 16;
/* FALLTHRU */
case 2:
k1 ^= tail[1] << 8;
/* FALLTHRU */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
return h1;
}
inline uint32_t finalize(uint32_t seed, uint32_t len) const
{
seed ^= len;
seed = fmix(seed);
return seed;
}
};
class Hasher128
{
public:
inline uint64_t operator()(const char* data, uint64_t len) const
public:
inline uint64_t operator()(const char* data, uint64_t len) const
{
const int nblocks = len / 16;
uint64_t h1 = 0;
uint64_t h2 = 0;
const uint64_t c1 = 0x87c37b91114253d5ULL;
const uint64_t c2 = 0x4cf5ad432745937fULL;
//----------
// body
const uint64_t* blocks = (const uint64_t*)(data);
for (int i = 0; i < nblocks; i++)
{
const int nblocks = len / 16;
uint64_t k1 = blocks[i * 2 + 0];
uint64_t k2 = blocks[i * 2 + 1];
uint64_t h1 = 0;
uint64_t h2 = 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
const uint64_t c1 = 0x87c37b91114253d5ULL;
const uint64_t c2 = 0x4cf5ad432745937fULL;
h1 = rotl64(h1, 27);
h1 += h2;
h1 = h1 * 5 + 0x52dce729;
//----------
// body
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
const uint64_t* blocks = (const uint64_t*) (data);
for (int i = 0; i < nblocks; i++)
{
uint64_t k1 = blocks[i * 2 + 0];
uint64_t k2 = blocks[i * 2 + 1];
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
h1 = rotl64(h1, 27);
h1 += h2;
h1 = h1 * 5 + 0x52dce729;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
h2 = rotl64(h2, 31);
h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*) (data + nblocks * 16);
uint64_t k1 = 0;
uint64_t k2 = 0;
switch (len & 15)
{
case 15:
k2 ^= uint64_t(tail[14]) << 48;
/* FALLTHRU */
case 14:
k2 ^= uint64_t(tail[13]) << 40;
/* FALLTHRU */
case 13:
k2 ^= uint64_t(tail[12]) << 32;
/* FALLTHRU */
case 12:
k2 ^= uint64_t(tail[11]) << 24;
/* FALLTHRU */
case 11:
k2 ^= uint64_t(tail[10]) << 16;
/* FALLTHRU */
case 10:
k2 ^= uint64_t(tail[9]) << 8;
/* FALLTHRU */
case 9:
k2 ^= uint64_t(tail[8]) << 0;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
/* FALLTHRU */
case 8:
k1 ^= uint64_t(tail[7]) << 56;
/* FALLTHRU */
case 7:
k1 ^= uint64_t(tail[6]) << 48;
/* FALLTHRU */
case 6:
k1 ^= uint64_t(tail[5]) << 40;
/* FALLTHRU */
case 5:
k1 ^= uint64_t(tail[4]) << 32;
/* FALLTHRU */
case 4:
k1 ^= uint64_t(tail[3]) << 24;
/* FALLTHRU */
case 3:
k1 ^= uint64_t(tail[2]) << 16;
/* FALLTHRU */
case 2:
k1 ^= uint64_t(tail[1]) << 8;
/* FALLTHRU */
case 1:
k1 ^= uint64_t(tail[0]) << 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h2 ^= len;
h1 += h2;
h2 += h1;
h1 = fmix(h1);
h2 = fmix(h2);
h1 += h2;
h2 += h1;
return h1;
h2 = rotl64(h2, 31);
h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*)(data + nblocks * 16);
uint64_t k1 = 0;
uint64_t k2 = 0;
switch (len & 15)
{
case 15:
k2 ^= uint64_t(tail[14]) << 48;
/* FALLTHRU */
case 14:
k2 ^= uint64_t(tail[13]) << 40;
/* FALLTHRU */
case 13:
k2 ^= uint64_t(tail[12]) << 32;
/* FALLTHRU */
case 12:
k2 ^= uint64_t(tail[11]) << 24;
/* FALLTHRU */
case 11:
k2 ^= uint64_t(tail[10]) << 16;
/* FALLTHRU */
case 10:
k2 ^= uint64_t(tail[9]) << 8;
/* FALLTHRU */
case 9:
k2 ^= uint64_t(tail[8]) << 0;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
/* FALLTHRU */
case 8:
k1 ^= uint64_t(tail[7]) << 56;
/* FALLTHRU */
case 7:
k1 ^= uint64_t(tail[6]) << 48;
/* FALLTHRU */
case 6:
k1 ^= uint64_t(tail[5]) << 40;
/* FALLTHRU */
case 5:
k1 ^= uint64_t(tail[4]) << 32;
/* FALLTHRU */
case 4:
k1 ^= uint64_t(tail[3]) << 24;
/* FALLTHRU */
case 3:
k1 ^= uint64_t(tail[2]) << 16;
/* FALLTHRU */
case 2:
k1 ^= uint64_t(tail[1]) << 8;
/* FALLTHRU */
case 1:
k1 ^= uint64_t(tail[0]) << 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h2 ^= len;
h1 += h2;
h2 += h1;
h1 = fmix(h1);
h2 = fmix(h2);
h1 += h2;
h2 += h1;
return h1;
}
};
// TODO a copy of these classes also exists in primitiveprocessor.h; consolidate
class Hash128
{
public:
inline size_t operator()(const int128_t i) const
{
return (uint64_t)((int64_t) i);
}
public:
inline size_t operator()(const int128_t i) const
{
return (uint64_t)((int64_t)i);
}
};
class Equal128
{
public:
inline bool operator()(const int128_t f1, const int128_t f2) const
{
return f1 == f2;
}
public:
inline bool operator()(const int128_t f1, const int128_t f2) const
{
return f1 == f2;
}
};
//------------------------------------------------------------------------------
@ -372,17 +372,19 @@ public:
//------------------------------------------------------------------------------
class TupleHasher
{
public:
TupleHasher(uint32_t len) : fHashLen(len) {}
inline uint64_t operator()(const uint8_t* hashKey) const
{
return fHasher(reinterpret_cast<const char*>(hashKey), fHashLen);
}
private:
Hasher fHasher;
uint32_t fHashLen;
};
public:
TupleHasher(uint32_t len) : fHashLen(len)
{
}
inline uint64_t operator()(const uint8_t* hashKey) const
{
return fHasher(reinterpret_cast<const char*>(hashKey), fHashLen);
}
private:
Hasher fHasher;
uint32_t fHashLen;
};
//------------------------------------------------------------------------------
/** @brief class TupleComparator
@ -391,16 +393,17 @@ private:
//------------------------------------------------------------------------------
class TupleComparator
{
public:
TupleComparator(uint32_t len) : fCmpLen(len) {}
inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const
{
return (memcmp(hashKey1, hashKey2, fCmpLen) == 0);
}
private:
uint32_t fCmpLen;
public:
TupleComparator(uint32_t len) : fCmpLen(len)
{
}
inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const
{
return (memcmp(hashKey1, hashKey2, fCmpLen) == 0);
}
private:
uint32_t fCmpLen;
};
}
} // namespace utils

46
utils/common/hashfamily.h
View File

@ -22,34 +22,32 @@
namespace utils
{
class HashFamily
{
public:
HashFamily(const utils::Hasher_r& h,
const uint64_t intermediateHash,
const uint64_t len,
const datatypes::MariaDBHasher& hM) : mHasher(h),
mMariaDBHasher(hM),
mHasher_rHash(intermediateHash),
mHasher_rLen(len)
{ }
public:
HashFamily(const utils::Hasher_r& h, const uint64_t intermediateHash, const uint64_t len,
const datatypes::MariaDBHasher& hM)
: mHasher(h), mMariaDBHasher(hM), mHasher_rHash(intermediateHash), mHasher_rLen(len)
{
}
// Algorithm, seed and factor are taken from this discussion
// https://stackoverflow.com/questions/1646807/quick-and-simple-hash-code-combinations
inline uint64_t finalize() const
{
return (seed * factor + mHasher.finalize(mHasher_rHash, mHasher_rLen)) * factor + mMariaDBHasher.finalize();
}
private:
constexpr static uint64_t seed = 1009ULL;
constexpr static uint64_t factor = 9176ULL;
// Algorithm, seed and factor are taken from this discussion
// https://stackoverflow.com/questions/1646807/quick-and-simple-hash-code-combinations
inline uint64_t finalize() const
{
return (seed * factor + mHasher.finalize(mHasher_rHash, mHasher_rLen)) * factor +
mMariaDBHasher.finalize();
}
const utils::Hasher_r& mHasher;
const datatypes::MariaDBHasher& mMariaDBHasher;
const uint64_t mHasher_rHash;
const uint32_t mHasher_rLen;
private:
constexpr static uint64_t seed = 1009ULL;
constexpr static uint64_t factor = 9176ULL;
const utils::Hasher_r& mHasher;
const datatypes::MariaDBHasher& mMariaDBHasher;
const uint64_t mHasher_rHash;
const uint32_t mHasher_rLen;
};
}
} // namespace utils
// vim:ts=2 sw=2:

1
utils/common/mariadb_my_sys.h
View File

@ -34,4 +34,3 @@
#include <my_sys.h>
#include "mcsconfig_conflicting_defs_restore.h"

6
utils/common/mcs_basic_types.h
View File

@ -14,13 +14,11 @@
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA.
*/
MA 02110-1301, USA.
*/
#pragma once
using int128_t = __int128;
using uint128_t = unsigned __int128;
// vim:ts=2 sw=2:

363
utils/common/nullvaluemanip.cpp
View File

@ -16,7 +16,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "nullvaluemanip.h"
#include <sstream>
@ -25,234 +24,180 @@ using namespace execplan;
namespace utils
{
uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth)
{
switch (t)
switch (t)
{
case CalpontSystemCatalog::TINYINT: return joblist::TINYINTNULL;
case CalpontSystemCatalog::SMALLINT: return joblist::SMALLINTNULL;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT: return joblist::INTNULL;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT: return joblist::FLOATNULL;
case CalpontSystemCatalog::DATE: return joblist::DATENULL;
case CalpontSystemCatalog::BIGINT: return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP: return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME: return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
case CalpontSystemCatalog::TINYINT:
return joblist::TINYINTNULL;
switch (colWidth)
{
case 1: return joblist::CHAR1NULL;
case CalpontSystemCatalog::SMALLINT:
return joblist::SMALLINTNULL;
case 2: return joblist::CHAR2NULL;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return joblist::INTNULL;
case 3:
case 4: return joblist::CHAR4NULL;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return joblist::FLOATNULL;
case 5:
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
case CalpontSystemCatalog::DATE:
return joblist::DATENULL;
default: throw logic_error("getNullValue() Can't return the NULL string");
}
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP:
return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME:
return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
switch (colWidth)
{
case 1:
return joblist::CHAR1NULL;
case 2:
return joblist::CHAR2NULL;
case 3:
case 4:
return joblist::CHAR4NULL;
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("getNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1 :
return joblist::TINYINTNULL;
case 2 :
return joblist::SMALLINTNULL;
case 4 :
return joblist::INTNULL;
default:
return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return joblist::UTINYINTNULL;
case CalpontSystemCatalog::USMALLINT:
return joblist::USMALLINTNULL;
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return joblist::UINTNULL;
case CalpontSystemCatalog::UBIGINT:
return joblist::UBIGINTNULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1: return joblist::TINYINTNULL;
case 2: return joblist::SMALLINTNULL;
case 4: return joblist::INTNULL;
default: return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT: return joblist::UTINYINTNULL;
case CalpontSystemCatalog::USMALLINT: return joblist::USMALLINTNULL;
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT: return joblist::UINTNULL;
case CalpontSystemCatalog::UBIGINT: return joblist::UBIGINTNULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl;
throw logic_error(os.str());
}
}
int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth)
{
switch (t)
switch (t)
{
case CalpontSystemCatalog::TINYINT: return (int64_t)((int8_t)joblist::TINYINTNULL);
case CalpontSystemCatalog::SMALLINT: return (int64_t)((int16_t)joblist::SMALLINTNULL);
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT: return (int64_t)((int32_t)joblist::INTNULL);
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT: return (int64_t)((int32_t)joblist::FLOATNULL);
case CalpontSystemCatalog::DATE: return (int64_t)((int32_t)joblist::DATENULL);
case CalpontSystemCatalog::BIGINT: return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP: return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME: return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
case CalpontSystemCatalog::TINYINT:
return (int64_t) ((int8_t) joblist::TINYINTNULL);
switch (colWidth)
{
case 1: return (int64_t)((int8_t)joblist::CHAR1NULL);
case CalpontSystemCatalog::SMALLINT:
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case 2: return (int64_t)((int16_t)joblist::CHAR2NULL);
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return (int64_t) ((int32_t) joblist::INTNULL);
case 3:
case 4: return (int64_t)((int32_t)joblist::CHAR4NULL);
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return (int64_t) ((int32_t) joblist::FLOATNULL);
case 5:
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
case CalpontSystemCatalog::DATE:
return (int64_t) ((int32_t) joblist::DATENULL);
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP:
return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME:
return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
switch (colWidth)
{
case 1:
return (int64_t) ((int8_t) joblist::CHAR1NULL);
case 2:
return (int64_t) ((int16_t) joblist::CHAR2NULL);
case 3:
case 4:
return (int64_t) ((int32_t) joblist::CHAR4NULL);
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("getSignedNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1 :
return (int64_t) ((int8_t) joblist::TINYINTNULL);
case 2 :
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case 4 :
return (int64_t) ((int32_t) joblist::INTNULL);
case 8:
return joblist::BIGINTNULL;
default:
ostringstream os;
os << "getSignedNullValue(): got bad column width (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return (int64_t) ((int8_t) joblist::UTINYINTNULL);
case CalpontSystemCatalog::USMALLINT:
return (int64_t) ((int16_t) joblist::USMALLINTNULL);
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return (int64_t) ((int32_t) joblist::UINTNULL);
case CalpontSystemCatalog::UBIGINT:
return (int64_t)joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE:
return (int64_t)joblist::LONGDOUBLENULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getSignedNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
default: throw logic_error("getSignedNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1: return (int64_t)((int8_t)joblist::TINYINTNULL);
case 2: return (int64_t)((int16_t)joblist::SMALLINTNULL);
case 4: return (int64_t)((int32_t)joblist::INTNULL);
case 8: return joblist::BIGINTNULL;
default:
ostringstream os;
os << "getSignedNullValue(): got bad column width (" << t << "). Width=" << colWidth << endl;
throw logic_error(os.str());
}
break;
}
case CalpontSystemCatalog::UTINYINT: return (int64_t)((int8_t)joblist::UTINYINTNULL);
case CalpontSystemCatalog::USMALLINT: return (int64_t)((int16_t)joblist::USMALLINTNULL);
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT: return (int64_t)((int32_t)joblist::UINTNULL);
case CalpontSystemCatalog::UBIGINT: return (int64_t)joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE: return (int64_t)joblist::LONGDOUBLENULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getSignedNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl;
throw logic_error(os.str());
}
}
}
} // namespace utils

6
utils/common/nullvaluemanip.h
View File

@ -15,19 +15,15 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "../../dbcon/execplan/calpontsystemcatalog.h"
namespace utils
{
// returns the NULL value for our 'numeric' types including short strings.
// The width is only relevant for long string columns.
uint64_t getNullValue(execplan::CalpontSystemCatalog::ColDataType, uint32_t colWidth = 0);
int64_t getSignedNullValue(execplan::CalpontSystemCatalog::ColDataType, uint32_t colWidth = 0);
}
} // namespace utils

49
utils/common/pipe.h
View File

@ -23,11 +23,12 @@
class Pipe
{
int fd[2];
public:
public:
Pipe()
{
fd[0]= 0;
fd[1]= 0;
fd[0] = 0;
fd[1] = 0;
}
~Pipe()
{
@ -45,35 +46,35 @@ public:
{
return ::pipe(fd) == -1;
}
bool init() // TODO: remove this
bool init() // TODO: remove this
{
return open();
}
ssize_t read(char *str, size_t nbytes)
ssize_t read(char* str, size_t nbytes)
{
return ::read(fd[0], str, nbytes);
}
ssize_t readtm(const struct timeval &tv, void *buf, size_t nbytes)
ssize_t readtm(const struct timeval& tv, void* buf, size_t nbytes)
{
fd_set rfds;
FD_ZERO(&rfds);
FD_SET(fd[0], &rfds);
struct timeval tmptv = tv;
int retval = select(fd[0] + 1, &rfds, NULL, NULL, &tmptv);
if (retval == -1)
return -1;
if (!retval)
{
errno= ETIME;
return -1;
}
return ::read(fd[0], buf, nbytes);
fd_set rfds;
FD_ZERO(&rfds);
FD_SET(fd[0], &rfds);
struct timeval tmptv = tv;
int retval = select(fd[0] + 1, &rfds, NULL, NULL, &tmptv);
if (retval == -1)
return -1;
if (!retval)
{
errno = ETIME;
return -1;
}
return ::read(fd[0], buf, nbytes);
}
ssize_t write(const char *str, size_t nbytes)
ssize_t write(const char* str, size_t nbytes)
{
return ::write(fd[1], str, nbytes);
}
ssize_t write(const std::string &str)
ssize_t write(const std::string& str)
{
return write(str.data(), str.length());
}
@ -82,14 +83,12 @@ public:
if (fd[0])
{
::close(fd[0]);
fd[0]= 0;
fd[0] = 0;
}
if (fd[1])
{
::close(fd[1]);
fd[1]= 0;
fd[1] = 0;
}
}
};

95
utils/common/poolallocator.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
#include <iostream>
//#define NDEBUG
@ -31,72 +31,71 @@ using namespace boost;
namespace utils
{
PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v)
{
allocSize = v.allocSize;
tmpSpace = v.tmpSpace;
useLock = v.useLock;
deallocateAll();
return *this;
allocSize = v.allocSize;
tmpSpace = v.tmpSpace;
useLock = v.useLock;
deallocateAll();
return *this;
}
void PoolAllocator::deallocateAll()
{
capacityRemaining = 0;
nextAlloc = NULL;
memUsage = 0;
mem.clear();
oob.clear();
capacityRemaining = 0;
nextAlloc = NULL;
memUsage = 0;
mem.clear();
oob.clear();
}
void PoolAllocator::newBlock()
{
shared_array<uint8_t> next;
shared_array<uint8_t> next;
capacityRemaining = allocSize;
capacityRemaining = allocSize;
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[allocSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
nextAlloc = mem.front().get();
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[allocSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
nextAlloc = mem.front().get();
}
void * PoolAllocator::allocOOB(uint64_t size)
void* PoolAllocator::allocOOB(uint64_t size)
{
OOBMemInfo memInfo;
OOBMemInfo memInfo;
memUsage += size;
memInfo.mem.reset(new uint8_t[size]);
memInfo.size = size;
void *ret = (void*) memInfo.mem.get();
oob[ret] = memInfo;
return ret;
memUsage += size;
memInfo.mem.reset(new uint8_t[size]);
memInfo.size = size;
void* ret = (void*)memInfo.mem.get();
oob[ret] = memInfo;
return ret;
}
void PoolAllocator::deallocate(void* p)
{
bool _false = false;
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
OutOfBandMap::iterator it = oob.find(p);
bool _false = false;
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
OutOfBandMap::iterator it = oob.find(p);
if (it == oob.end())
{
if (useLock)
lock.store(false, std::memory_order_release);
return;
}
memUsage -= it->second.size;
oob.erase(it);
if (it == oob.end())
{
if (useLock)
lock.store(false, std::memory_order_release);
lock.store(false, std::memory_order_release);
return;
}
memUsage -= it->second.size;
oob.erase(it);
if (useLock)
lock.store(false, std::memory_order_release);
}
}
} // namespace utils

167
utils/common/poolallocator.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/* This allocator is an attempt to consolidate small allocations and
deallocations to boost performance and reduce mem fragmentation. */
@ -34,100 +34,105 @@
namespace utils
{
class PoolAllocator
{
public:
static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages
public:
static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages
explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false, bool _useLock = false) :
allocSize(windowSize),
tmpSpace(isTmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0),
useLock(_useLock),
lock(false) { }
PoolAllocator(const PoolAllocator& p) :
allocSize(p.allocSize),
tmpSpace(p.tmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0),
useLock(p.useLock),
lock(false) { }
virtual ~PoolAllocator() {}
explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false,
bool _useLock = false)
: allocSize(windowSize)
, tmpSpace(isTmpSpace)
, capacityRemaining(0)
, memUsage(0)
, nextAlloc(0)
, useLock(_useLock)
, lock(false)
{
}
PoolAllocator(const PoolAllocator& p)
: allocSize(p.allocSize)
, tmpSpace(p.tmpSpace)
, capacityRemaining(0)
, memUsage(0)
, nextAlloc(0)
, useLock(p.useLock)
, lock(false)
{
}
virtual ~PoolAllocator()
{
}
PoolAllocator& operator=(const PoolAllocator&);
PoolAllocator& operator=(const PoolAllocator&);
void* allocate(uint64_t size);
void deallocate(void* p);
void deallocateAll();
void* allocate(uint64_t size);
void deallocate(void* p);
void deallocateAll();
inline uint64_t getMemUsage() const
{
return memUsage;
}
unsigned getWindowSize() const
{
return allocSize;
}
inline uint64_t getMemUsage() const
{
return memUsage;
}
unsigned getWindowSize() const
{
return allocSize;
}
void setUseLock(bool ul)
{
useLock = ul;
}
void setUseLock(bool ul)
{
useLock = ul;
}
private:
void newBlock();
void *allocOOB(uint64_t size);
private:
void newBlock();
void* allocOOB(uint64_t size);
unsigned allocSize;
std::vector<boost::shared_array<uint8_t> > mem;
bool tmpSpace;
unsigned capacityRemaining;
uint64_t memUsage;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
unsigned allocSize;
std::vector<boost::shared_array<uint8_t> > mem;
bool tmpSpace;
unsigned capacityRemaining;
uint64_t memUsage;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
struct OOBMemInfo
{
boost::shared_array<uint8_t> mem;
uint64_t size;
};
typedef std::map<void*, OOBMemInfo> OutOfBandMap;
OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd
struct OOBMemInfo
{
boost::shared_array<uint8_t> mem;
uint64_t size;
};
typedef std::map<void*, OOBMemInfo> OutOfBandMap;
OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd
};
inline void* PoolAllocator::allocate(uint64_t size)
{
void *ret;
bool _false = false;
void* ret;
bool _false = false;
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
if (size > allocSize)
{
ret = allocOOB(size);
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
if (size > allocSize)
{
ret = allocOOB(size);
if (useLock)
lock.store(false, std::memory_order_release);
return ret;
}
if (size > capacityRemaining)
newBlock();
ret = (void*) nextAlloc;
nextAlloc += size;
capacityRemaining -= size;
memUsage += size;
if (useLock)
lock.store(false, std::memory_order_release);
lock.store(false, std::memory_order_release);
return ret;
}
if (size > capacityRemaining)
newBlock();
ret = (void*)nextAlloc;
nextAlloc += size;
capacityRemaining -= size;
memUsage += size;
if (useLock)
lock.store(false, std::memory_order_release);
return ret;
}
}
} // namespace utils

4143
utils/common/robin_hood.h
View File

File diff suppressed because it is too large Load Diff

141
utils/common/service.h
View File

@ -20,95 +20,94 @@
#include <signal.h>
#include "pipe.h"
class Service
{
protected:
// The service name, for logging
const std::string m_name;
// The pipe to send messages from the child to the parent
Pipe m_pipe;
protected:
// The service name, for logging
const std::string m_name;
// The pipe to send messages from the child to the parent
Pipe m_pipe;
static void common_signal_handler_CHLD(int sig)
{
}
static void common_signal_handler_CHLD(int sig)
{ }
void InitCommonSignalHandlers()
{
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = common_signal_handler_CHLD;
sigaction(SIGCHLD, &sa, NULL);
}
void InitCommonSignalHandlers()
int RunForking()
{
int err;
InitCommonSignalHandlers();
if (m_pipe.open() || (err = fork()) < 0)
{
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = common_signal_handler_CHLD;
sigaction(SIGCHLD, &sa, NULL);
// Pipe or fork failed
LogErrno();
return 1;
}
int RunForking()
if (err > 0) // Parent
return Parent();
return Child();
}
virtual int Parent()
{
char str[100];
// Read the message from the child
ssize_t nbytes = m_pipe.readtm({120, 0}, str, sizeof(str));
if (nbytes >= 0)
{
int err;
InitCommonSignalHandlers();
if (m_pipe.open() || (err = fork()) < 0)
{
// Pipe or fork failed
LogErrno();
return 1;
}
if (err > 0) // Parent
return Parent();
return Child();
ParentLogChildMessage(std::string(str, nbytes));
}
virtual int Parent()
else
{
char str[100];
// Read the message from the child
ssize_t nbytes= m_pipe.readtm({120,0}, str, sizeof(str));
if (nbytes >= 0)
{
ParentLogChildMessage(std::string(str, nbytes));
}
else
{
// read() failed
LogErrno();
return 1;
}
return 0;
// read() failed
LogErrno();
return 1;
}
return 0;
}
public:
Service(const std::string &name)
:m_name(name)
{ }
virtual ~Service() { }
public:
Service(const std::string& name) : m_name(name)
{
}
virtual ~Service()
{
}
void NotifyServiceStarted()
void NotifyServiceStarted()
{
if (m_pipe.is_open_for_write())
{
if (m_pipe.is_open_for_write())
{
std::ostringstream str;
str << m_name << " main process has started";
m_pipe.write(str.str());
}
std::ostringstream str;
str << m_name << " main process has started";
m_pipe.write(str.str());
}
}
void NotifyServiceInitializationFailed()
void NotifyServiceInitializationFailed()
{
if (m_pipe.is_open_for_write())
{
if (m_pipe.is_open_for_write())
{
std::ostringstream str;
str << m_name << " main process initialization failed";
m_pipe.write(str.str());
}
std::ostringstream str;
str << m_name << " main process initialization failed";
m_pipe.write(str.str());
}
}
// Used by both Parent and Child to log errors
virtual void LogErrno()= 0;
// Used by Parent to log an initialization notification message from child
virtual void ParentLogChildMessage(const std::string &str)= 0;
// The main service process job
virtual int Child()= 0;
// Used by both Parent and Child to log errors
virtual void LogErrno() = 0;
// Used by Parent to log an initialization notification message from child
virtual void ParentLogChildMessage(const std::string& str) = 0;
// The main service process job
virtual int Child() = 0;
};

829
utils/common/simd_sse.h
View File

@ -17,455 +17,456 @@
#pragma once
#if defined(__x86_64__ )
#if defined(__x86_64__)
#include <cstdint>
#include <type_traits>
#ifdef __OPTIMIZE__
#include <smmintrin.h>
#include <emmintrin.h>
#define MCS_FORCE_INLINE __attribute__((__always_inline__))
#include <smmintrin.h>
#include <emmintrin.h>
#define MCS_FORCE_INLINE __attribute__((__always_inline__))
#else
#define __OPTIMIZE__
#include <smmintrin.h>
#include <emmintrin.h>
#undef __OPTIMIZE__
#define MCS_FORCE_INLINE inline
#define __OPTIMIZE__
#include <smmintrin.h>
#include <emmintrin.h>
#undef __OPTIMIZE__
#define MCS_FORCE_INLINE inline
#endif
#include <mcs_datatype.h>
namespace simd
{
using vi128_t = __m128i;
using msk128_t = uint16_t;
using int128_t = __int128;
using MT = uint16_t;
// This ugly wrapper used to allow to use __m128i as a template class parameter argument
struct vi128_wr
using vi128_t = __m128i;
using msk128_t = uint16_t;
using int128_t = __int128;
using MT = uint16_t;
// This ugly wrapper used to allow to use __m128i as a template class parameter argument
struct vi128_wr
{
__m128i v;
};
template <typename VT, int WIDTH>
class SimdFilterProcessor
{
};
template <>
class SimdFilterProcessor<vi128_wr, 16>
{
// This is a dummy class that is not currently used.
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = int128_t;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
__m128i v;
};
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(&fill));
}
template<typename VT, int WIDTH>
class SimdFilterProcessor
{ };
template<>
class SimdFilterProcessor<vi128_wr, 16>
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
// This is a dummy class that is not currently used.
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = int128_t;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(&fill));
}
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
MCS_FORCE_INLINE MT cmpDummy(vi128_t& x, vi128_t& y)
{
return 0xFFFF;
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE uint16_t convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 8>
MCS_FORCE_INLINE MT cmpDummy(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<8>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set_epi64x(fill, fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_or_si128(_mm_cmpgt_epi64(x, y),_mm_cmpeq_epi64(x, y)));
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi64(x, y));
}
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpNe(x, y) ^ cmpGt(x, y);
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 4>
return 0xFFFF;
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<4>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi32(fill);
}
return cmpDummy(x, y);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 2>
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<2>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi16(fill);
}
return cmpDummy(x, y);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 1>
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<1>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi8(fill);
}
return cmpDummy(x, y);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
// misc
MCS_FORCE_INLINE uint16_t convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi8(x, y));
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF;
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
// permute
/* TODO Available in AVX-512
MCS_FORCE_INLINE vi128_t perm8Bits(vi128_t& x, vi128_t& idx)
{
return _mm_permutexvar_epi8(x, idx);
}
*/
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
template <>
class SimdFilterProcessor<vi128_wr, 8>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<8>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set_epi64x(fill, fill);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
// Compare
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_or_si128(_mm_cmpgt_epi64(x, y), _mm_cmpeq_epi64(x, y)));
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi64(x, y));
}
} // end of simd
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y));
}
#endif // if defined(__x86_64__ )
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpNe(x, y) ^ cmpGt(x, y);
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template <>
class SimdFilterProcessor<vi128_wr, 4>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<4>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi32(fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template <>
class SimdFilterProcessor<vi128_wr, 2>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<2>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi16(fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template <>
class SimdFilterProcessor<vi128_wr, 1>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<1>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi8(fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// permute
/* TODO Available in AVX-512
MCS_FORCE_INLINE vi128_t perm8Bits(vi128_t& x, vi128_t& idx)
{
return _mm_permutexvar_epi8(x, idx);
}
*/
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
} // namespace simd
#endif // if defined(__x86_64__ )
// vim:ts=2 sw=2:

289
utils/common/simpleallocator.h
View File

@ -37,11 +37,11 @@
namespace utils
{
// A specialized allocator for std::tr1::unordered_multimap<uint64_t, uint64_t> based joiner
// or std::tr1::unordered_map<uint8_t*, uint8_t*> based aggregation.
// User shall initialize a pool and pass it to allocator, release the pool when map is done.
template<typename T> class SimpleAllocator;
template <typename T>
class SimpleAllocator;
// this pool is best for node size of 3*sizeof(int64).
// map nodes are taken from fixed size blocks, and control hash tables are from ::new.
@ -51,208 +51,211 @@ template<typename T> class SimpleAllocator;
#define OPT_NODE_UNITS 10
class SimplePool
{
public:
SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0) {}
~SimplePool()
{
reset();
}
public:
SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0)
{
}
~SimplePool()
{
reset();
}
inline void* allocate(size_t n, const void* = 0);
inline void deallocate(void* p, size_t n);
inline size_t max_size() const throw();
inline uint64_t getMemUsage() const;
inline void* allocate(size_t n, const void* = 0);
inline void deallocate(void* p, size_t n);
inline size_t max_size() const throw();
inline uint64_t getMemUsage() const;
private:
static const size_t fUnitPerChunk = OPT_NODE_UNITS * 10240;
private:
static const size_t fUnitPerChunk = OPT_NODE_UNITS * 10240;
inline void reset();
inline void allocateNewChunk();
inline void reset();
inline void allocateNewChunk();
// MemUnit stores a pointer to next unit before allocated, and T after allocated.
union MemUnit
{
MemUnit* fNext;
uint64_t fData;
}* fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit
// MemUnit stores a pointer to next unit before allocated, and T after allocated.
union MemUnit
{
MemUnit* fNext;
uint64_t fData;
} * fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit
std::list<MemUnit*> fBlockList;
uint64_t fTableMemSize;
std::list<MemUnit*> fBlockList;
uint64_t fTableMemSize;
static const size_t fUnitSize = sizeof(MemUnit);
static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS;
static const size_t fChunkSize = fUnitSize * fUnitPerChunk;
static const size_t fUnitSize = sizeof(MemUnit);
static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS;
static const size_t fChunkSize = fUnitSize * fUnitPerChunk;
};
template<typename T>
template <typename T>
class SimpleAllocator
{
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<typename U> struct rebind
{
typedef SimpleAllocator<U> other;
};
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template <typename U>
struct rebind
{
typedef SimpleAllocator<U> other;
};
SimpleAllocator() throw() {}
SimpleAllocator(std::shared_ptr<SimplePool> pool) throw()
{
fPool = pool;
}
SimpleAllocator(const SimpleAllocator& alloc)
{
fPool = alloc.fPool;
}
template<class U> SimpleAllocator(const SimpleAllocator<U>& alloc)
{
fPool = alloc.fPool;
}
SimpleAllocator() throw()
{
}
SimpleAllocator(std::shared_ptr<SimplePool> pool) throw()
{
fPool = pool;
}
SimpleAllocator(const SimpleAllocator& alloc)
{
fPool = alloc.fPool;
}
template <class U>
SimpleAllocator(const SimpleAllocator<U>& alloc)
{
fPool = alloc.fPool;
}
~SimpleAllocator() throw() { }
~SimpleAllocator() throw()
{
}
inline pointer address(reference x) const
{
return &x;
}
inline const_pointer address(const_reference x) const
{
return &x;
}
inline pointer address(reference x) const
{
return &x;
}
inline const_pointer address(const_reference x) const
{
return &x;
}
inline pointer allocate(size_type n, const void* = 0)
{
return static_cast<pointer>(fPool->allocate(n * sizeof(T)));
}
inline void deallocate(pointer p, size_type n)
{
fPool->deallocate(p, n * sizeof(T));
}
inline pointer allocate(size_type n, const void* = 0)
{
return static_cast<pointer>(fPool->allocate(n * sizeof(T)));
}
inline void deallocate(pointer p, size_type n)
{
fPool->deallocate(p, n * sizeof(T));
}
#ifdef _MSC_VER
//The MSVC STL library really needs this to return a big number...
size_type max_size() const throw()
{
return std::numeric_limits<size_type>::max();
}
// The MSVC STL library really needs this to return a big number...
size_type max_size() const throw()
{
return std::numeric_limits<size_type>::max();
}
#else
inline size_type max_size() const throw()
{
return fPool->max_size() / sizeof(T);
}
inline size_type max_size() const throw()
{
return fPool->max_size() / sizeof(T);
}
#endif
inline void construct(pointer ptr, const T& val)
{
new ((void*)ptr) T(val);
}
inline void destroy(pointer ptr)
{
ptr->T::~T();
}
inline void construct(pointer ptr, const T& val)
{
new ((void*)ptr) T(val);
}
inline void destroy(pointer ptr)
{
ptr->T::~T();
}
inline void setPool(SimplePool* pool)
{
fPool.reset(pool);
}
inline void setPool(SimplePool* pool)
{
fPool.reset(pool);
}
std::shared_ptr<SimplePool> fPool;
std::shared_ptr<SimplePool> fPool;
};
// inlines
inline void* SimplePool::allocate(size_t n, const void* dur)
{
// make sure the block allocated is on unit boundary
size_t unitCount = n / fUnitSize;
// make sure the block allocated is on unit boundary
size_t unitCount = n / fUnitSize;
if ((n % fUnitSize) != 0)
unitCount += 1;
if ((n % fUnitSize) != 0)
unitCount += 1;
// if for control table, let new allocator handle it.
if (unitCount > OPT_NODE_UNITS)
// if for control table, let new allocator handle it.
if (unitCount > OPT_NODE_UNITS)
{
fTableMemSize += n;
return new uint8_t[n];
}
// allocate node
MemUnit* curr = fNext;
do
{
if (curr == NULL)
{
fTableMemSize += n;
return new uint8_t[n];
allocateNewChunk();
curr = fNext;
}
// allocate node
MemUnit* curr = fNext;
fNext = curr + unitCount;
do
{
if (curr == NULL)
{
allocateNewChunk();
curr = fNext;
}
if (fNext > fEnd)
curr = NULL;
} while (!curr);
fNext = curr + unitCount;
if (fNext > fEnd)
curr = NULL;
}
while (!curr);
return curr;
return curr;
}
inline void SimplePool::deallocate(void* p, size_t n)
{
// only delete the old control table, which is allocated by new allocator.
if (n > fMaxNodeSize)
{
fTableMemSize -= n;
delete [] (static_cast<uint8_t*>(p));
}
// only delete the old control table, which is allocated by new allocator.
if (n > fMaxNodeSize)
{
fTableMemSize -= n;
delete[](static_cast<uint8_t*>(p));
}
}
inline size_t SimplePool::max_size() const throw()
{
return fUnitSize * fUnitPerChunk;
return fUnitSize * fUnitPerChunk;
}
inline uint64_t SimplePool::getMemUsage() const
{
return fTableMemSize + fBlockList.size() * fChunkSize +
// add list overhead, element type is a pointer, and
// lists store a next pointer.
fBlockList.size() * 2 * sizeof(void*);
return fTableMemSize + fBlockList.size() * fChunkSize +
// add list overhead, element type is a pointer, and
// lists store a next pointer.
fBlockList.size() * 2 * sizeof(void*);
}
inline void SimplePool::reset()
{
for (std::list<MemUnit*>::iterator i = fBlockList.begin(); i != fBlockList.end(); i++)
delete [] (*i);
for (std::list<MemUnit*>::iterator i = fBlockList.begin(); i != fBlockList.end(); i++)
delete[](*i);
fNext = NULL;
fEnd = NULL;
fNext = NULL;
fEnd = NULL;
}
inline void SimplePool::allocateNewChunk()
{
MemUnit* chunk = new MemUnit[fUnitPerChunk];
fBlockList.push_back(chunk);
fNext = chunk;
fEnd = chunk + fUnitPerChunk;
MemUnit* chunk = new MemUnit[fUnitPerChunk];
fBlockList.push_back(chunk);
fNext = chunk;
fEnd = chunk + fUnitPerChunk;
}
template <typename T1, typename T2>
inline bool operator==(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&)
{
return true;
return true;
}
template <typename T1, typename T2>
inline bool operator!=(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&)
{
return false;
return false;
}
}
} // namespace utils

23
utils/common/spinlock.h
View File

@ -4,24 +4,23 @@
namespace utils
{
inline void getSpinlock(std::atomic<bool> &lock)
inline void getSpinlock(std::atomic<bool>& lock)
{
bool _false = false;
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
bool _false = false;
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
}
inline bool trySpinlock(std::atomic<bool> &lock)
inline bool trySpinlock(std::atomic<bool>& lock)
{
bool _false = false;
bool ret = lock.compare_exchange_weak(_false, true, std::memory_order_acquire);
return ret;
bool _false = false;
bool ret = lock.compare_exchange_weak(_false, true, std::memory_order_acquire);
return ret;
}
inline void releaseSpinlock(std::atomic<bool> &lock)
inline void releaseSpinlock(std::atomic<bool>& lock)
{
lock.store(false, std::memory_order_release);
lock.store(false, std::memory_order_release);
}
}
} // namespace utils

606
utils/common/statistics.cpp
View File

@ -35,405 +35,407 @@ using ColumnsCache = std::vector<std::unordered_set<uint32_t>>;
StatisticsManager* StatisticsManager::instance()
{
static StatisticsManager* sm = new StatisticsManager();
return sm;
static StatisticsManager* sm = new StatisticsManager();
return sm;
}
void StatisticsManager::analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace)
{
std::lock_guard<std::mutex> lock(mut);
auto rowCount = rowGroup.getRowCount();
const auto columnCount = rowGroup.getColumnCount();
if (!rowCount || !columnCount)
return;
std::lock_guard<std::mutex> lock(mut);
auto rowCount = rowGroup.getRowCount();
const auto columnCount = rowGroup.getColumnCount();
if (!rowCount || !columnCount)
return;
auto& oids = rowGroup.getOIDs();
auto& oids = rowGroup.getOIDs();
rowgroup::Row r;
rowGroup.initRow(&r);
rowGroup.getRow(0, &r);
rowgroup::Row r;
rowGroup.initRow(&r);
rowGroup.getRow(0, &r);
ColumnsCache columns(columnCount, std::unordered_set<uint32_t>());
// Init key types.
for (uint32_t index = 0; index < columnCount; ++index)
keyTypes[oids[index]] = KeyType::PK;
ColumnsCache columns(columnCount, std::unordered_set<uint32_t>());
// Init key types.
for (uint32_t index = 0; index < columnCount; ++index)
keyTypes[oids[index]] = KeyType::PK;
const uint32_t maxRowCount = 4096;
// TODO: We should read just couple of blocks from columns, not all data, but this requires
// more deep refactoring of column commands.
rowCount = std::min(rowCount, maxRowCount);
// This is strange, it's a CS but I'm processing data as row by row, how to fix it?
for (uint32_t i = 0; i < rowCount; ++i)
const uint32_t maxRowCount = 4096;
// TODO: We should read just couple of blocks from columns, not all data, but this requires
// more deep refactoring of column commands.
rowCount = std::min(rowCount, maxRowCount);
// This is strange, it's a CS but I'm processing data as row by row, how to fix it?
for (uint32_t i = 0; i < rowCount; ++i)
{
for (uint32_t j = 0; j < columnCount; ++j)
{
for (uint32_t j = 0; j < columnCount; ++j)
{
if (r.isNullValue(j) || columns[j].count(r.getIntField(j)))
keyTypes[oids[j]] = KeyType::FK;
else
columns[j].insert(r.getIntField(j));
}
r.nextRow();
if (r.isNullValue(j) || columns[j].count(r.getIntField(j)))
keyTypes[oids[j]] = KeyType::FK;
else
columns[j].insert(r.getIntField(j));
}
r.nextRow();
}
if (trace)
output(StatisticsType::PK_FK);
if (trace)
output(StatisticsType::PK_FK);
}
void StatisticsManager::output(StatisticsType statisticsType)
{
if (statisticsType == StatisticsType::PK_FK)
{
std::cout << "Columns count: " << keyTypes.size() << std::endl;
for (const auto& p : keyTypes)
std::cout << p.first << " " << (int) p.second << std::endl;
}
if (statisticsType == StatisticsType::PK_FK)
{
std::cout << "Columns count: " << keyTypes.size() << std::endl;
for (const auto& p : keyTypes)
std::cout << p.first << " " << (int)p.second << std::endl;
}
}
// Someday it will be a virtual method, based on statistics type we processing.
std::unique_ptr<char[]> StatisticsManager::convertStatsToDataStream(uint64_t& dataStreamSize)
{
// Number of pairs.
uint64_t count = keyTypes.size();
// count, [[uid, keyType], ... ]
dataStreamSize = sizeof(uint64_t) + count * (sizeof(uint32_t) + sizeof(KeyType));
// Number of pairs.
uint64_t count = keyTypes.size();
// count, [[uid, keyType], ... ]
dataStreamSize = sizeof(uint64_t) + count * (sizeof(uint32_t) + sizeof(KeyType));
// Allocate memory for data stream.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Initialize the data stream.
uint64_t offset = 0;
std::memcpy(dataStream, reinterpret_cast<char*>(&count), sizeof(uint64_t));
offset += sizeof(uint64_t);
// Allocate memory for data stream.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Initialize the data stream.
uint64_t offset = 0;
std::memcpy(dataStream, reinterpret_cast<char*>(&count), sizeof(uint64_t));
offset += sizeof(uint64_t);
// For each pair [oid, key type].
for (const auto& p : keyTypes)
{
uint32_t oid = p.first;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&oid), sizeof(uint32_t));
offset += sizeof(uint32_t);
// For each pair [oid, key type].
for (const auto& p : keyTypes)
{
uint32_t oid = p.first;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&oid), sizeof(uint32_t));
offset += sizeof(uint32_t);
KeyType keyType = p.second;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&keyType), sizeof(KeyType));
offset += sizeof(KeyType);
}
KeyType keyType = p.second;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&keyType), sizeof(KeyType));
offset += sizeof(KeyType);
}
return dataStreamSmartPtr;
return dataStreamSmartPtr;
}
void StatisticsManager::saveToFile()
{
std::lock_guard<std::mutex> lock(mut);
std::lock_guard<std::mutex> lock(mut);
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> out(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "wb", 1));
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> out(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "wb", 1));
if (!out)
{
BRM::log_errno("StatisticsManager::saveToFile(): open");
throw ios_base::failure("StatisticsManager::saveToFile(): open failed.");
}
if (!out)
{
BRM::log_errno("StatisticsManager::saveToFile(): open");
throw ios_base::failure("StatisticsManager::saveToFile(): open failed.");
}
// Compute hash.
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
utils::Hasher128 hasher;
// Compute hash.
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
utils::Hasher128 hasher;
// Prepare a statistics file header.
const uint32_t headerSize = sizeof(StatisticsFileHeader);
StatisticsFileHeader fileHeader;
std::memset(&fileHeader, 0, headerSize);
fileHeader.version = version;
fileHeader.epoch = epoch;
fileHeader.dataSize = dataStreamSize;
// Compute hash from the data.
fileHeader.dataHash = hasher(dataStreamSmartPtr.get(), dataStreamSize);
// Prepare a statistics file header.
const uint32_t headerSize = sizeof(StatisticsFileHeader);
StatisticsFileHeader fileHeader;
std::memset(&fileHeader, 0, headerSize);
fileHeader.version = version;
fileHeader.epoch = epoch;
fileHeader.dataSize = dataStreamSize;
// Compute hash from the data.
fileHeader.dataHash = hasher(dataStreamSmartPtr.get(), dataStreamSize);
// Write statistics file header.
uint64_t size = out->write(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
// Write statistics file header.
uint64_t size = out->write(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
// Write data.
size = out->write(dataStreamSmartPtr.get(), dataStreamSize);
if (size != dataStreamSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
// Write data.
size = out->write(dataStreamSmartPtr.get(), dataStreamSize);
if (size != dataStreamSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
}
void StatisticsManager::loadFromFile()
{
std::lock_guard<std::mutex> lock(mut);
// Check that stats file does exist.
if (!boost::filesystem::exists(statsFile))
return;
std::lock_guard<std::mutex> lock(mut);
// Check that stats file does exist.
if (!boost::filesystem::exists(statsFile))
return;
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> in(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", 1));
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> in(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", 1));
if (!in)
{
BRM::log_errno("StatisticsManager::loadFromFile(): open");
throw ios_base::failure(
"StatisticsManager::loadFromFile(): open failed. Check the error log.");
}
if (!in)
{
BRM::log_errno("StatisticsManager::loadFromFile(): open");
throw ios_base::failure("StatisticsManager::loadFromFile(): open failed. Check the error log.");
}
// Read the file header.
StatisticsFileHeader fileHeader;
const uint32_t headerSize = sizeof(StatisticsFileHeader);
int64_t size = in->read(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
// Read the file header.
StatisticsFileHeader fileHeader;
const uint32_t headerSize = sizeof(StatisticsFileHeader);
int64_t size = in->read(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
// Initialize fields from the file header.
version = fileHeader.version;
epoch = fileHeader.epoch;
const auto dataHash = fileHeader.dataHash;
const auto dataStreamSize = fileHeader.dataSize;
// Initialize fields from the file header.
version = fileHeader.version;
epoch = fileHeader.epoch;
const auto dataHash = fileHeader.dataHash;
const auto dataStreamSize = fileHeader.dataSize;
// Allocate the memory for the file data.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Allocate the memory for the file data.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Read the data.
uint64_t dataOffset = 0;
auto sizeToRead = dataStreamSize;
size = in->read(dataStream, sizeToRead);
sizeToRead -= size;
dataOffset += size;
while (sizeToRead > 0)
{
size = in->read(dataStream + dataOffset, sizeToRead);
if (size < 0)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
// Read the data.
uint64_t dataOffset = 0;
auto sizeToRead = dataStreamSize;
size = in->read(dataStream, sizeToRead);
sizeToRead -= size;
dataOffset += size;
}
while (sizeToRead > 0)
{
size = in->read(dataStream + dataOffset, sizeToRead);
if (size < 0)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
utils::Hasher128 hasher;
auto computedDataHash = hasher(dataStream, dataStreamSize);
if (dataHash != computedDataHash)
throw ios_base::failure("StatisticsManager::loadFromFile(): invalid file hash. ");
sizeToRead -= size;
dataOffset += size;
}
uint64_t count = 0;
std::memcpy(reinterpret_cast<char*>(&count), dataStream, sizeof(uint64_t));
uint64_t offset = sizeof(uint64_t);
utils::Hasher128 hasher;
auto computedDataHash = hasher(dataStream, dataStreamSize);
if (dataHash != computedDataHash)
throw ios_base::failure("StatisticsManager::loadFromFile(): invalid file hash. ");
uint64_t count = 0;
std::memcpy(reinterpret_cast<char*>(&count), dataStream, sizeof(uint64_t));
uint64_t offset = sizeof(uint64_t);
// For each pair.
for (uint64_t i = 0; i < count; ++i)
{
uint32_t oid;
KeyType keyType;
std::memcpy(reinterpret_cast<char*>(&oid), &dataStream[offset], sizeof(uint32_t));
offset += sizeof(uint32_t);
std::memcpy(reinterpret_cast<char*>(&keyType), &dataStream[offset], sizeof(KeyType));
offset += sizeof(KeyType);
// Insert pair.
keyTypes[oid] = keyType;
}
// For each pair.
for (uint64_t i = 0; i < count; ++i)
{
uint32_t oid;
KeyType keyType;
std::memcpy(reinterpret_cast<char*>(&oid), &dataStream[offset], sizeof(uint32_t));
offset += sizeof(uint32_t);
std::memcpy(reinterpret_cast<char*>(&keyType), &dataStream[offset], sizeof(KeyType));
offset += sizeof(KeyType);
// Insert pair.
keyTypes[oid] = keyType;
}
}
uint64_t StatisticsManager::computeHashFromStats()
{
utils::Hasher128 hasher;
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
return hasher(dataStreamSmartPtr.get(), dataStreamSize);
utils::Hasher128 hasher;
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
return hasher(dataStreamSmartPtr.get(), dataStreamSize);
}
void StatisticsManager::serialize(messageqcpp::ByteStream& bs)
{
uint64_t count = keyTypes.size();
bs << version;
bs << epoch;
bs << count;
uint64_t count = keyTypes.size();
bs << version;
bs << epoch;
bs << count;
for (const auto& keyType : keyTypes)
{
bs << keyType.first;
bs << (uint32_t) keyType.second;
}
for (const auto& keyType : keyTypes)
{
bs << keyType.first;
bs << (uint32_t)keyType.second;
}
}
void StatisticsManager::unserialize(messageqcpp::ByteStream& bs)
{
uint64_t count;
bs >> version;
bs >> epoch;
bs >> count;
uint64_t count;
bs >> version;
bs >> epoch;
bs >> count;
for (uint32_t i = 0; i < count; ++i)
{
uint32_t oid, keyType;
bs >> oid;
bs >> keyType;
keyTypes[oid] = static_cast<KeyType>(keyType);
}
for (uint32_t i = 0; i < count; ++i)
{
uint32_t oid, keyType;
bs >> oid;
bs >> keyType;
keyTypes[oid] = static_cast<KeyType>(keyType);
}
}
bool StatisticsManager::hasKey(uint32_t oid) { return keyTypes.count(oid) > 0 ? true : false; }
bool StatisticsManager::hasKey(uint32_t oid)
{
return keyTypes.count(oid) > 0 ? true : false;
}
KeyType StatisticsManager::getKeyType(uint32_t oid) { return keyTypes[oid]; }
KeyType StatisticsManager::getKeyType(uint32_t oid)
{
return keyTypes[oid];
}
StatisticsDistributor* StatisticsDistributor::instance()
{
static StatisticsDistributor* sd = new StatisticsDistributor();
return sd;
static StatisticsDistributor* sd = new StatisticsDistributor();
return sd;
}
void StatisticsDistributor::distributeStatistics()
{
countClients();
countClients();
{
std::lock_guard<std::mutex> lock(mut);
// No clients.
if (clientsCount == 0)
return;
#ifdef DEBUG_STATISTICS
std::cout << "Distribute statistics from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
#endif
messageqcpp::ByteStream msg, statsHash, statsBs;
// Current hash.
statsHash << statistics::StatisticsManager::instance()->computeHashFromStats();
// Statistics.
statistics::StatisticsManager::instance()->serialize(statsBs);
for (uint32_t i = 0; i < clientsCount; ++i)
{
std::lock_guard<std::mutex> lock(mut);
// No clients.
if (clientsCount == 0)
return;
try
{
messageqcpp::ByteStream::quadbyte qb = ANALYZE_TABLE_REC_STATS;
msg << qb;
auto exeMgrID = "ExeMgr" + std::to_string(i + 2);
// Create a client.
std::unique_ptr<messageqcpp::MessageQueueClient> exemgrClient(
new messageqcpp::MessageQueueClient(exeMgrID));
#ifdef DEBUG_STATISTICS
std::cout << "Distribute statistics from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
std::cout << "Try to connect to " << exeMgrID << std::endl;
#endif
messageqcpp::ByteStream msg, statsHash, statsBs;
// Current hash.
statsHash << statistics::StatisticsManager::instance()->computeHashFromStats();
// Statistics.
statistics::StatisticsManager::instance()->serialize(statsBs);
for (uint32_t i = 0; i < clientsCount; ++i)
// Try to connect to the client.
if (!exemgrClient->connect())
{
try
{
messageqcpp::ByteStream::quadbyte qb = ANALYZE_TABLE_REC_STATS;
msg << qb;
auto exeMgrID = "ExeMgr" + std::to_string(i + 2);
// Create a client.
std::unique_ptr<messageqcpp::MessageQueueClient> exemgrClient(
new messageqcpp::MessageQueueClient(exeMgrID));
msg.restart();
#ifdef DEBUG_STATISTICS
std::cout << "Try to connect to " << exeMgrID << std::endl;
std::cout << "Unable to connect to " << exeMgrID << std::endl;
#endif
// Try to connect to the client.
if (!exemgrClient->connect())
{
msg.restart();
#ifdef DEBUG_STATISTICS
std::cout << "Unable to connect to " << exeMgrID << std::endl;
#endif
continue;
}
#ifdef DEBUG_STATISTICS
std::cout
<< "Write flag ANALYZE_TABLE_REC_STATS from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a flag to client ExeMgr.
exemgrClient->write(msg);
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics hash from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a hash of the stats.
exemgrClient->write(statsHash);
// Read the state from Client.
msg.restart();
msg = exemgrClient->read();
msg >> qb;
// Do not need a stats.
if (qb == ANALYZE_TABLE_SUCCESS)
{
msg.restart();
continue;
}
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics bytestream from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a statistics to client ExeMgr.
exemgrClient->write(statsBs);
// Read the flag back from the client ExeMgr.
msg.restart();
msg = exemgrClient->read();
if (msg.length() == 0)
throw runtime_error("Lost conection to ExeMgr.");
#ifdef DEBUG_STATISTICS
std::cout << "Read flag on ExeMgr(Server) from ExeMgr(Client) " << std::endl;
#endif
msg.restart();
}
catch (std::exception& e)
{
msg.restart();
std::cerr << "distributeStatistics() failed with error: " << e.what() << std::endl;
}
catch (...)
{
msg.restart();
std::cerr << "distributeStatistics() failed with unknown error." << std::endl;
}
continue;
}
#ifdef DEBUG_STATISTICS
std::cout << "Write flag ANALYZE_TABLE_REC_STATS from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a flag to client ExeMgr.
exemgrClient->write(msg);
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics hash from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
#endif
// Write a hash of the stats.
exemgrClient->write(statsHash);
// Read the state from Client.
msg.restart();
msg = exemgrClient->read();
msg >> qb;
// Do not need a stats.
if (qb == ANALYZE_TABLE_SUCCESS)
{
msg.restart();
continue;
}
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics bytestream from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
#endif
// Write a statistics to client ExeMgr.
exemgrClient->write(statsBs);
// Read the flag back from the client ExeMgr.
msg.restart();
msg = exemgrClient->read();
if (msg.length() == 0)
throw runtime_error("Lost conection to ExeMgr.");
#ifdef DEBUG_STATISTICS
std::cout << "Read flag on ExeMgr(Server) from ExeMgr(Client) " << std::endl;
#endif
msg.restart();
}
catch (std::exception& e)
{
msg.restart();
std::cerr << "distributeStatistics() failed with error: " << e.what() << std::endl;
}
catch (...)
{
msg.restart();
std::cerr << "distributeStatistics() failed with unknown error." << std::endl;
}
}
}
}
void StatisticsDistributor::countClients()
{
#ifdef DEBUG_STATISTICS
std::cout << "count clients to distribute statistics " << std::endl;
std::cout << "count clients to distribute statistics " << std::endl;
#endif
auto* config = config::Config::makeConfig();
// Starting from the ExeMgr2, since the Server starts on the ExeMgr1.
std::atomic<uint32_t> exeMgrNumber(2);
auto* config = config::Config::makeConfig();
// Starting from the ExeMgr2, since the Server starts on the ExeMgr1.
std::atomic<uint32_t> exeMgrNumber(2);
try
try
{
while (true)
{
while (true)
{
auto exeMgrID = "ExeMgr" + std::to_string(exeMgrNumber);
auto exeMgrIP = config->getConfig(exeMgrID, "IPAddr");
if (exeMgrIP == "")
break;
auto exeMgrID = "ExeMgr" + std::to_string(exeMgrNumber);
auto exeMgrIP = config->getConfig(exeMgrID, "IPAddr");
if (exeMgrIP == "")
break;
#ifdef DEBUG_STATISTICS
std::cout << "Client: " << exeMgrID << std::endl;
std::cout << "Client: " << exeMgrID << std::endl;
#endif
++exeMgrNumber;
}
}
catch (std::exception& e)
{
std::cerr << "countClients() failed with error: " << e.what() << std::endl;
}
catch (...)
{
std::cerr << "countClients() failed with unknown error: ";
++exeMgrNumber;
}
}
catch (std::exception& e)
{
std::cerr << "countClients() failed with error: " << e.what() << std::endl;
}
catch (...)
{
std::cerr << "countClients() failed with unknown error: ";
}
clientsCount = exeMgrNumber - 2;
clientsCount = exeMgrNumber - 2;
#ifdef DEBUG_STATISTICS
std::cout << "Number of clients: " << clientsCount << std::endl;
std::cout << "Number of clients: " << clientsCount << std::endl;
#endif
}
} // namespace statistics
} // namespace statistics

111
utils/common/statistics.h
View File

@ -37,31 +37,30 @@ using namespace idbdatafile;
namespace statistics
{
// Represents a column key type:
// PK - primary key.
// FK - foreign key.
enum class KeyType : uint32_t
{
PK,
FK
PK,
FK
};
// Rerpresents types of statistics CS supports.
enum class StatisticsType : uint32_t
{
// A special statistics type, made to solve circular inner join problem.
PK_FK
// A special statistics type, made to solve circular inner join problem.
PK_FK
};
// Represetns a header for the statistics file.
struct StatisticsFileHeader
{
uint64_t version;
uint64_t epoch;
uint64_t dataHash;
uint64_t dataSize;
uint8_t offset[1024];
uint64_t version;
uint64_t epoch;
uint64_t dataHash;
uint64_t dataSize;
uint8_t offset[1024];
};
// This class is responsible for processing and storing statistics.
@ -69,56 +68,64 @@ struct StatisticsFileHeader
// the updated statistics into the special file.
class StatisticsManager
{
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsManager* instance();
// Analyzes the given `rowGroup` by processing it row by row and searching for foreign key.
void analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace);
// Ouputs stats to out stream.
void output(StatisticsType statisticsType = StatisticsType::PK_FK);
// Saves stats to the file.
void saveToFile();
// Loads stats from the file.
void loadFromFile();
void incEpoch() { ++epoch; }
// Serialize stats to the given `bs`.
void serialize(messageqcpp::ByteStream& bs);
// Unserialize stats from the given `bs`.
void unserialize(messageqcpp::ByteStream& bs);
// Computes hash from the current statistics data.
uint64_t computeHashFromStats();
// Checks whether statistics is available for the given `oid`.
bool hasKey(uint32_t oid);
// Returns a KeyType for the given `oid`.
KeyType getKeyType(uint32_t oid);
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsManager* instance();
// Analyzes the given `rowGroup` by processing it row by row and searching for foreign key.
void analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace);
// Ouputs stats to out stream.
void output(StatisticsType statisticsType = StatisticsType::PK_FK);
// Saves stats to the file.
void saveToFile();
// Loads stats from the file.
void loadFromFile();
void incEpoch()
{
++epoch;
}
// Serialize stats to the given `bs`.
void serialize(messageqcpp::ByteStream& bs);
// Unserialize stats from the given `bs`.
void unserialize(messageqcpp::ByteStream& bs);
// Computes hash from the current statistics data.
uint64_t computeHashFromStats();
// Checks whether statistics is available for the given `oid`.
bool hasKey(uint32_t oid);
// Returns a KeyType for the given `oid`.
KeyType getKeyType(uint32_t oid);
private:
std::map<uint32_t, KeyType> keyTypes;
StatisticsManager() : epoch(0), version(1) { IDBPolicy::init(true, false, "", 0); }
std::unique_ptr<char[]> convertStatsToDataStream(uint64_t& dataStreamSize);
private:
std::map<uint32_t, KeyType> keyTypes;
StatisticsManager() : epoch(0), version(1)
{
IDBPolicy::init(true, false, "", 0);
}
std::unique_ptr<char[]> convertStatsToDataStream(uint64_t& dataStreamSize);
std::mutex mut;
uint32_t epoch;
uint32_t version;
std::string statsFile = "/var/lib/columnstore/local/statistics";
std::mutex mut;
uint32_t epoch;
uint32_t version;
std::string statsFile = "/var/lib/columnstore/local/statistics";
};
// This class is responsible for distributing the statistics across all `ExeMgr` in a cluster.
class StatisticsDistributor
{
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsDistributor* instance();
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsDistributor* instance();
// Distribute stats across all `ExeMgr` in cluster by connecting to them using config file.
void distributeStatistics();
// Distribute stats across all `ExeMgr` in cluster by connecting to them using config file.
void distributeStatistics();
private:
StatisticsDistributor() : clientsCount(0) {}
// Count the number of clients by reading config file and evaluating `ExeMgr` fields.
void countClients();
uint32_t clientsCount;
std::mutex mut;
private:
StatisticsDistributor() : clientsCount(0)
{
}
// Count the number of clients by reading config file and evaluating `ExeMgr` fields.
void countClients();
uint32_t clientsCount;
std::mutex mut;
};
} // namespace statistics
} // namespace statistics

145
utils/common/stlpoolallocator.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/* Makes PoolAllocator STL-compliant */
@ -33,152 +33,151 @@
namespace utils
{
/* If using the pool allocator with a boost smart ptr, use an instance of this
as the deleter. */
struct BoostPoolDeallocator
{
inline void operator()(void* ptr) { };
inline void operator()(void* ptr){};
};
/* This is an STL-compliant wrapper for PoolAllocator + an optimization for containers
* that aren't entirely node based (ex: vectors and hash tables)
*/
template<class T>
template <class T>
class STLPoolAllocator
{
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<class U> struct rebind
{
typedef STLPoolAllocator<U> other;
};
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template <class U>
struct rebind
{
typedef STLPoolAllocator<U> other;
};
STLPoolAllocator() throw();
STLPoolAllocator(const STLPoolAllocator&) throw();
STLPoolAllocator(uint32_t capacity) throw();
template<class U> STLPoolAllocator(const STLPoolAllocator<U>&) throw();
~STLPoolAllocator();
STLPoolAllocator() throw();
STLPoolAllocator(const STLPoolAllocator&) throw();
STLPoolAllocator(uint32_t capacity) throw();
template <class U>
STLPoolAllocator(const STLPoolAllocator<U>&) throw();
~STLPoolAllocator();
STLPoolAllocator<T>& operator=(const STLPoolAllocator<T>&);
STLPoolAllocator<T>& operator=(const STLPoolAllocator<T>&);
void usePoolAllocator(boost::shared_ptr<PoolAllocator> b);
boost::shared_ptr<utils::PoolAllocator> getPoolAllocator();
void usePoolAllocator(boost::shared_ptr<PoolAllocator> b);
boost::shared_ptr<utils::PoolAllocator> getPoolAllocator();
pointer allocate(size_type, const void* hint = 0);
void deallocate(pointer p, size_type n);
size_type max_size() const throw();
inline uint64_t getMemUsage() const
{
return pa->getMemUsage();
}
pointer allocate(size_type, const void* hint = 0);
void deallocate(pointer p, size_type n);
size_type max_size() const throw();
inline uint64_t getMemUsage() const
{
return pa->getMemUsage();
}
void construct(pointer p, const T& val);
void destroy(pointer p);
void construct(pointer p, const T& val);
void destroy(pointer p);
static const uint32_t DEFAULT_SIZE = 32768 * sizeof(T);
static const uint32_t DEFAULT_SIZE = 32768 * sizeof(T);
boost::shared_ptr<utils::PoolAllocator> pa;
boost::shared_ptr<utils::PoolAllocator> pa;
};
template<class T>
template <class T>
STLPoolAllocator<T>::STLPoolAllocator() throw()
{
pa.reset(new PoolAllocator(DEFAULT_SIZE));
pa.reset(new PoolAllocator(DEFAULT_SIZE));
}
template<class T>
template <class T>
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<T>& s) throw()
{
pa = s.pa;
pa = s.pa;
}
template<class T>
template <class T>
STLPoolAllocator<T>::STLPoolAllocator(uint32_t capacity) throw()
{
pa.reset(new PoolAllocator(capacity));
pa.reset(new PoolAllocator(capacity));
}
template<class T>
template<class U>
template <class T>
template <class U>
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<U>& s) throw()
{
pa = s.pa;
pa = s.pa;
}
template<class T>
template <class T>
STLPoolAllocator<T>::~STLPoolAllocator()
{
}
template<class T>
template <class T>
void STLPoolAllocator<T>::usePoolAllocator(boost::shared_ptr<PoolAllocator> p)
{
pa = p;
pa = p;
}
template<class T>
template <class T>
boost::shared_ptr<utils::PoolAllocator> STLPoolAllocator<T>::getPoolAllocator()
{
return pa;
return pa;
}
template<class T>
typename STLPoolAllocator<T>::pointer
STLPoolAllocator<T>::allocate(typename STLPoolAllocator<T>::size_type s,
typename std::allocator<void>::const_pointer hint)
template <class T>
typename STLPoolAllocator<T>::pointer STLPoolAllocator<T>::allocate(
typename STLPoolAllocator<T>::size_type s, typename std::allocator<void>::const_pointer hint)
{
return (pointer) pa->allocate(s * sizeof(T));
return (pointer)pa->allocate(s * sizeof(T));
}
template<class T>
template <class T>
void STLPoolAllocator<T>::deallocate(typename STLPoolAllocator<T>::pointer p,
typename STLPoolAllocator<T>::size_type n)
{
pa->deallocate((void*) p);
pa->deallocate((void*)p);
}
template<class T>
template <class T>
typename STLPoolAllocator<T>::size_type STLPoolAllocator<T>::max_size() const throw()
{
return std::numeric_limits<uint64_t>::max() / sizeof(T);
return std::numeric_limits<uint64_t>::max() / sizeof(T);
}
template<class T>
template <class T>
void STLPoolAllocator<T>::construct(typename STLPoolAllocator<T>::pointer p, const T& val)
{
new ((void*)p) T(val);
new ((void*)p) T(val);
}
template<class T>
template <class T>
void STLPoolAllocator<T>::destroy(typename STLPoolAllocator<T>::pointer p)
{
p->T::~T();
p->T::~T();
}
template<class T>
template <class T>
STLPoolAllocator<T>& STLPoolAllocator<T>::operator=(const STLPoolAllocator<T>& c)
{
pa = c.pa;
return *this;
pa = c.pa;
return *this;
}
template<typename T>
template <typename T>
bool operator==(const STLPoolAllocator<T>&, const STLPoolAllocator<T>&)
{
return true;
return true;
}
template<typename T>
template <typename T>
bool operator!=(const STLPoolAllocator<T>&, const STLPoolAllocator<T>&)
{
return false;
}
return false;
}
} // namespace utils

159
utils/common/syncstream.h
View File

@ -30,28 +30,28 @@
void copyStream(istream& iss, ostream& oss)
{
string line;
getline(iss, line);
while (iss.good())
{
oss << line << endl;
getline(iss, line);
}
string line;
getline(iss, line);
while (iss.good())
{
oss << line << endl;
getline(iss, line);
}
}
main()
{
FILE* ifp;
FILE* ofp;
FILE* ifp;
FILE* ofp;
...
...
isyncstream iss(ifp);
osyncstream oss(ofp);
isyncstream iss(ifp);
osyncstream oss(ofp);
copyStream(iss, oss);
copyStream(iss, oss);
...
...
}
*/
@ -62,89 +62,96 @@ main()
namespace syncstream
{
/** A streambuf implementation for C stdio FILE* streams.
*
* Adapted from http://www.drdobbs.com/184401305
*/
class syncbuf : public std::streambuf
{
public:
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f) {}
public:
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f)
{
}
protected:
/** Write character in the case of overflow */
virtual int overflow(int c = EOF)
{
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow()
{
int c = getc(fptr);
protected:
/** Write character in the case of overflow */
virtual int overflow(int c = EOF)
{
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow()
{
int c = getc(fptr);
if (c != EOF)
ungetc(c, fptr);
if (c != EOF)
ungetc(c, fptr);
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow()
{
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF)
{
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync()
{
return fflush(fptr);
}
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow()
{
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF)
{
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync()
{
return fflush(fptr);
}
private:
FILE* fptr;
private:
FILE* fptr;
};
/** An istream adaptor for input FILE* streams */
class isyncstream : public std::istream
{
public:
/** ctor */
isyncstream() : istream(&buf), buf(0) {}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
public:
/** ctor */
isyncstream() : istream(&buf), buf(0)
{
}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
private:
syncbuf buf;
};
/** An ostream adaptor for output FILE* streams */
class osyncstream : public std::ostream
{
public:
/** ctor */
osyncstream() : ostream(&buf), buf(0) {}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
public:
/** ctor */
osyncstream() : ostream(&buf), buf(0)
{
}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
private:
syncbuf buf;
};
}
} // namespace syncstream

22
utils/common/threadnaming.cpp
View File

@ -20,15 +20,15 @@
namespace utils
{
void setThreadName(const char *threadName)
{
prctl(PR_SET_NAME, threadName, 0, 0, 0);
}
void setThreadName(const char* threadName)
{
prctl(PR_SET_NAME, threadName, 0, 0, 0);
}
std::string getThreadName()
{
char buf[32];
prctl(PR_GET_NAME, buf, 0, 0, 0);
return std::string(buf);
}
} // end of namespace
std::string getThreadName()
{
char buf[32];
prctl(PR_GET_NAME, buf, 0, 0, 0);
return std::string(buf);
}
} // namespace utils

6
utils/common/threadnaming.h
View File

@ -20,6 +20,6 @@
namespace utils
{
void setThreadName(const char *threadName);
std::string getThreadName();
} // end of namespace
void setThreadName(const char* threadName);
std::string getThreadName();
} // namespace utils

15
utils/common/utils_utf8.cpp
View File

@ -17,21 +17,16 @@
#include "mariadb_my_sys.h"
namespace datatypes
{
static inline CHARSET_INFO & get_charset_or_bin(int32_t charsetNumber)
static inline CHARSET_INFO& get_charset_or_bin(int32_t charsetNumber)
{
CHARSET_INFO *cs= get_charset(charsetNumber, MYF(MY_WME));
return cs ? *cs : my_charset_bin;
CHARSET_INFO* cs = get_charset(charsetNumber, MYF(MY_WME));
return cs ? *cs : my_charset_bin;
}
Charset::Charset(uint32_t charsetNumber)
:mCharset(&get_charset_or_bin(charsetNumber))
Charset::Charset(uint32_t charsetNumber) : mCharset(&get_charset_or_bin(charsetNumber))
{
}
}
} // namespace datatypes

139
utils/common/utils_utf8.h
View File

@ -18,11 +18,8 @@
// $Id$
#pragma once
#include <string>
#if defined(_MSC_VER)
#include <malloc.h>
@ -40,120 +37,116 @@
// Change the name from utf8. Even change the file name to something resembling char helper
namespace utf8
{
const int MAX_UTF8_BYTES_PER_CHAR = 4;
// BUG 5241
// Infinidb specific mbstowcs(). This will handle both windows and unix platforms
// Params dest and max should have enough length to accomodate NULL
inline
size_t idb_mbstowcs(wchar_t* dest, const char* src, size_t max)
inline size_t idb_mbstowcs(wchar_t* dest, const char* src, size_t max)
{
#ifdef _MSC_VER
// 4th param (-1) denotes to convert till hit NULL char
// if 6th param max = 0, will return the required buffer size
size_t strwclen = MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)max);
// decrement the count of NULL; will become -1 on failure
return --strwclen;
// 4th param (-1) denotes to convert till hit NULL char
// if 6th param max = 0, will return the required buffer size
size_t strwclen = MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)max);
// decrement the count of NULL; will become -1 on failure
return --strwclen;
#else
return mbstowcs(dest, src, max);
return mbstowcs(dest, src, max);
#endif
}
// BUG 5241
// Infinidb specific wcstombs(). This will handle both windows and unix platforms
// Params dest and max should have enough length to accomodate NULL
inline
size_t idb_wcstombs(char* dest, const wchar_t* src, size_t max)
inline size_t idb_wcstombs(char* dest, const wchar_t* src, size_t max)
{
#ifdef _MSC_VER
// 4th param (-1) denotes to convert till hit NULL char
//if 6th param max = 0, will return the required buffer size
size_t strmblen = WideCharToMultiByte( CP_UTF8, 0, src, -1, dest, (int)max, NULL, NULL);
// decrement the count of NULL; will become -1 on failure
return --strmblen;
// 4th param (-1) denotes to convert till hit NULL char
// if 6th param max = 0, will return the required buffer size
size_t strmblen = WideCharToMultiByte(CP_UTF8, 0, src, -1, dest, (int)max, NULL, NULL);
// decrement the count of NULL; will become -1 on failure
return --strmblen;
#else
return wcstombs(dest, src, max);
return wcstombs(dest, src, max);
#endif
}
// convert UTF-8 string to wstring
inline
std::wstring utf8_to_wstring (const std::string& str)
inline std::wstring utf8_to_wstring(const std::string& str)
{
size_t bufsize = str.length() + 1;
size_t bufsize = str.length() + 1;
// Convert to wide characters. Do all further work in wide characters
wchar_t* wcbuf = new wchar_t[bufsize];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strwclen = idb_mbstowcs(wcbuf, str.c_str(), str.length() + 1);
// Convert to wide characters. Do all further work in wide characters
wchar_t* wcbuf = new wchar_t[bufsize];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strwclen = idb_mbstowcs(wcbuf, str.c_str(), str.length() + 1);
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if ( strwclen == static_cast<size_t>(-1) )
strwclen = 0;
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if (strwclen == static_cast<size_t>(-1))
strwclen = 0;
std::wstring ret(wcbuf, strwclen);
std::wstring ret(wcbuf, strwclen);
delete [] wcbuf;
return ret;
delete[] wcbuf;
return ret;
}
// convert wstring to UTF-8 string
inline
std::string wstring_to_utf8 (const std::wstring& str)
inline std::string wstring_to_utf8(const std::wstring& str)
{
char* outbuf = new char[(str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strmblen = idb_wcstombs(outbuf, str.c_str(), str.length() * MAX_UTF8_BYTES_PER_CHAR + 1);
char* outbuf = new char[(str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strmblen = idb_wcstombs(outbuf, str.c_str(), str.length() * MAX_UTF8_BYTES_PER_CHAR + 1);
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if ( strmblen == static_cast<size_t>(-1) )
strmblen = 0;
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if (strmblen == static_cast<size_t>(-1))
strmblen = 0;
std::string ret(outbuf, strmblen);
std::string ret(outbuf, strmblen);
delete [] outbuf;
return ret;
delete[] outbuf;
return ret;
}
inline
uint8_t utf8_truncate_point(const char* input, size_t length)
inline uint8_t utf8_truncate_point(const char* input, size_t length)
{
// Find the beginning of a multibyte char to truncate at and return the
// number of bytes to truncate1`
if (length < 3)
{
return 0;
}
const unsigned char* b = (const unsigned char*)(input) + length - 3;
if (b[2] & 0x80)
{
// First byte in a new multi-byte sequence
if (b[2] & 0x40) return 1;
// 3 byte sequence
else if ((b[1] & 0xe0) == 0xe0) return 2;
// 4 byte sequence
else if ((b[0] & 0xf0) == 0xf0) return 3;
}
// Find the beginning of a multibyte char to truncate at and return the
// number of bytes to truncate1`
if (length < 3)
{
return 0;
}
const unsigned char* b = (const unsigned char*)(input) + length - 3;
if (b[2] & 0x80)
{
// First byte in a new multi-byte sequence
if (b[2] & 0x40)
return 1;
// 3 byte sequence
else if ((b[1] & 0xe0) == 0xe0)
return 2;
// 4 byte sequence
else if ((b[0] & 0xf0) == 0xf0)
return 3;
}
return 0;
}
int mcs_strcoll(const char* str1, const char* str2, const uint32_t charsetNumber);
int mcs_strcoll(const char* str1, const uint32_t l1, const char* str2, const uint32_t l2, const uint32_t charsetNumber);
int mcs_strcoll(const char* str1, const uint32_t l1, const char* str2, const uint32_t l2,
const uint32_t charsetNumber);
int mcs_strcoll(const std::string* str1, const std::string* str2, const uint32_t charsetNumber);
int mcs_strcoll(const std::string& str1, const std::string& str2, const uint32_t charsetNumber);
int mcs_strcollsp(const char* str1, const char* str2, const uint32_t charsetNumber);
int mcs_strcollsp(const char* str1, uint32_t l1, const char* str2, const uint32_t l2, const uint32_t charsetNumber);
int mcs_strcollsp(const char* str1, uint32_t l1, const char* str2, const uint32_t l2,
const uint32_t charsetNumber);
int mcs_strcollsp(const std::string* str1, const std::string* str2, const uint32_t charsetNumber);
int mcs_strcollsp(const std::string& str1, const std::string& str2, const uint32_t charsetNumber);
} //namespace utf8
} // namespace utf8

154
utils/common/vlarray.h
View File

@ -16,83 +16,109 @@
MA 02110-1301, USA. */
#pragma once
namespace utils {
template <typename T, size_t SIZE=64>
namespace utils
{
template <typename T, size_t SIZE = 64>
class VLArray
{
public:
VLArray(size_t sz) :
sz(sz),
stack_storage(NULL),
dyn_storage(NULL),
ptr(NULL)
public:
VLArray(size_t sz) : sz(sz), stack_storage(NULL), dyn_storage(NULL), ptr(NULL)
{
if (sz > SIZE)
{
if (sz > SIZE) {
dyn_storage = new T[sz];
ptr = dyn_storage;
} else {
stack_storage = new (stack) T[sz];
ptr = stack_storage;
}
dyn_storage = new T[sz];
ptr = dyn_storage;
}
VLArray(size_t sz, const T& initval) : VLArray(sz)
else
{
for (size_t i= 0; i < sz; ++i)
ptr[i]= initval;
stack_storage = new (stack) T[sz];
ptr = stack_storage;
}
}
VLArray(const VLArray&) = delete;
VLArray(VLArray&&) = delete;
VLArray& operator=(const VLArray&) = delete;
VLArray& operator=(VLArray&&) = delete;
VLArray(size_t sz, const T& initval) : VLArray(sz)
{
for (size_t i = 0; i < sz; ++i)
ptr[i] = initval;
}
~VLArray() {
if (dyn_storage) {
delete [] dyn_storage;
} else {
// we cannot use `delete [] stack_storage` here so call d-tors explicitly
if (!std::is_trivially_destructible<T>::value) {
for (size_t i = 0; i < sz; ++i)
stack_storage[i].~T();
}
}
VLArray(const VLArray&) = delete;
VLArray(VLArray&&) = delete;
VLArray& operator=(const VLArray&) = delete;
VLArray& operator=(VLArray&&) = delete;
~VLArray()
{
if (dyn_storage)
{
delete[] dyn_storage;
}
size_t size() const { return sz; }
const T* data() const { return ptr; }
T* data() { return ptr; }
const T& operator[](size_t i) const { return ptr[i]; }
T& operator[](size_t i) { return ptr[i]; }
const T& at(size_t i) const {
if (i >= sz) {
throw std::out_of_range("index out of range: " + std::to_string(i) +
" >= size " + std::to_string(sz));
}
return ptr[i];
else
{
// we cannot use `delete [] stack_storage` here so call d-tors explicitly
if (!std::is_trivially_destructible<T>::value)
{
for (size_t i = 0; i < sz; ++i)
stack_storage[i].~T();
}
}
}
T& at(size_t i) {
if (i >= sz) {
throw std::out_of_range("index out of range: " + std::to_string(i) +
" >= size " + std::to_string(sz));
}
return ptr[i];
size_t size() const
{
return sz;
}
const T* data() const
{
return ptr;
}
T* data()
{
return ptr;
}
const T& operator[](size_t i) const
{
return ptr[i];
}
T& operator[](size_t i)
{
return ptr[i];
}
const T& at(size_t i) const
{
if (i >= sz)
{
throw std::out_of_range("index out of range: " + std::to_string(i) + " >= size " + std::to_string(sz));
}
return ptr[i];
}
operator const T* () const { return ptr; }
operator T* () { return ptr; }
T& at(size_t i)
{
if (i >= sz)
{
throw std::out_of_range("index out of range: " + std::to_string(i) + " >= size " + std::to_string(sz));
}
return ptr[i];
}
private:
const size_t sz;
alignas(T) char stack[SIZE * sizeof(T)];
T* stack_storage;
T* dyn_storage;
T* ptr;
operator const T*() const
{
return ptr;
}
operator T*()
{
return ptr;
}
private:
const size_t sz;
alignas(T) char stack[SIZE * sizeof(T)];
T* stack_storage;
T* dyn_storage;
T* ptr;
};
} // namespace utils
} // namespace utils

39
utils/common/widedecimalutils.h
View File

@ -22,26 +22,25 @@
namespace utils
{
const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64;
const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL;
const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64;
const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL;
inline void int128Max(int128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0x7FFFFFFFFFFFFFFF;
}
inline void int128Min(int128_t& val)
{
val = int128_t(0x8000000000000000LL) << 64;
}
inline void uint128Max(uint128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0xFFFFFFFFFFFFFFFF;
}
inline void int128Max(int128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0x7FFFFFFFFFFFFFFF;
}
inline void int128Min(int128_t& val)
{
val = int128_t(0x8000000000000000LL) << 64;
}
inline void uint128Max(uint128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0xFFFFFFFFFFFFFFFF;
}
} // namespace utils

632
utils/compress/idbcompress.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: idbcompress.cpp 3907 2013-06-18 13:32:46Z dcathey $
*
******************************************************************************************/
* $Id: idbcompress.cpp 3907 2013-06-18 13:32:46Z dcathey $
*
******************************************************************************************/
#include <cstring>
#include <iostream>
#include <stdexcept>
@ -34,8 +34,9 @@ using namespace std;
#include "lz4.h"
#else
// Taken from lz4.h.
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ4_COMPRESSBOUND(isize) \
((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize) / 255) + 16)
#endif
#define IDBCOMP_DLLEXPORT
@ -47,7 +48,6 @@ namespace
const uint64_t MAGIC_NUMBER = 0xfdc119a384d0778eULL;
const uint64_t VERSION_NUM3 = 3;
// version 1.1 of the chunk data has a short header
// QuickLZ compressed data never has the high bit set on the first byte
const int SIG_OFFSET = 0;
@ -60,164 +60,159 @@ const uint32_t LBID_MAX_SIZE = 10;
struct CompressedDBFileHeader
{
uint64_t fMagicNumber;
uint64_t fVersionNum;
uint64_t fCompressionType;
uint64_t fHeaderSize;
uint64_t fBlockCount;
uint64_t fColumnWidth;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
uint64_t fLBIDCount;
uint64_t fLBIDS[LBID_MAX_SIZE];
uint64_t fMagicNumber;
uint64_t fVersionNum;
uint64_t fCompressionType;
uint64_t fHeaderSize;
uint64_t fBlockCount;
uint64_t fColumnWidth;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
uint64_t fLBIDCount;
uint64_t fLBIDS[LBID_MAX_SIZE];
};
// Make the header to be 4K, regardless number of fields being defined/used in header.
union CompressedDBFileHeaderBlock
{
CompressedDBFileHeader fHeader;
char fDummy[compress::CompressInterface::HDR_BUF_LEN];
CompressedDBFileHeader fHeader;
char fDummy[compress::CompressInterface::HDR_BUF_LEN];
};
void initCompressedDBFileHeader(
void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
int compressionType, int hdrSize)
void initCompressedDBFileHeader(void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType, int compressionType,
int hdrSize)
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM3;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = columnWidth;
hdr->fHeader.fColDataType = colDataType;
hdr->fHeader.fLBIDCount = 0;
std::memset(hdr->fHeader.fLBIDS, 0, sizeof(hdr->fHeader.fLBIDS));
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM3;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = columnWidth;
hdr->fHeader.fColDataType = colDataType;
hdr->fHeader.fLBIDCount = 0;
std::memset(hdr->fHeader.fLBIDS, 0, sizeof(hdr->fHeader.fLBIDS));
}
} // namespace
} // namespace
namespace compress
{
#ifndef SKIP_IDB_COMPRESSION
CompressInterface::CompressInterface(unsigned int numUserPaddingBytes) :
fNumUserPaddingBytes(numUserPaddingBytes)
{ }
CompressInterface::CompressInterface(unsigned int numUserPaddingBytes)
: fNumUserPaddingBytes(numUserPaddingBytes)
{
}
/* V1 is really only available for decompression, we kill any DDL using V1 by hand.
* Maybe should have a new api, isDecompressionAvail() ? Any request to compress
* using V1 will silently be changed to V2.
*/
*/
/*static*/
bool CompressInterface::isCompressionAvail(int compressionType)
{
return ((compressionType == 0) || (compressionType == 1) ||
(compressionType == 2) || (compressionType == 3));
return ((compressionType == 0) || (compressionType == 1) || (compressionType == 2) ||
(compressionType == 3));
}
size_t CompressInterface::getMaxCompressedSizeGeneric(size_t inLen)
{
return std::max(snappy::MaxCompressedLength(inLen),
LZ4_COMPRESSBOUND(inLen)) +
HEADER_SIZE;
return std::max(snappy::MaxCompressedLength(inLen), LZ4_COMPRESSBOUND(inLen)) + HEADER_SIZE;
}
//------------------------------------------------------------------------------
// Compress a block of data
//------------------------------------------------------------------------------
int CompressInterface::compressBlock(const char* in, const size_t inLen,
unsigned char* out, size_t& outLen) const
int CompressInterface::compressBlock(const char* in, const size_t inLen, unsigned char* out,
size_t& outLen) const
{
size_t snaplen = 0;
utils::Hasher128 hasher;
size_t snaplen = 0;
utils::Hasher128 hasher;
// loose input checking.
if (outLen < maxCompressedSize(inLen))
{
cerr << "got outLen = " << outLen << " for inLen = " << inLen
<< ", needed " << (maxCompressedSize(inLen)) << endl;
return ERR_BADOUTSIZE;
}
// loose input checking.
if (outLen < maxCompressedSize(inLen))
{
cerr << "got outLen = " << outLen << " for inLen = " << inLen << ", needed " << (maxCompressedSize(inLen))
<< endl;
return ERR_BADOUTSIZE;
}
auto rc = compress(in, inLen, reinterpret_cast<char*>(&out[HEADER_SIZE]),
&outLen);
if (rc != ERR_OK)
{
return rc;
}
auto rc = compress(in, inLen, reinterpret_cast<char*>(&out[HEADER_SIZE]), &outLen);
if (rc != ERR_OK)
{
return rc;
}
snaplen = outLen;
uint8_t* signature = (uint8_t*) &out[SIG_OFFSET];
uint32_t* checksum = (uint32_t*) &out[CHECKSUM_OFFSET];
uint32_t* len = (uint32_t*) &out[LEN_OFFSET];
*signature = getChunkMagicNumber();
*checksum = hasher((char*) &out[HEADER_SIZE], snaplen);
*len = snaplen;
snaplen = outLen;
uint8_t* signature = (uint8_t*)&out[SIG_OFFSET];
uint32_t* checksum = (uint32_t*)&out[CHECKSUM_OFFSET];
uint32_t* len = (uint32_t*)&out[LEN_OFFSET];
*signature = getChunkMagicNumber();
*checksum = hasher((char*)&out[HEADER_SIZE], snaplen);
*len = snaplen;
//cerr << "cb: " << inLen << '/' << outLen << '/' << (snappy::MaxCompressedLength(inLen) + HEADER_SIZE) <<
// " : " << (snaplen + HEADER_SIZE) << endl;
// cerr << "cb: " << inLen << '/' << outLen << '/' << (snappy::MaxCompressedLength(inLen) + HEADER_SIZE) <<
// " : " << (snaplen + HEADER_SIZE) << endl;
outLen = snaplen + HEADER_SIZE;
outLen = snaplen + HEADER_SIZE;
return ERR_OK;
return ERR_OK;
}
//------------------------------------------------------------------------------
// Decompress a block of data
//------------------------------------------------------------------------------
int CompressInterface::uncompressBlock(const char* in, const size_t inLen,
unsigned char* out,
int CompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out,
size_t& outLen) const
{
uint32_t realChecksum;
uint32_t storedChecksum;
uint32_t storedLen;
uint8_t storedMagic;
utils::Hasher128 hasher;
auto tmpOutLen = outLen;
outLen = 0;
uint32_t realChecksum;
uint32_t storedChecksum;
uint32_t storedLen;
uint8_t storedMagic;
utils::Hasher128 hasher;
auto tmpOutLen = outLen;
outLen = 0;
if (inLen < 1)
return ERR_BADINPUT;
if (inLen < 1)
return ERR_BADINPUT;
storedMagic = *((uint8_t*) &in[SIG_OFFSET]);
storedMagic = *((uint8_t*)&in[SIG_OFFSET]);
if (storedMagic == getChunkMagicNumber())
if (storedMagic == getChunkMagicNumber())
{
if (inLen < HEADER_SIZE)
return ERR_BADINPUT;
storedChecksum = *((uint32_t*)&in[CHECKSUM_OFFSET]);
storedLen = *((uint32_t*)(&in[LEN_OFFSET]));
if (inLen < storedLen + HEADER_SIZE)
return ERR_BADINPUT;
realChecksum = hasher(&in[HEADER_SIZE], storedLen);
if (storedChecksum != realChecksum)
return ERR_CHECKSUM;
auto rc = uncompress(&in[HEADER_SIZE], storedLen, reinterpret_cast<char*>(out), &tmpOutLen);
if (rc != ERR_OK)
{
if (inLen < HEADER_SIZE)
return ERR_BADINPUT;
storedChecksum = *((uint32_t*) &in[CHECKSUM_OFFSET]);
storedLen = *((uint32_t*) (&in[LEN_OFFSET]));
if (inLen < storedLen + HEADER_SIZE)
return ERR_BADINPUT;
realChecksum = hasher(&in[HEADER_SIZE], storedLen);
if (storedChecksum != realChecksum)
return ERR_CHECKSUM;
auto rc = uncompress(&in[HEADER_SIZE], storedLen, reinterpret_cast<char*>(out), &tmpOutLen);
if (rc != ERR_OK)
{
cerr << "uncompressBlock failed!" << endl;
return ERR_DECOMPRESS;
}
outLen = tmpOutLen;
}
else
{
// v1 compression or bad header
return ERR_BADINPUT;
cerr << "uncompressBlock failed!" << endl;
return ERR_DECOMPRESS;
}
//cerr << "ub: " << inLen << " : " << outLen << endl;
outLen = tmpOutLen;
}
else
{
// v1 compression or bad header
return ERR_BADINPUT;
}
return ERR_OK;
// cerr << "ub: " << inLen << " : " << outLen << endl;
return ERR_OK;
}
//------------------------------------------------------------------------------
@ -225,42 +220,41 @@ int CompressInterface::uncompressBlock(const char* in, const size_t inLen,
//------------------------------------------------------------------------------
int CompressInterface::verifyHdr(const void* hdrBuf)
{
const CompressedDBFileHeader* hdr = reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf);
const CompressedDBFileHeader* hdr = reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf);
if (hdr->fMagicNumber != MAGIC_NUMBER)
return -1;
if (hdr->fMagicNumber != MAGIC_NUMBER)
return -1;
if (!isCompressionAvail(hdr->fCompressionType))
return -2;
if (!isCompressionAvail(hdr->fCompressionType))
return -2;
return 0;
return 0;
}
//------------------------------------------------------------------------------
// Extract compression pointer information out of the pointer buffer that is
// passed in. ptrBuf points to the pointer section of the compression hdr.
//------------------------------------------------------------------------------
int CompressInterface::getPtrList(const char* ptrBuf, const int ptrBufSize,
CompChunkPtrList& chunkPtrs)
int CompressInterface::getPtrList(const char* ptrBuf, const int ptrBufSize, CompChunkPtrList& chunkPtrs)
{
int rc = 0;
chunkPtrs.clear();
int rc = 0;
chunkPtrs.clear();
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
for (unsigned int i = 0; (i < NUM_PTRS) && (rc == 0); i++)
{
if (ptrs[i + 1] == 0) // 0 offset means end of data
break;
for (unsigned int i = 0; (i < NUM_PTRS) && (rc == 0); i++)
{
if (ptrs[i + 1] == 0) // 0 offset means end of data
break;
if (ptrs[i + 1] > ptrs[i])
chunkPtrs.push_back(make_pair( ptrs[i], (ptrs[i + 1] - ptrs[i])));
else
rc = -1;
}
if (ptrs[i + 1] > ptrs[i])
chunkPtrs.push_back(make_pair(ptrs[i], (ptrs[i + 1] - ptrs[i])));
else
rc = -1;
}
return rc;
return rc;
}
//------------------------------------------------------------------------------
@ -269,31 +263,30 @@ int CompressInterface::getPtrList(const char* ptrBuf, const int ptrBufSize,
// one for the file header, and one for the list of pointers.
// Wrapper of above method for backward compatibility.
//------------------------------------------------------------------------------
int CompressInterface::getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs )
int CompressInterface::getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs)
{
return getPtrList(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN, chunkPtrs);
return getPtrList(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN, chunkPtrs);
}
//------------------------------------------------------------------------------
// Count the number of chunk pointers in the pointer header(s)
//------------------------------------------------------------------------------
unsigned int CompressInterface::getPtrCount(const char* ptrBuf,
const int ptrBufSize)
unsigned int CompressInterface::getPtrCount(const char* ptrBuf, const int ptrBufSize)
{
unsigned int chunkCount = 0;
unsigned int chunkCount = 0;
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
for (unsigned int i = 0; i < NUM_PTRS; i++)
{
if (ptrs[i + 1] == 0) // 0 offset means end of data
break;
for (unsigned int i = 0; i < NUM_PTRS; i++)
{
if (ptrs[i + 1] == 0) // 0 offset means end of data
break;
chunkCount++;
}
chunkCount++;
}
return chunkCount;
return chunkCount;
}
//------------------------------------------------------------------------------
@ -304,22 +297,21 @@ unsigned int CompressInterface::getPtrCount(const char* ptrBuf,
//------------------------------------------------------------------------------
unsigned int CompressInterface::getPtrCount(const char* hdrBuf)
{
return getPtrCount(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN);
return getPtrCount(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN);
}
//------------------------------------------------------------------------------
// Store list of compression pointers into the specified header.
//------------------------------------------------------------------------------
void CompressInterface::storePtrs(const std::vector<uint64_t>& ptrs,
void* ptrBuf, int ptrSectionSize)
void CompressInterface::storePtrs(const std::vector<uint64_t>& ptrs, void* ptrBuf, int ptrSectionSize)
{
memset((ptrBuf), 0, ptrSectionSize); // reset the pointer section to 0
uint64_t* hdrPtrs = reinterpret_cast<uint64_t*>(ptrBuf);
memset((ptrBuf), 0, ptrSectionSize); // reset the pointer section to 0
uint64_t* hdrPtrs = reinterpret_cast<uint64_t*>(ptrBuf);
for (unsigned i = 0; i < ptrs.size(); i++)
{
hdrPtrs[i] = ptrs[i];
}
for (unsigned i = 0; i < ptrs.size(); i++)
{
hdrPtrs[i] = ptrs[i];
}
}
//------------------------------------------------------------------------------
@ -327,34 +319,29 @@ void CompressInterface::storePtrs(const std::vector<uint64_t>& ptrs,
//------------------------------------------------------------------------------
void CompressInterface::storePtrs(const std::vector<uint64_t>& ptrs, void* ptrBuf)
{
storePtrs(ptrs, reinterpret_cast<char*>(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN);
storePtrs(ptrs, reinterpret_cast<char*>(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a dictionary file.
//------------------------------------------------------------------------------
void CompressInterface::initHdr(
void* hdrBuf, void* ptrBuf, uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType, int hdrSize)
void CompressInterface::initHdr(void* hdrBuf, void* ptrBuf, uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType,
int hdrSize)
{
memset(hdrBuf, 0, HDR_BUF_LEN);
memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN);
initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType,
hdrSize);
memset(hdrBuf, 0, HDR_BUF_LEN);
memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN);
initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType, hdrSize);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a column file.
//------------------------------------------------------------------------------
void CompressInterface::initHdr(
void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType)
void CompressInterface::initHdr(void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType)
{
memset(hdrBuf, 0, HDR_BUF_LEN * 2);
initCompressedDBFileHeader(hdrBuf, columnWidth, columnType,
compressionType, HDR_BUF_LEN * 2);
memset(hdrBuf, 0, HDR_BUF_LEN * 2);
initCompressedDBFileHeader(hdrBuf, columnWidth, columnType, compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
@ -362,8 +349,7 @@ void CompressInterface::initHdr(
//------------------------------------------------------------------------------
uint64_t CompressInterface::getVersionNumber(const void* hdrBuf)
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fVersionNum);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fVersionNum);
}
//------------------------------------------------------------------------------
@ -371,7 +357,7 @@ uint64_t CompressInterface::getVersionNumber(const void* hdrBuf)
//------------------------------------------------------------------------------
void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count)
{
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fBlockCount = count;
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fBlockCount = count;
}
//------------------------------------------------------------------------------
@ -379,7 +365,7 @@ void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count)
//------------------------------------------------------------------------------
uint64_t CompressInterface::getBlockCount(const void* hdrBuf)
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fBlockCount);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fBlockCount);
}
//------------------------------------------------------------------------------
@ -387,8 +373,7 @@ uint64_t CompressInterface::getBlockCount(const void* hdrBuf)
//------------------------------------------------------------------------------
uint64_t CompressInterface::getCompressionType(const void* hdrBuf)
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)
->fCompressionType);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fCompressionType);
}
//------------------------------------------------------------------------------
@ -396,7 +381,7 @@ uint64_t CompressInterface::getCompressionType(const void* hdrBuf)
//------------------------------------------------------------------------------
void CompressInterface::setHdrSize(void* hdrBuf, uint64_t size)
{
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fHeaderSize = size;
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fHeaderSize = size;
}
//------------------------------------------------------------------------------
@ -404,17 +389,15 @@ void CompressInterface::setHdrSize(void* hdrBuf, uint64_t size)
//------------------------------------------------------------------------------
uint64_t CompressInterface::getHdrSize(const void* hdrBuf)
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fHeaderSize);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fHeaderSize);
}
//------------------------------------------------------------------------------
// Get column type
//-----------------------------------------------------------------------------
execplan::CalpontSystemCatalog::ColDataType
CompressInterface::getColDataType(const void* hdrBuf)
execplan::CalpontSystemCatalog::ColDataType CompressInterface::getColDataType(const void* hdrBuf)
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColDataType);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColDataType);
}
//------------------------------------------------------------------------------
@ -422,8 +405,7 @@ CompressInterface::getColDataType(const void* hdrBuf)
//------------------------------------------------------------------------------
uint64_t CompressInterface::getColumnWidth(const void* hdrBuf)
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColumnWidth);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColumnWidth);
}
//------------------------------------------------------------------------------
@ -431,9 +413,9 @@ uint64_t CompressInterface::getColumnWidth(const void* hdrBuf)
//------------------------------------------------------------------------------
uint64_t CompressInterface::getLBIDByIndex(const void* hdrBuf, uint64_t index)
{
if (index < LBID_MAX_SIZE)
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index]);
return 0;
if (index < LBID_MAX_SIZE)
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index]);
return 0;
}
//------------------------------------------------------------------------------
@ -441,12 +423,12 @@ uint64_t CompressInterface::getLBIDByIndex(const void* hdrBuf, uint64_t index)
//------------------------------------------------------------------------------
void CompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index)
{
if (lbid && index < LBID_MAX_SIZE)
{
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index] = lbid;
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount =
std::max(index + 1, reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount);
}
if (lbid && index < LBID_MAX_SIZE)
{
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDS[index] = lbid;
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount =
std::max(index + 1, reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBIDCount);
}
}
//------------------------------------------------------------------------------
@ -454,25 +436,24 @@ void CompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t ind
//------------------------------------------------------------------------------
uint64_t CompressInterface::getLBIDCount(void* hdrBuf)
{
return reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDCount;
return reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBIDCount;
}
//------------------------------------------------------------------------------
// Calculates the chunk and block offset within the chunk for the specified
// block number.
//------------------------------------------------------------------------------
void CompressInterface::locateBlock(unsigned int block,
unsigned int& chunkIndex,
void CompressInterface::locateBlock(unsigned int block, unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const
{
const uint64_t BUFLEN = UNCOMPRESSED_INBUF_LEN;
const uint64_t BUFLEN = UNCOMPRESSED_INBUF_LEN;
uint64_t byteOffset = (uint64_t)block * BLOCK_SIZE;
uint64_t chunk = byteOffset / BUFLEN;
uint64_t blockInChunk = (byteOffset % BUFLEN) / BLOCK_SIZE;
uint64_t byteOffset = (uint64_t)block * BLOCK_SIZE;
uint64_t chunk = byteOffset / BUFLEN;
uint64_t blockInChunk = (byteOffset % BUFLEN) / BLOCK_SIZE;
chunkIndex = chunk;
blockOffsetWithinChunk = blockInChunk;
chunkIndex = chunk;
blockOffsetWithinChunk = blockInChunk;
}
//------------------------------------------------------------------------------
@ -480,238 +461,215 @@ void CompressInterface::locateBlock(unsigned int block,
// also expand to allow for user requested padding. Lastly, initialize padding
// bytes to 0.
//------------------------------------------------------------------------------
int CompressInterface::padCompressedChunks(unsigned char* buf, size_t& len,
unsigned int maxLen) const
int CompressInterface::padCompressedChunks(unsigned char* buf, size_t& len, unsigned int maxLen) const
{
int nPaddingBytes = 0;
int nRem = len % COMPRESSED_CHUNK_INCREMENT_SIZE;
int nPaddingBytes = 0;
int nRem = len % COMPRESSED_CHUNK_INCREMENT_SIZE;
if (nRem != 0)
{
nPaddingBytes = COMPRESSED_CHUNK_INCREMENT_SIZE - nRem;
}
if (nRem != 0)
{
nPaddingBytes = COMPRESSED_CHUNK_INCREMENT_SIZE - nRem;
}
nPaddingBytes = nPaddingBytes + fNumUserPaddingBytes;
nPaddingBytes = nPaddingBytes + fNumUserPaddingBytes;
if (nPaddingBytes > 0)
{
if ((len + nPaddingBytes) > maxLen)
return -1;
if (nPaddingBytes > 0)
{
if ((len + nPaddingBytes) > maxLen)
return -1;
memset(buf + len, 0, nPaddingBytes);
len = len + nPaddingBytes;
}
memset(buf + len, 0, nPaddingBytes);
len = len + nPaddingBytes;
}
return 0;
return 0;
}
// Snappy
CompressInterfaceSnappy::CompressInterfaceSnappy(uint32_t numUserPaddingBytes)
: CompressInterface(numUserPaddingBytes)
: CompressInterface(numUserPaddingBytes)
{
}
int32_t CompressInterfaceSnappy::compress(const char* in, size_t inLen,
char* out, size_t* outLen) const
int32_t CompressInterfaceSnappy::compress(const char* in, size_t inLen, char* out, size_t* outLen) const
{
snappy::RawCompress(in, inLen, out, outLen);
snappy::RawCompress(in, inLen, out, outLen);
#ifdef DEBUG_COMPRESSION
std::cout << "Snappy::compress: inLen " << inLen << ", outLen " << *outLen
<< std::endl;
std::cout << "Snappy::compress: inLen " << inLen << ", outLen " << *outLen << std::endl;
#endif
return ERR_OK;
return ERR_OK;
}
int32_t CompressInterfaceSnappy::uncompress(const char* in, size_t inLen,
char* out, size_t* outLen) const
int32_t CompressInterfaceSnappy::uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const
{
size_t realOutLen = 0;
auto rc = snappy::GetUncompressedLength(in, inLen, &realOutLen);
size_t realOutLen = 0;
auto rc = snappy::GetUncompressedLength(in, inLen, &realOutLen);
if (!rc || realOutLen > *outLen)
{
cerr << "snappy::GetUncompressedLength failed. InLen: " << inLen
<< ", outLen: " << *outLen << ", realOutLen: " << realOutLen
<< endl;
return ERR_DECOMPRESS;
}
if (!rc || realOutLen > *outLen)
{
cerr << "snappy::GetUncompressedLength failed. InLen: " << inLen << ", outLen: " << *outLen
<< ", realOutLen: " << realOutLen << endl;
return ERR_DECOMPRESS;
}
rc = snappy::RawUncompress(in, inLen, out);
rc = snappy::RawUncompress(in, inLen, out);
if (!rc)
{
cerr << "snappy::RawUnompress failed. InLen: " << inLen
<< ", outLen: " << *outLen << endl;
return ERR_DECOMPRESS;
}
if (!rc)
{
cerr << "snappy::RawUnompress failed. InLen: " << inLen << ", outLen: " << *outLen << endl;
return ERR_DECOMPRESS;
}
#ifdef DEBUG_COMPRESSION
std::cout << "Snappy::uncompress: inLen " << inLen << ", outLen "
<< *outLen << std::endl;
std::cout << "Snappy::uncompress: inLen " << inLen << ", outLen " << *outLen << std::endl;
#endif
*outLen = realOutLen;
*outLen = realOutLen;
return ERR_OK;
return ERR_OK;
}
size_t CompressInterfaceSnappy::maxCompressedSize(size_t uncompSize) const
{
return (snappy::MaxCompressedLength(uncompSize) + HEADER_SIZE);
return (snappy::MaxCompressedLength(uncompSize) + HEADER_SIZE);
}
bool CompressInterfaceSnappy::getUncompressedSize(char* in, size_t inLen,
size_t* outLen) const
bool CompressInterfaceSnappy::getUncompressedSize(char* in, size_t inLen, size_t* outLen) const
{
return snappy::GetUncompressedLength(in, inLen, outLen);
return snappy::GetUncompressedLength(in, inLen, outLen);
}
uint8_t CompressInterfaceSnappy::getChunkMagicNumber() const
{
return CHUNK_MAGIC_SNAPPY;
return CHUNK_MAGIC_SNAPPY;
}
// LZ4
CompressInterfaceLZ4::CompressInterfaceLZ4(uint32_t numUserPaddingBytes)
: CompressInterface(numUserPaddingBytes)
: CompressInterface(numUserPaddingBytes)
{
}
int32_t CompressInterfaceLZ4::compress(const char* in, size_t inLen, char* out,
size_t* outLen) const
int32_t CompressInterfaceLZ4::compress(const char* in, size_t inLen, char* out, size_t* outLen) const
{
#ifdef HAVE_LZ4
auto compressedLen = LZ4_compress_default(in, out, inLen, *outLen);
auto compressedLen = LZ4_compress_default(in, out, inLen, *outLen);
if (!compressedLen)
{
cerr << "LZ_compress_default failed. InLen: " << inLen
<< ", compressedLen: " << compressedLen << endl;
return ERR_COMPRESS;
}
#ifdef DEBUG_COMPRESSION
std::cout << "LZ4::compress: inLen " << inLen << ", comressedLen "
<< compressedLen << std::endl;
#endif
*outLen = compressedLen;
return ERR_OK;
#else
if (!compressedLen)
{
cerr << "LZ_compress_default failed. InLen: " << inLen << ", compressedLen: " << compressedLen << endl;
return ERR_COMPRESS;
}
#ifdef DEBUG_COMPRESSION
std::cout << "LZ4::compress: inLen " << inLen << ", comressedLen " << compressedLen << std::endl;
#endif
*outLen = compressedLen;
return ERR_OK;
#else
return ERR_COMPRESS;
#endif
}
int32_t CompressInterfaceLZ4::uncompress(const char* in, size_t inLen,
char* out, size_t* outLen) const
int32_t CompressInterfaceLZ4::uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const
{
#ifdef HAVE_LZ4
auto decompressedLen = LZ4_decompress_safe(in, out, inLen, *outLen);
auto decompressedLen = LZ4_decompress_safe(in, out, inLen, *outLen);
if (decompressedLen < 0)
{
cerr << "LZ_decompress_safe failed with error code " << decompressedLen
<< endl;
cerr << "InLen: " << inLen << ", outLen: " << *outLen << endl;
return ERR_DECOMPRESS;
}
if (decompressedLen < 0)
{
cerr << "LZ_decompress_safe failed with error code " << decompressedLen << endl;
cerr << "InLen: " << inLen << ", outLen: " << *outLen << endl;
return ERR_DECOMPRESS;
}
*outLen = decompressedLen;
*outLen = decompressedLen;
#ifdef DEBUG_COMPRESSION
std::cout << "LZ4::uncompress: inLen " << inLen << ", outLen " << *outLen
<< std::endl;
std::cout << "LZ4::uncompress: inLen " << inLen << ", outLen " << *outLen << std::endl;
#endif
return ERR_OK;
return ERR_OK;
#else
return ERR_DECOMPRESS;
return ERR_DECOMPRESS;
#endif
}
size_t CompressInterfaceLZ4::maxCompressedSize(size_t uncompSize) const
{
return (LZ4_COMPRESSBOUND(uncompSize) + HEADER_SIZE);
return (LZ4_COMPRESSBOUND(uncompSize) + HEADER_SIZE);
}
bool CompressInterfaceLZ4::getUncompressedSize(char* in, size_t inLen,
size_t* outLen) const
bool CompressInterfaceLZ4::getUncompressedSize(char* in, size_t inLen, size_t* outLen) const
{
// LZ4 does not have such function.
idbassert(false);
return false;
// LZ4 does not have such function.
idbassert(false);
return false;
}
uint8_t CompressInterfaceLZ4::getChunkMagicNumber() const
{
return CHUNK_MAGIC_LZ4;
return CHUNK_MAGIC_LZ4;
}
CompressInterface* getCompressInterfaceByType(uint32_t compressionType,
uint32_t numUserPaddingBytes)
CompressInterface* getCompressInterfaceByType(uint32_t compressionType, uint32_t numUserPaddingBytes)
{
switch (compressionType)
{
switch (compressionType)
{
case 1:
case 2:
return new CompressInterfaceSnappy(numUserPaddingBytes);
case 3:
return new CompressInterfaceLZ4(numUserPaddingBytes);
}
case 2: return new CompressInterfaceSnappy(numUserPaddingBytes);
case 3: return new CompressInterfaceLZ4(numUserPaddingBytes);
}
return nullptr;
return nullptr;
}
CompressInterface* getCompressInterfaceByName(const std::string& compressionName,
uint32_t numUserPaddingBytes)
{
if (compressionName == "SNAPPY")
return new CompressInterfaceSnappy(numUserPaddingBytes);
else if (compressionName == "LZ4")
return new CompressInterfaceLZ4(numUserPaddingBytes);
return nullptr;
if (compressionName == "SNAPPY")
return new CompressInterfaceSnappy(numUserPaddingBytes);
else if (compressionName == "LZ4")
return new CompressInterfaceLZ4(numUserPaddingBytes);
return nullptr;
}
void initializeCompressorPool(
std::unordered_map<uint32_t, std::shared_ptr<CompressInterface>>&
compressorPool,
std::unordered_map<uint32_t, std::shared_ptr<CompressInterface>>& compressorPool,
uint32_t numUserPaddingBytes)
{
compressorPool = {
make_pair(2, std::shared_ptr<CompressInterface>(
new CompressInterfaceSnappy(numUserPaddingBytes))),
make_pair(3, std::shared_ptr<CompressInterface>(
new CompressInterfaceLZ4(numUserPaddingBytes)))};
compressorPool = {
make_pair(2, std::shared_ptr<CompressInterface>(new CompressInterfaceSnappy(numUserPaddingBytes))),
make_pair(3, std::shared_ptr<CompressInterface>(new CompressInterfaceLZ4(numUserPaddingBytes)))};
}
std::shared_ptr<CompressInterface> getCompressorByType(
std::unordered_map<uint32_t, std::shared_ptr<CompressInterface>>&
compressorPool,
std::unordered_map<uint32_t, std::shared_ptr<CompressInterface>>& compressorPool,
uint32_t compressionType)
{
switch (compressionType)
{
switch (compressionType)
{
case 1:
case 2:
if (!compressorPool.count(2))
{
return nullptr;
}
return compressorPool[2];
if (!compressorPool.count(2))
{
return nullptr;
}
return compressorPool[2];
case 3:
if (!compressorPool.count(3))
{
return nullptr;
}
return compressorPool[3];
}
if (!compressorPool.count(3))
{
return nullptr;
}
return compressorPool[3];
}
return nullptr;
return nullptr;
}
#endif
} // namespace compress
} // namespace compress
// vim:ts=4 sw=4:

683
utils/compress/idbcompress.h
View File

@ -37,356 +37,332 @@
namespace compress
{
typedef std::pair<uint64_t, uint64_t> CompChunkPtr;
typedef std::vector<CompChunkPtr> CompChunkPtrList;
class CompressInterface
{
public:
static const unsigned int HDR_BUF_LEN = 4096;
static const unsigned int UNCOMPRESSED_INBUF_LEN = 512 * 1024 * 8;
static const uint32_t COMPRESSED_CHUNK_INCREMENT_SIZE = 8192;
public:
static const unsigned int HDR_BUF_LEN = 4096;
static const unsigned int UNCOMPRESSED_INBUF_LEN = 512 * 1024 * 8;
static const uint32_t COMPRESSED_CHUNK_INCREMENT_SIZE = 8192;
// error codes from uncompressBlock()
static const int ERR_OK = 0;
static const int ERR_CHECKSUM = -1;
static const int ERR_DECOMPRESS = -2;
static const int ERR_BADINPUT = -3;
static const int ERR_BADOUTSIZE = -4;
static const int ERR_COMPRESS = -5;
// error codes from uncompressBlock()
static const int ERR_OK = 0;
static const int ERR_CHECKSUM = -1;
static const int ERR_DECOMPRESS = -2;
static const int ERR_BADINPUT = -3;
static const int ERR_BADOUTSIZE = -4;
static const int ERR_COMPRESS = -5;
/**
* When CompressInterface object is being used to compress a chunk, this
* construct can be used to specify the padding added by padCompressedChunks
*/
EXPORT explicit CompressInterface(unsigned int numUserPaddingBytes = 0);
/**
* When CompressInterface object is being used to compress a chunk, this
* construct can be used to specify the padding added by padCompressedChunks
*/
EXPORT explicit CompressInterface(unsigned int numUserPaddingBytes = 0);
/**
* dtor
*/
EXPORT virtual ~CompressInterface() = default;
/**
* dtor
*/
EXPORT virtual ~CompressInterface() = default;
/**
* see if the algo is available in this lib
*/
EXPORT static bool isCompressionAvail(int compressionType = 0);
/**
* see if the algo is available in this lib
*/
EXPORT static bool isCompressionAvail(int compressionType = 0);
/**
* Returns the maximum compressed size from all available compression
* types.
*/
EXPORT static size_t getMaxCompressedSizeGeneric(size_t inLen);
/**
* Returns the maximum compressed size from all available compression
* types.
*/
EXPORT static size_t getMaxCompressedSizeGeneric(size_t inLen);
/**
* Compresses specified "in" buffer of length "inLen" bytes.
* Compressed data and size are returned in "out" and "outLen".
* "out" should be sized using maxCompressedSize() to allow for incompressible data.
* Returns 0 if success.
*/
/**
* Compresses specified "in" buffer of length "inLen" bytes.
* Compressed data and size are returned in "out" and "outLen".
* "out" should be sized using maxCompressedSize() to allow for incompressible data.
* Returns 0 if success.
*/
EXPORT int compressBlock(const char* in, const size_t inLen,
unsigned char* out, size_t& outLen) const;
EXPORT int compressBlock(const char* in, const size_t inLen, unsigned char* out, size_t& outLen) const;
/**
* outLen must be initialized with the size of the out buffer before calling uncompressBlock.
* On return, outLen will have the number of bytes used in out.
*/
EXPORT int uncompressBlock(const char* in, const size_t inLen,
unsigned char* out, size_t& outLen) const;
/**
* outLen must be initialized with the size of the out buffer before calling uncompressBlock.
* On return, outLen will have the number of bytes used in out.
*/
EXPORT int uncompressBlock(const char* in, const size_t inLen, unsigned char* out, size_t& outLen) const;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk.
*/
EXPORT virtual int compress(const char* in, size_t inLen, char* out,
size_t* outLen) const = 0;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk.
*/
EXPORT virtual int compress(const char* in, size_t inLen, char* out, size_t* outLen) const = 0;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk. The caller needs to make sure out is big
* enough to contain the output by using getUncompressedSize().
*/
EXPORT virtual int uncompress(const char* in, size_t inLen, char* out,
size_t* outLen) const = 0;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk. The caller needs to make sure out is big
* enough to contain the output by using getUncompressedSize().
*/
EXPORT virtual int uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const = 0;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN bytes
* @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes
*/
EXPORT static void
initHdr(void* hdrBuf, void* ptrBuf, uint32_t columnWidht,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType, int hdrSize);
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes
*/
EXPORT static void
initHdr(void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType);
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN bytes
* @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes
*/
EXPORT static void initHdr(void* hdrBuf, void* ptrBuf, uint32_t columnWidht,
execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType,
int hdrSize);
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes
*/
EXPORT static void initHdr(void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType columnType, int compressionType);
/**
* Verify the passed in buffer contains a compressed db file header.
*/
EXPORT static int verifyHdr(const void* hdrBuf);
/**
* Verify the passed in buffer contains a compressed db file header.
*/
EXPORT static int verifyHdr(const void* hdrBuf);
/**
* Extracts list of compression pointers from the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
* chunkPtrs is a vector of offset, size pairs for the compressed chunks.
* Returns 0 if success.
*/
EXPORT static int getPtrList(const char* ptrBuf, const int ptrBufSize,
CompChunkPtrList& chunkPtrs);
/**
* Extracts list of compression pointers from the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
* chunkPtrs is a vector of offset, size pairs for the compressed chunks.
* Returns 0 if success.
*/
EXPORT static int getPtrList(const char* ptrBuf, const int ptrBufSize, CompChunkPtrList& chunkPtrs);
/**
* Extracts list of compression pointers from the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT static int getPtrList(const char* hdrBuf,
CompChunkPtrList& chunkPtrs);
/**
* Extracts list of compression pointers from the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT static int getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs);
/**
* Return the number of chunk pointers contained in the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
*/
EXPORT static unsigned int getPtrCount(const char* ptrBuf,
const int ptrBufSize);
/**
* Return the number of chunk pointers contained in the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
*/
EXPORT static unsigned int getPtrCount(const char* ptrBuf, const int ptrBufSize);
/**
* Return the number of chunk pointers contained in the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* For non-dictionary columns.
*/
EXPORT static unsigned int getPtrCount(const char* hdrBuf);
/**
* Return the number of chunk pointers contained in the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* For non-dictionary columns.
*/
EXPORT static unsigned int getPtrCount(const char* hdrBuf);
/**
* Store vector of pointers into the specified buffer header's pointer section.
*/
EXPORT static void storePtrs(const std::vector<uint64_t>& ptrs,
void* hdrBuf, int ptrSectionSize);
/**
* Store vector of pointers into the specified buffer header's pointer section.
*/
EXPORT static void storePtrs(const std::vector<uint64_t>& ptrs, void* hdrBuf, int ptrSectionSize);
/**
* Store vector of pointers into the specified buffer header.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT static void storePtrs(const std::vector<uint64_t>& ptrs,
void* hdrBuf);
/**
* Store vector of pointers into the specified buffer header.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT static void storePtrs(const std::vector<uint64_t>& ptrs, void* hdrBuf);
/**
* Calculates the chunk, and the block offset within the chunk, for the
* specified block number.
*/
EXPORT void locateBlock(unsigned int block, unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const;
/**
* Calculates the chunk, and the block offset within the chunk, for the
* specified block number.
*/
EXPORT void locateBlock(unsigned int block, unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const;
/**
* Pads the specified compressed chunk to the nearest compressed chunk
* increment, by padding buf with 0's, and updating len accordingly.
* maxLen is the maximum size for buf. nonzero return code means the
* result output buffer length is > than maxLen.
*/
EXPORT int padCompressedChunks(unsigned char* buf, size_t& len,
unsigned int maxLen) const;
/**
* Pads the specified compressed chunk to the nearest compressed chunk
* increment, by padding buf with 0's, and updating len accordingly.
* maxLen is the maximum size for buf. nonzero return code means the
* result output buffer length is > than maxLen.
*/
EXPORT int padCompressedChunks(unsigned char* buf, size_t& len, unsigned int maxLen) const;
/*
* Mutator methods for the block count in the file
*/
/**
* getVersionNumber
*/
EXPORT static uint64_t getVersionNumber(const void* hdrBuf);
/*
* Mutator methods for the block count in the file
*/
/**
* getVersionNumber
*/
EXPORT static uint64_t getVersionNumber(const void* hdrBuf);
/**
* setBlockCount
*/
EXPORT static void setBlockCount(void* hdrBuf, uint64_t count);
/**
* setBlockCount
*/
EXPORT static void setBlockCount(void* hdrBuf, uint64_t count);
/**
* getBlockCount
*/
EXPORT static uint64_t getBlockCount(const void* hdrBuf);
/**
* getBlockCount
*/
EXPORT static uint64_t getBlockCount(const void* hdrBuf);
/**
* getCompressionType
*/
EXPORT static uint64_t getCompressionType(const void* hdrBuf);
/**
* getCompressionType
*/
EXPORT static uint64_t getCompressionType(const void* hdrBuf);
/*
* Mutator methods for the overall header size
*/
/**
* setHdrSize
*/
EXPORT static void setHdrSize(void* hdrBuf, uint64_t size);
/*
* Mutator methods for the overall header size
*/
/**
* setHdrSize
*/
EXPORT static void setHdrSize(void* hdrBuf, uint64_t size);
/**
* getHdrSize
*/
EXPORT static uint64_t getHdrSize(const void* hdrBuf);
/**
* getHdrSize
*/
EXPORT static uint64_t getHdrSize(const void* hdrBuf);
/**
* getColumnType
*/
EXPORT static execplan::CalpontSystemCatalog::ColDataType
getColDataType(const void* hdrBuf);
/**
* getColumnType
*/
EXPORT static execplan::CalpontSystemCatalog::ColDataType getColDataType(const void* hdrBuf);
/**
* getColumnWidth
*/
EXPORT static uint64_t getColumnWidth(const void* hdrBuf);
/**
* getColumnWidth
*/
EXPORT static uint64_t getColumnWidth(const void* hdrBuf);
/**
* getLBIDByIndex
*/
EXPORT static uint64_t getLBIDByIndex(const void* hdrBuf, uint64_t index);
/**
* getLBIDByIndex
*/
EXPORT static uint64_t getLBIDByIndex(const void* hdrBuf, uint64_t index);
/**
* setLBIDByIndex
*/
EXPORT static void setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index);
/**
* setLBIDByIndex
*/
EXPORT static void setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint64_t index);
/**
* getLBIDCount
*/
EXPORT static uint64_t getLBIDCount(void* hdrBuf);
/**
* getLBIDCount
*/
EXPORT static uint64_t getLBIDCount(void* hdrBuf);
/**
* Mutator methods for the user padding bytes
*/
/**
* set numUserPaddingBytes
*/
EXPORT void numUserPaddingBytes(uint64_t num)
{
fNumUserPaddingBytes = num;
}
/**
* Mutator methods for the user padding bytes
*/
/**
* set numUserPaddingBytes
*/
EXPORT void numUserPaddingBytes(uint64_t num)
{
fNumUserPaddingBytes = num;
}
/**
* get numUserPaddingBytes
*/
EXPORT uint64_t numUserPaddingBytes() const
{
return fNumUserPaddingBytes;
}
/**
* get numUserPaddingBytes
*/
EXPORT uint64_t numUserPaddingBytes() const
{
return fNumUserPaddingBytes;
}
/**
* Given an input, uncompressed block, what's the maximum possible output,
* compressed size?
*/
EXPORT virtual size_t maxCompressedSize(size_t uncompSize) const = 0;
/**
* Given an input, uncompressed block, what's the maximum possible output,
* compressed size?
*/
EXPORT virtual size_t maxCompressedSize(size_t uncompSize) const = 0;
/**
* Given a compressed block, returns the uncompressed size in outLen.
* Returns false on error, true on success.
*/
EXPORT virtual bool getUncompressedSize(char* in, size_t inLen,
size_t* outLen) const = 0;
/**
* Given a compressed block, returns the uncompressed size in outLen.
* Returns false on error, true on success.
*/
EXPORT virtual bool getUncompressedSize(char* in, size_t inLen, size_t* outLen) const = 0;
protected:
virtual uint8_t getChunkMagicNumber() const = 0;
protected:
virtual uint8_t getChunkMagicNumber() const = 0;
private:
//defaults okay
//CompressInterface(const CompressInterface& rhs);
//CompressInterface& operator=(const CompressInterface& rhs);
private:
// defaults okay
// CompressInterface(const CompressInterface& rhs);
// CompressInterface& operator=(const CompressInterface& rhs);
unsigned int fNumUserPaddingBytes; // Num bytes to pad compressed chunks
unsigned int fNumUserPaddingBytes; // Num bytes to pad compressed chunks
};
class CompressInterfaceSnappy : public CompressInterface
{
public:
EXPORT CompressInterfaceSnappy(uint32_t numUserPaddingBytes = 0);
EXPORT ~CompressInterfaceSnappy() = default;
/**
* Compress the given block using snappy compression API.
*/
EXPORT int32_t compress(const char* in, size_t inLen, char* out,
size_t* outLen) const override;
/**
* Uncompress the given block using snappy compression API.
*/
EXPORT int32_t uncompress(const char* in, size_t inLen, char* out,
size_t* outLen) const override;
/**
* Get max compressed size for the given `uncompSize` value using snappy
* compression API.
*/
EXPORT size_t maxCompressedSize(size_t uncompSize) const override;
public:
EXPORT CompressInterfaceSnappy(uint32_t numUserPaddingBytes = 0);
EXPORT ~CompressInterfaceSnappy() = default;
/**
* Compress the given block using snappy compression API.
*/
EXPORT int32_t compress(const char* in, size_t inLen, char* out, size_t* outLen) const override;
/**
* Uncompress the given block using snappy compression API.
*/
EXPORT int32_t uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const override;
/**
* Get max compressed size for the given `uncompSize` value using snappy
* compression API.
*/
EXPORT size_t maxCompressedSize(size_t uncompSize) const override;
/**
* Get uncompressed size for the given block using snappy
* compression API.
*/
EXPORT
bool getUncompressedSize(char* in, size_t inLen,
size_t* outLen) const override;
/**
* Get uncompressed size for the given block using snappy
* compression API.
*/
EXPORT
bool getUncompressedSize(char* in, size_t inLen, size_t* outLen) const override;
protected:
uint8_t getChunkMagicNumber() const override;
protected:
uint8_t getChunkMagicNumber() const override;
private:
const uint8_t CHUNK_MAGIC_SNAPPY = 0xfd;
private:
const uint8_t CHUNK_MAGIC_SNAPPY = 0xfd;
};
class CompressInterfaceLZ4 : public CompressInterface
{
public:
EXPORT CompressInterfaceLZ4(uint32_t numUserPaddingBytes = 0);
EXPORT ~CompressInterfaceLZ4() = default;
/**
* Compress the given block using LZ4 compression API.
*/
EXPORT int32_t compress(const char* in, size_t inLen, char* out,
size_t* outLen) const override;
/**
* Uncompress the given block using LZ4 compression API.
*/
EXPORT int32_t uncompress(const char* in, size_t inLen, char* out,
size_t* outLen) const override;
/**
* Get max compressed size for the given `uncompSize` value using LZ4
* compression API.
*/
EXPORT size_t maxCompressedSize(size_t uncompSize) const override;
public:
EXPORT CompressInterfaceLZ4(uint32_t numUserPaddingBytes = 0);
EXPORT ~CompressInterfaceLZ4() = default;
/**
* Compress the given block using LZ4 compression API.
*/
EXPORT int32_t compress(const char* in, size_t inLen, char* out, size_t* outLen) const override;
/**
* Uncompress the given block using LZ4 compression API.
*/
EXPORT int32_t uncompress(const char* in, size_t inLen, char* out, size_t* outLen) const override;
/**
* Get max compressed size for the given `uncompSize` value using LZ4
* compression API.
*/
EXPORT size_t maxCompressedSize(size_t uncompSize) const override;
/**
* Get uncompressed size for the given block using LZ4
* compression API.
*/
EXPORT
bool getUncompressedSize(char* in, size_t inLen,
size_t* outLen) const override;
/**
* Get uncompressed size for the given block using LZ4
* compression API.
*/
EXPORT
bool getUncompressedSize(char* in, size_t inLen, size_t* outLen) const override;
protected:
uint8_t getChunkMagicNumber() const override;
protected:
uint8_t getChunkMagicNumber() const override;
private:
const uint8_t CHUNK_MAGIC_LZ4 = 0xfc;
private:
const uint8_t CHUNK_MAGIC_LZ4 = 0xfc;
};
using CompressorPool =
std::unordered_map<uint32_t, std::shared_ptr<CompressInterface>>;
using CompressorPool = std::unordered_map<uint32_t, std::shared_ptr<CompressInterface>>;
/**
* Returns a pointer to the appropriate compression interface based on
* `compressionType`. `compressionType` must be greater than 0.
* Note: caller is responsible for memory deallocation.
*/
EXPORT CompressInterface*
getCompressInterfaceByType(uint32_t compressionType,
uint32_t numUserPaddingBytes = 0);
EXPORT CompressInterface* getCompressInterfaceByType(uint32_t compressionType,
uint32_t numUserPaddingBytes = 0);
/**
* Returns a pointer to the appropriate compression interface based on
@ -400,121 +376,152 @@ EXPORT CompressInterface* getCompressInterfaceByName(const std::string& compress
* Initializes a given `unordered_map` with all available compression
* interfaces.
*/
EXPORT void initializeCompressorPool(CompressorPool& compressorPool,
uint32_t numUserPaddingBytes = 0);
EXPORT void initializeCompressorPool(CompressorPool& compressorPool, uint32_t numUserPaddingBytes = 0);
/**
* Returns a `shared_ptr` to the appropriate compression interface.
*/
EXPORT std::shared_ptr<CompressInterface>
getCompressorByType(CompressorPool& compressorPool, uint32_t compressionType);
EXPORT std::shared_ptr<CompressInterface> getCompressorByType(CompressorPool& compressorPool,
uint32_t compressionType);
#ifdef SKIP_IDB_COMPRESSION
inline CompressInterface::CompressInterface(unsigned int /*numUserPaddingBytes*/) {}
inline CompressInterface::CompressInterface(unsigned int /*numUserPaddingBytes*/)
{
}
inline bool CompressInterface::isCompressionAvail(int c)
{
return (c == 0);
return (c == 0);
}
inline int CompressInterface::compressBlock(const char*, const size_t, unsigned char*, size_t&) const
{
return -1;
return -1;
}
inline int CompressInterface::uncompressBlock(const char* in,
const size_t inLen,
unsigned char* out,
inline int CompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out,
size_t& outLen) const
{
return -1;
return -1;
}
inline void initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int)
{
}
inline void initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) {}
inline int CompressInterface::verifyHdr(const void*)
{
return -1;
return -1;
}
inline void CompressInterface::initHdr(void*, void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType,
int, int)
{
}
inline void CompressInterface::initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType,
int) const
{
}
inline void CompressInterface::initHdr(void*, void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int, int) {}
inline void CompressInterface::initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) const {}
inline int CompressInterface::getPtrList(const char*, const int, CompChunkPtrList&)
{
return -1;
return -1;
}
inline unsigned int CompressInterface::getPtrCount(const char*, const int)
{
return 0;
return 0;
}
inline unsigned int CompressInterface::getPtrCount(const char*)
{
return 0;
return 0;
}
inline void CompressInterface::storePtrs(const std::vector<uint64_t>&, void*, int) {}
inline void CompressInterface::storePtrs(const std::vector<uint64_t>&, void*) {}
inline void
CompressInterface::locateBlock(unsigned int block, unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const
inline void CompressInterface::storePtrs(const std::vector<uint64_t>&, void*, int)
{
}
inline int CompressInterface::padCompressedChunks(unsigned char* buf, unsigned int& len, unsigned int maxLen) const
inline void CompressInterface::storePtrs(const std::vector<uint64_t>&, void*)
{
return -1;
}
inline void CompressInterface::locateBlock(unsigned int block, unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const
{
}
inline int CompressInterface::padCompressedChunks(unsigned char* buf, unsigned int& len,
unsigned int maxLen) const
{
return -1;
}
inline uint64_t CompressInterface::getVersionNumber(const void* hdrBuf)
{
return 0;
return 0;
}
inline void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count)
{
}
inline void CompressInterface::setBlockCount(void* hdrBuf, uint64_t count) {}
inline uint64_t CompressInterface::getBlockCount(const void* hdrBuf)
{
return 0;
return 0;
}
inline uint64_t CompressInterface::getCompressionType(const void* hdrBuf)
{
return 0;
return 0;
}
inline execplan::CalpontSystemCatalog::ColDataType
CompressInterface::getColDataType(const void* hdrBuf)
inline execplan::CalpontSystemCatalog::ColDataType CompressInterface::getColDataType(const void* hdrBuf)
{
return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
}
inline uint64_t CompressInterface::getColumnWidth(const void* hdrBuf) const
{
return 0;
return 0;
}
inline uint64_t getLBID0(const void* hdrBuf)
{
return 0;
}
void setLBID0(void* hdrBuf, uint64_t lbid)
{
}
inline uint64_t getLBID1(const void* hdrBuf)
{
return 0;
}
void setLBID1(void* hdrBuf, uint64_t lbid)
{
}
inline void CompressInterface::setHdrSize(void*, uint64_t)
{
}
inline uint64_t getLBID0(const void* hdrBuf) { return 0; }
void setLBID0(void* hdrBuf, uint64_t lbid) {}
inline uint64_t getLBID1(const void* hdrBuf) { return 0; }
void setLBID1(void* hdrBuf, uint64_t lbid) {}
inline void CompressInterface::setHdrSize(void*, uint64_t) {}
inline uint64_t CompressInterface::getHdrSize(const void*)
{
return 0;
return 0;
}
CompressInterfaceSnappy::CompressInterfaceSnappy(uint32_t numUserPaddingBytes)
: CompressInterface(numUserPaddingBytes)
: CompressInterface(numUserPaddingBytes)
{
}
inline uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const { return 0; }
inline uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const
{
return 0;
}
inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize)
{
return uncompSize;
return uncompSize;
}
inline uint64_t IDBCompressInterface::getLBIDByIndex(const void* hdrBuf, uint32_t index) const
{
return 0;
return 0;
}
void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint32_t index) const
{
}
void IDBCompressInterface::getLBIDCount(void* hdrBuf) const
{
}
void IDBCompressInterface::setLBIDByIndex(void* hdrBuf, uint64_t lbid, uint32_t index) const {}
void IDBCompressInterface::getLBIDCount(void* hdrBuf) const {}
inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen)
{
return false;
return false;
}
uint8_t getChunkMagicNumber() const { return 0; }
CompressInterface* getCompressInterfaceByType(uint32_t compressionType,
uint32_t numUserPaddingBytes)
uint8_t getChunkMagicNumber() const
{
return nullptr;
return 0;
}
CompressInterface* getCompressInterfaceByType(uint32_t compressionType, uint32_t numUserPaddingBytes)
{
return nullptr;
}
#endif
}
} // namespace compress
#undef EXPORT

843
utils/configcpp/configcpp.cpp
View File

File diff suppressed because it is too large Load Diff

336
utils/configcpp/configcpp.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: configcpp.h 3495 2013-01-21 14:09:51Z rdempsey $
*
******************************************************************************************/
* $Id: configcpp.h 3495 2013-01-21 14:09:51Z rdempsey $
*
******************************************************************************************/
/**
* @file
*/
@ -38,7 +38,6 @@
namespace messageqcpp
{
class ByteStream;
}
@ -51,7 +50,6 @@ class ByteStream;
namespace config
{
/** @brief a config file I/F class
*
* This is a singleton pattern: you must use the makeConfig() factory method to get a
@ -61,201 +59,199 @@ namespace config
*/
class Config
{
public:
/** @brief Config factory method
*
* Creates a singleton Config object
*/
EXPORT static Config* makeConfig(const char* cf = 0);
public:
/** @brief Config factory method
*
* Creates a singleton Config object
*/
EXPORT static Config* makeConfig(const char* cf = 0);
/** @brief Config factory method
*
* Creates a singleton Config object
*/
EXPORT static Config* makeConfig(const std::string& cf);
/** @brief Config factory method
*
* Creates a singleton Config object
*/
EXPORT static Config* makeConfig(const std::string& cf);
/** @brief dtor
*/
EXPORT virtual ~Config();
/** @brief dtor
*/
EXPORT virtual ~Config();
/** @brief get name's value from section
*
* get name's value from section in the current config file.
* @param section the name of the config file section to search
* @param name the param name whose value is to be returned
*/
EXPORT const std::string getConfig(const std::string& section, const std::string& name);
/** @brief get name's value from section
*
* get name's value from section in the current config file.
* @param section the name of the config file section to search
* @param name the param name whose value is to be returned
*/
EXPORT const std::string getConfig(const std::string& section, const std::string& name);
/** @brief get name's value from section
*
* get name's value from section in the current config file re-reading the
* config file if it was updated.
* @param section the name of the config file section to search
* @param name the param name whose value is to be returned
*/
const std::string getFromActualConfig(const std::string& section, const std::string& name);
/** @brief get name's value from section
*
* get name's value from section in the current config file re-reading the
* config file if it was updated.
* @param section the name of the config file section to search
* @param name the param name whose value is to be returned
*/
const std::string getFromActualConfig(const std::string& section, const std::string& name);
/** @brief get all name's values from a section
*
* get name's values from section in the current config file.
* @param section the name of the config file section to search
* @param name the param name whose value is to be returned
* @param values the values in the section are returned in this vector
*/
EXPORT void getConfig(const std::string& section, const std::string& name,
std::vector<std::string>& values);
/** @brief get all name's values from a section
*
* get name's values from section in the current config file.
* @param section the name of the config file section to search
* @param name the param name whose value is to be returned
* @param values the values in the section are returned in this vector
*/
EXPORT void getConfig(const std::string& section, const std::string& name,
std::vector<std::string>& values);
/** @brief set name's value in section
*
* set name's value in section in the current config file.
* @param section the name of the config file section to update
* @param name the param name whose value is to be updated
* @param value the param value
*/
// !!!Don't ever ever use this in the engine code b/c it might result in a race
// b/w getConfig and setConfig methods.!!!
EXPORT void setConfig(const std::string& section, const std::string& name, const std::string& value);
/** @brief set name's value in section
*
* set name's value in section in the current config file.
* @param section the name of the config file section to update
* @param name the param name whose value is to be updated
* @param value the param value
*/
// !!!Don't ever ever use this in the engine code b/c it might result in a race
// b/w getConfig and setConfig methods.!!!
EXPORT void setConfig(const std::string& section, const std::string& name, const std::string& value);
/** @brief delete name from section
*
* delete name from section in the current config file.
* @param section the name of the config file section to search
* @param name the param name whose entry is to be deleted
* @note if you delete the last param from a section, the section will still remain
*/
EXPORT void delConfig(const std::string& section, const std::string& name);
/** @brief delete name from section
*
* delete name from section in the current config file.
* @param section the name of the config file section to search
* @param name the param name whose entry is to be deleted
* @note if you delete the last param from a section, the section will still remain
*/
EXPORT void delConfig(const std::string& section, const std::string& name);
/** @brief write the config file back out to disk
*
* write the config file back out to disk using the current filename.
*/
EXPORT void write(void) const;
/** @brief write the config file back out to disk
*
* write the config file back out to disk using the current filename.
*/
EXPORT void write(void) const;
/** @brief write the config file back out to disk as fileName
*
* write the config file out to disk as a new file fileName. Does not affect the current
* config filename.
*/
EXPORT void write(const std::string& fileName) const;
/** @brief write the config file back out to disk as fileName
*
* write the config file out to disk as a new file fileName. Does not affect the current
* config filename.
*/
EXPORT void write(const std::string& fileName) const;
/** @brief write a stream copy of config file to disk
*
* write a stream copy of config file to disk. used to distributed mass updates to system nodes
*
*/
EXPORT void writeConfigFile(messageqcpp::ByteStream msg) const;
/** @brief write a stream copy of config file to disk
*
* write a stream copy of config file to disk. used to distributed mass updates to system nodes
*
*/
EXPORT void writeConfigFile(messageqcpp::ByteStream msg) const;
/** @brief return the name of this config file
*
* return the name of this config file.
*/
EXPORT inline const std::string& configFile() const
{
return fConfigFile;
}
/** @brief return the name of this config file
*
* return the name of this config file.
*/
EXPORT inline const std::string& configFile() const
{
return fConfigFile;
}
/** @brief delete all config file instances
*
* deletes \b all config file maps
*/
EXPORT static void deleteInstanceMap();
/** @brief delete all config file instances
*
* deletes \b all config file maps
*/
EXPORT static void deleteInstanceMap();
/** @brief parse config file numerics
*
* Convert human-friendly number formats to machine-friendly. Handle suffixes 'K', 'M', 'G'.
* Handle decimal, hex and octal notation in the same way as the C compiler.
* Ignore any 'B' following [KMG]. Ignore case in suffixes.
* An empty string or an unparseable string returns 0.
* Return a signed numeric value.
*/
EXPORT static int64_t fromText(const std::string& text);
/** @brief parse config file numerics
*
* Convert human-friendly number formats to machine-friendly. Handle suffixes 'K', 'M', 'G'.
* Handle decimal, hex and octal notation in the same way as the C compiler.
* Ignore any 'B' following [KMG]. Ignore case in suffixes.
* An empty string or an unparseable string returns 0.
* Return a signed numeric value.
*/
EXPORT static int64_t fromText(const std::string& text);
/** @brief parse config file numerics
*
* Return an unsigned numeric value.
*/
EXPORT static inline uint64_t uFromText(const std::string& text)
{
return static_cast<uint64_t>(fromText(text));
}
/** @brief parse config file numerics
*
* Return an unsigned numeric value.
*/
EXPORT static inline uint64_t uFromText(const std::string& text)
{
return static_cast<uint64_t>(fromText(text));
}
/** @brief Used externally to check whether there has been a config change without loading everything
*
*/
inline time_t getLastMTime() const
{
return fMtime;
}
/** @brief Used externally to check whether there has been a config change without loading everything
*
*/
inline time_t getLastMTime() const
{
return fMtime;
}
/** @brief Used externally to check whether there has been a config change without loading everything
*
*/
EXPORT time_t getCurrentMTime();
/** @brief Used externally to check whether there has been a config change without loading everything
*
*/
EXPORT time_t getCurrentMTime();
/** @brief Enumerate all the sections in the config file
*
*/
EXPORT const std::vector<std::string> enumConfig();
/** @brief Enumerate all the sections in the config file
*
*/
EXPORT const std::vector<std::string> enumConfig();
/** @brief Enumerate all the names in a section in the config file
*
*/
EXPORT const std::vector<std::string> enumSection(const std::string& section);
/** @brief Enumerate all the names in a section in the config file
*
*/
EXPORT const std::vector<std::string> enumSection(const std::string& section);
enum class TempDirPurpose
{
Joins, ///< disk joins
Aggregates ///< disk-based aggregation
};
/** @brief Return temporaru directory path for the specified purpose */
EXPORT std::string getTempFileDir(TempDirPurpose what);
enum class TempDirPurpose
{
Joins, ///< disk joins
Aggregates ///< disk-based aggregation
};
/** @brief Return temporaru directory path for the specified purpose */
EXPORT std::string getTempFileDir(TempDirPurpose what);
protected:
/** @brief parse the XML file
*
*/
void parseDoc(void);
protected:
/** @brief parse the XML file
*
*/
void parseDoc(void);
/** @brief write the XML tree to disk
*
*/
EXPORT void writeConfig(const std::string& fileName) const;
/** @brief write the XML tree to disk
*
*/
EXPORT void writeConfig(const std::string& fileName) const;
/** @brief stop processing this XML file
*
*/
void closeConfig(void);
/** @brief stop processing this XML file
*
*/
void closeConfig(void);
private:
typedef std::map<std::string, Config*> configMap_t;
private:
typedef std::map<std::string, Config*> configMap_t;
/*
*/
Config(const Config& rhs);
/*
*/
Config& operator=(const Config& rhs);
/*
*/
Config(const Config& rhs);
/*
*/
Config& operator=(const Config& rhs);
/** @brief ctor with config file specified
*/
Config(const std::string& configFile);
/** @brief ctor with config file specified
*/
Config(const std::string& configFile);
static configMap_t fInstanceMap;
static boost::mutex fInstanceMapMutex;
static boost::mutex fXmlLock;
static boost::mutex fWriteXmlLock;
xmlDocPtr fDoc;
const std::string fConfigFile;
time_t fMtime;
mutable boost::recursive_mutex fLock;
XMLParser fParser;
static configMap_t fInstanceMap;
static boost::mutex fInstanceMapMutex;
static boost::mutex fXmlLock;
static boost::mutex fWriteXmlLock;
xmlDocPtr fDoc;
const std::string fConfigFile;
time_t fMtime;
mutable boost::recursive_mutex fLock;
XMLParser fParser;
};
}
} // namespace config
#undef EXPORT
// vim:ts=4 sw=4:

35
utils/configcpp/configstream.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
#include "mcsconfig.h"
#include <string>
@ -34,39 +34,34 @@ using namespace messageqcpp;
namespace config
{
ConfigStream::ConfigStream(const ByteStream& bs) :
fParser()
ConfigStream::ConfigStream(const ByteStream& bs) : fParser()
{
init(reinterpret_cast<const xmlChar*>(bs.buf()));
init(reinterpret_cast<const xmlChar*>(bs.buf()));
}
ConfigStream::ConfigStream(const string& str) :
fParser()
ConfigStream::ConfigStream(const string& str) : fParser()
{
init(reinterpret_cast<const xmlChar*>(str.c_str()));
init(reinterpret_cast<const xmlChar*>(str.c_str()));
}
ConfigStream::ConfigStream(const char* cptr) :
fParser()
ConfigStream::ConfigStream(const char* cptr) : fParser()
{
init(reinterpret_cast<const xmlChar*>(cptr));
init(reinterpret_cast<const xmlChar*>(cptr));
}
ConfigStream::~ConfigStream()
{
if (fDoc != NULL)
xmlFreeDoc(fDoc);
if (fDoc != NULL)
xmlFreeDoc(fDoc);
}
void ConfigStream::init(const xmlChar* xp)
{
fDoc = xmlParseDoc(xp);
fDoc = xmlParseDoc(xp);
if (fDoc == NULL)
throw runtime_error("ConfigStream::ConfigStream: bad XML stream");
if (fDoc == NULL)
throw runtime_error("ConfigStream::ConfigStream: bad XML stream");
}
} //namespace
} // namespace config
// vim:ts=4 sw=4:

40
utils/configcpp/configstream.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/**
* @file
*/
@ -33,34 +33,32 @@
namespace config
{
/** @brief a config ByteStream I/F class
*
*/
class ConfigStream
{
public:
ConfigStream(const messageqcpp::ByteStream& bs);
ConfigStream(const std::string& str);
ConfigStream(const char* cptr);
~ConfigStream();
public:
ConfigStream(const messageqcpp::ByteStream& bs);
ConfigStream(const std::string& str);
ConfigStream(const char* cptr);
~ConfigStream();
const std::string getConfig(const std::string& section, const std::string& name) const
{
return fParser.getConfig(fDoc, section, name);
}
const std::string getConfig(const std::string& section, const std::string& name) const
{
return fParser.getConfig(fDoc, section, name);
}
private:
ConfigStream(const ConfigStream& rhs);
ConfigStream& operator=(const ConfigStream& rhs);
private:
ConfigStream(const ConfigStream& rhs);
ConfigStream& operator=(const ConfigStream& rhs);
void init(const xmlChar* xp);
void init(const xmlChar* xp);
XMLParser fParser;
xmlDocPtr fDoc;
XMLParser fParser;
xmlDocPtr fDoc;
};
} //namespace
} // namespace config
// vim:ts=4 sw=4:

358
utils/configcpp/md5/md32_common.h
View File

@ -142,135 +142,146 @@
*/
#undef ROTATE
#ifndef PEDANTIC
# if defined(_MSC_VER) || defined(__ICC)
# define ROTATE(a,n) _lrotl(a,n)
# elif defined(__MWERKS__)
# if defined(__POWERPC__)
# define ROTATE(a,n) __rlwinm(a,n,0,31)
# elif defined(__MC68K__)
#if defined(_MSC_VER) || defined(__ICC)
#define ROTATE(a, n) _lrotl(a, n)
#elif defined(__MWERKS__)
#if defined(__POWERPC__)
#define ROTATE(a, n) __rlwinm(a, n, 0, 31)
#elif defined(__MC68K__)
/* Motorola specific tweak. <appro@fy.chalmers.se> */
# define ROTATE(a,n) ( n<24 ? __rol(a,n) : __ror(a,32-n) )
# else
# define ROTATE(a,n) __rol(a,n)
# endif
# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
#define ROTATE(a, n) (n < 24 ? __rol(a, n) : __ror(a, 32 - n))
#else
#define ROTATE(a, n) __rol(a, n)
#endif
#elif defined(__GNUC__) && __GNUC__ >= 2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
/*
* Some GNU C inline assembler templates. Note that these are
* rotates by *constant* number of bits! But that's exactly
* what we need here...
* <appro@fy.chalmers.se>
*/
# if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__)
# define ROTATE(a,n) ({ register unsigned int ret; \
asm ( \
"roll %1,%0" \
: "=r"(ret) \
: "I"(n), "0"((unsigned int)(a)) \
: "cc"); \
ret; \
})
# elif defined(_ARCH_PPC) || defined(_ARCH_PPC64) || \
defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__)
# define ROTATE(a,n) ({ register unsigned int ret; \
asm ( \
"rlwinm %0,%1,%2,0,31" \
: "=r"(ret) \
: "r"(a), "I"(n)); \
ret; \
})
# elif defined(__s390x__)
# define ROTATE(a,n) ({ register unsigned int ret; \
asm ("rll %0,%1,%2" \
: "=r"(ret) \
: "r"(a), "I"(n)); \
ret; \
})
# endif
# endif
#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__)
#define ROTATE(a, n) \
( \
{ \
register unsigned int ret; \
asm("roll %1,%0" : "=r"(ret) : "I"(n), "0"((unsigned int)(a)) : "cc"); \
ret; \
})
#elif defined(_ARCH_PPC) || defined(_ARCH_PPC64) || defined(__powerpc) || defined(__ppc__) || \
defined(__powerpc64__)
#define ROTATE(a, n) \
( \
{ \
register unsigned int ret; \
asm("rlwinm %0,%1,%2,0,31" : "=r"(ret) : "r"(a), "I"(n)); \
ret; \
})
#elif defined(__s390x__)
#define ROTATE(a, n) \
( \
{ \
register unsigned int ret; \
asm("rll %0,%1,%2" : "=r"(ret) : "r"(a), "I"(n)); \
ret; \
})
#endif
#endif
#endif /* PEDANTIC */
#ifndef ROTATE
#define ROTATE(a,n) (((a)<<(n))|(((a)&0xffffffff)>>(32-(n))))
#define ROTATE(a, n) (((a) << (n)) | (((a)&0xffffffff) >> (32 - (n))))
#endif
#if defined(DATA_ORDER_IS_BIG_ENDIAN)
#ifndef PEDANTIC
# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
# if ((defined(__i386) || defined(__i386__)) && !defined(I386_ONLY)) || \
(defined(__x86_64) || defined(__x86_64__))
# if !defined(B_ENDIAN)
#if defined(__GNUC__) && __GNUC__ >= 2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
#if ((defined(__i386) || defined(__i386__)) && !defined(I386_ONLY)) || \
(defined(__x86_64) || defined(__x86_64__))
#if !defined(B_ENDIAN)
/*
* This gives ~30-40% performance improvement in SHA-256 compiled
* with gcc [on P4]. Well, first macro to be frank. We can pull
* this trick on x86* platforms only, because these CPUs can fetch
* unaligned data without raising an exception.
*/
# define HOST_c2l(c,l) ({ unsigned int r=*((const unsigned int *)(c)); \
asm ("bswapl %0":"=r"(r):"0"(r)); \
(c)+=4; (l)=r; })
# define HOST_l2c(l,c) ({ unsigned int r=(l); \
asm ("bswapl %0":"=r"(r):"0"(r)); \
*((unsigned int *)(c))=r; (c)+=4; r; })
# endif
# endif
# endif
#define HOST_c2l(c, l) \
( \
{ \
unsigned int r = *((const unsigned int*)(c)); \
asm("bswapl %0" : "=r"(r) : "0"(r)); \
(c) += 4; \
(l) = r; \
})
#define HOST_l2c(l, c) \
( \
{ \
unsigned int r = (l); \
asm("bswapl %0" : "=r"(r) : "0"(r)); \
*((unsigned int*)(c)) = r; \
(c) += 4; \
r; \
})
#endif
#endif
#endif
#endif
#if defined(__s390__) || defined(__s390x__)
# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, (l))
# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, (l))
#define HOST_c2l(c, l) ((l) = *((const unsigned int*)(c)), (c) += 4, (l))
#define HOST_l2c(l, c) (*((unsigned int*)(c)) = (l), (c) += 4, (l))
#endif
#ifndef HOST_c2l
#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++)))<<24), \
l|=(((unsigned long)(*((c)++)))<<16), \
l|=(((unsigned long)(*((c)++)))<< 8), \
l|=(((unsigned long)(*((c)++))) ), \
l)
#define HOST_c2l(c, l) \
(l = (((unsigned long)(*((c)++))) << 24), l |= (((unsigned long)(*((c)++))) << 16), \
l |= (((unsigned long)(*((c)++))) << 8), l |= (((unsigned long)(*((c)++)))), l)
#endif
#ifndef HOST_l2c
#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l)>>24)&0xff), \
*((c)++)=(unsigned char)(((l)>>16)&0xff), \
*((c)++)=(unsigned char)(((l)>> 8)&0xff), \
*((c)++)=(unsigned char)(((l) )&0xff), \
l)
#define HOST_l2c(l, c) \
(*((c)++) = (unsigned char)(((l) >> 24) & 0xff), *((c)++) = (unsigned char)(((l) >> 16) & 0xff), \
*((c)++) = (unsigned char)(((l) >> 8) & 0xff), *((c)++) = (unsigned char)(((l)) & 0xff), l)
#endif
#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN)
#ifndef PEDANTIC
# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
# if defined(__s390x__)
# define HOST_c2l(c,l) ({ asm ("lrv %0,%1" \
:"=d"(l) :"m"(*(const unsigned int *)(c)));\
(c)+=4; (l); })
# define HOST_l2c(l,c) ({ asm ("strv %1,%0" \
:"=m"(*(unsigned int *)(c)) :"d"(l));\
(c)+=4; (l); })
# endif
# endif
#if defined(__GNUC__) && __GNUC__ >= 2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
#if defined(__s390x__)
#define HOST_c2l(c, l) \
( \
{ \
asm("lrv %0,%1" : "=d"(l) : "m"(*(const unsigned int*)(c))); \
(c) += 4; \
(l); \
})
#define HOST_l2c(l, c) \
( \
{ \
asm("strv %1,%0" : "=m"(*(unsigned int*)(c)) : "d"(l)); \
(c) += 4; \
(l); \
})
#endif
#endif
#endif
#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__)
# ifndef B_ENDIAN
#ifndef B_ENDIAN
/* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */
# define HOST_c2l(c,l) ((l)=*((const unsigned int *)(c)), (c)+=4, l)
# define HOST_l2c(l,c) (*((unsigned int *)(c))=(l), (c)+=4, l)
# endif
#define HOST_c2l(c, l) ((l) = *((const unsigned int*)(c)), (c) += 4, l)
#define HOST_l2c(l, c) (*((unsigned int*)(c)) = (l), (c) += 4, l)
#endif
#endif
#ifndef HOST_c2l
#define HOST_c2l(c,l) (l =(((unsigned long)(*((c)++))) ), \
l|=(((unsigned long)(*((c)++)))<< 8), \
l|=(((unsigned long)(*((c)++)))<<16), \
l|=(((unsigned long)(*((c)++)))<<24), \
l)
#define HOST_c2l(c, l) \
(l = (((unsigned long)(*((c)++)))), l |= (((unsigned long)(*((c)++))) << 8), \
l |= (((unsigned long)(*((c)++))) << 16), l |= (((unsigned long)(*((c)++))) << 24), l)
#endif
#ifndef HOST_l2c
#define HOST_l2c(l,c) (*((c)++)=(unsigned char)(((l) )&0xff), \
*((c)++)=(unsigned char)(((l)>> 8)&0xff), \
*((c)++)=(unsigned char)(((l)>>16)&0xff), \
*((c)++)=(unsigned char)(((l)>>24)&0xff), \
l)
#define HOST_l2c(l, c) \
(*((c)++) = (unsigned char)(((l)) & 0xff), *((c)++) = (unsigned char)(((l) >> 8) & 0xff), \
*((c)++) = (unsigned char)(((l) >> 16) & 0xff), *((c)++) = (unsigned char)(((l) >> 24) & 0xff), l)
#endif
#endif
@ -279,113 +290,112 @@
* Time for some action:-)
*/
int HASH_UPDATE (HASH_CTX* c, const void* data_, size_t len)
int HASH_UPDATE(HASH_CTX* c, const void* data_, size_t len)
{
const unsigned char* data = data_;
unsigned char* p;
HASH_LONG l;
size_t n;
if (len == 0) return 1;
l = (c->Nl + (((HASH_LONG)len) << 3)) & 0xffffffffUL;
/* 95-05-24 eay Fixed a bug with the overflow handling, thanks to
* Wei Dai <weidai@eskimo.com> for pointing it out. */
if (l < c->Nl) /* overflow */
c->Nh++;
c->Nh += (HASH_LONG)(len >> 29); /* might cause compiler warning on 16-bit */
c->Nl = l;
n = c->num;
if (n != 0)
{
p = (unsigned char*)c->data;
if (len >= HASH_CBLOCK || len + n >= HASH_CBLOCK)
{
memcpy (p + n, data, HASH_CBLOCK - n);
HASH_BLOCK_DATA_ORDER (c, p, 1);
n = HASH_CBLOCK - n;
data += n;
len -= n;
c->num = 0;
memset (p, 0, HASH_CBLOCK); /* keep it zeroed */
}
else
{
memcpy (p + n, data, len);
c->num += (unsigned int)len;
return 1;
}
}
n = len / HASH_CBLOCK;
if (n > 0)
{
HASH_BLOCK_DATA_ORDER (c, data, n);
n *= HASH_CBLOCK;
data += n;
len -= n;
}
if (len != 0)
{
p = (unsigned char*)c->data;
c->num = (unsigned int)len;
memcpy (p, data, len);
}
const unsigned char* data = data_;
unsigned char* p;
HASH_LONG l;
size_t n;
if (len == 0)
return 1;
}
l = (c->Nl + (((HASH_LONG)len) << 3)) & 0xffffffffUL;
void HASH_TRANSFORM (HASH_CTX* c, const unsigned char* data)
{
HASH_BLOCK_DATA_ORDER (c, data, 1);
}
/* 95-05-24 eay Fixed a bug with the overflow handling, thanks to
* Wei Dai <weidai@eskimo.com> for pointing it out. */
if (l < c->Nl) /* overflow */
c->Nh++;
c->Nh += (HASH_LONG)(len >> 29); /* might cause compiler warning on 16-bit */
c->Nl = l;
int HASH_FINAL (unsigned char* md, HASH_CTX* c)
{
unsigned char* p = (unsigned char*)c->data;
size_t n = c->num;
n = c->num;
p[n] = 0x80; /* there is always room for one */
n++;
if (n != 0)
{
p = (unsigned char*)c->data;
if (n > (HASH_CBLOCK - 8))
if (len >= HASH_CBLOCK || len + n >= HASH_CBLOCK)
{
memset (p + n, 0, HASH_CBLOCK - n);
n = 0;
HASH_BLOCK_DATA_ORDER (c, p, 1);
memcpy(p + n, data, HASH_CBLOCK - n);
HASH_BLOCK_DATA_ORDER(c, p, 1);
n = HASH_CBLOCK - n;
data += n;
len -= n;
c->num = 0;
memset(p, 0, HASH_CBLOCK); /* keep it zeroed */
}
else
{
memcpy(p + n, data, len);
c->num += (unsigned int)len;
return 1;
}
}
memset (p + n, 0, HASH_CBLOCK - 8 - n);
n = len / HASH_CBLOCK;
p += HASH_CBLOCK - 8;
#if defined(DATA_ORDER_IS_BIG_ENDIAN)
(void)HOST_l2c(c->Nh, p);
(void)HOST_l2c(c->Nl, p);
if (n > 0)
{
HASH_BLOCK_DATA_ORDER(c, data, n);
n *= HASH_CBLOCK;
data += n;
len -= n;
}
if (len != 0)
{
p = (unsigned char*)c->data;
c->num = (unsigned int)len;
memcpy(p, data, len);
}
return 1;
}
void HASH_TRANSFORM(HASH_CTX* c, const unsigned char* data)
{
HASH_BLOCK_DATA_ORDER(c, data, 1);
}
int HASH_FINAL(unsigned char* md, HASH_CTX* c)
{
unsigned char* p = (unsigned char*)c->data;
size_t n = c->num;
p[n] = 0x80; /* there is always room for one */
n++;
if (n > (HASH_CBLOCK - 8))
{
memset(p + n, 0, HASH_CBLOCK - n);
n = 0;
HASH_BLOCK_DATA_ORDER(c, p, 1);
}
memset(p + n, 0, HASH_CBLOCK - 8 - n);
p += HASH_CBLOCK - 8;
#if defined(DATA_ORDER_IS_BIG_ENDIAN)
(void)HOST_l2c(c->Nh, p);
(void)HOST_l2c(c->Nl, p);
#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN)
(void)HOST_l2c(c->Nl, p);
(void)HOST_l2c(c->Nh, p);
(void)HOST_l2c(c->Nl, p);
(void)HOST_l2c(c->Nh, p);
#endif
p -= HASH_CBLOCK;
HASH_BLOCK_DATA_ORDER (c, p, 1);
c->num = 0;
memset (p, 0, HASH_CBLOCK);
p -= HASH_CBLOCK;
HASH_BLOCK_DATA_ORDER(c, p, 1);
c->num = 0;
memset(p, 0, HASH_CBLOCK);
#ifndef HASH_MAKE_STRING
#error "HASH_MAKE_STRING must be defined!"
#else
HASH_MAKE_STRING(c, md);
HASH_MAKE_STRING(c, md);
#endif
return 1;
return 1;
}
#ifndef MD32_REG_T

40
utils/configcpp/md5/md5.h
View File

@ -60,20 +60,21 @@
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#ifdef __cplusplus
extern "C"
{
#endif
#ifdef OPENSSL_NO_MD5
#error MD5 is disabled.
#endif
/*
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
* ! MD5_LONG has to be at least 32 bits wide. If it's wider, then !
* ! MD5_LONG_LOG2 has to be defined along. !
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
*/
/*
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
* ! MD5_LONG has to be at least 32 bits wide. If it's wider, then !
* ! MD5_LONG_LOG2 has to be defined along. !
* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
*/
#if defined(__LP32__)
#define MD5_LONG unsigned long
@ -91,24 +92,23 @@ extern "C" {
#define MD5_LONG unsigned int
#endif
#define MD5_CBLOCK 64
#define MD5_LBLOCK (MD5_CBLOCK/4)
#define MD5_CBLOCK 64
#define MD5_LBLOCK (MD5_CBLOCK / 4)
#define MD5_DIGEST_LENGTH 16
typedef struct MD5state_st
{
typedef struct MD5state_st
{
MD5_LONG A, B, C, D;
MD5_LONG Nl, Nh;
MD5_LONG data[MD5_LBLOCK];
unsigned int num;
} MD5_CTX;
} MD5_CTX;
int MD5_Init(MD5_CTX* c);
int MD5_Update(MD5_CTX* c, const void* data, size_t len);
int MD5_Final(unsigned char* md, MD5_CTX* c);
unsigned char* MD5(const unsigned char* d, size_t n, unsigned char* md);
void MD5_Transform(MD5_CTX* c, const unsigned char* b);
#ifdef __cplusplus
int MD5_Init(MD5_CTX* c);
int MD5_Update(MD5_CTX* c, const void* data, size_t len);
int MD5_Final(unsigned char* md, MD5_CTX* c);
unsigned char* MD5(const unsigned char* d, size_t n, unsigned char* md);
void MD5_Transform(MD5_CTX* c, const unsigned char* b);
#ifdef __cplusplus
}
#endif

96
utils/configcpp/md5/md5_locl.h
View File

@ -68,32 +68,38 @@
#endif
#ifdef MD5_ASM
# if defined(__i386) || defined(__i386__) || defined(_M_IX86) || defined(__INTEL__) || \
defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
# define md5_block_data_order md5_block_asm_data_order
# elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64)
# define md5_block_data_order md5_block_asm_data_order
# endif
#if defined(__i386) || defined(__i386__) || defined(_M_IX86) || defined(__INTEL__) || defined(__x86_64) || \
defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
#define md5_block_data_order md5_block_asm_data_order
#elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64)
#define md5_block_data_order md5_block_asm_data_order
#endif
#endif
void md5_block_data_order (MD5_CTX* c, const void* p, size_t num);
void md5_block_data_order(MD5_CTX* c, const void* p, size_t num);
#define DATA_ORDER_IS_LITTLE_ENDIAN
#define HASH_LONG MD5_LONG
#define HASH_CTX MD5_CTX
#define HASH_CBLOCK MD5_CBLOCK
#define HASH_UPDATE MD5_Update
#define HASH_TRANSFORM MD5_Transform
#define HASH_FINAL MD5_Final
#define HASH_MAKE_STRING(c,s) do { \
unsigned long ll; \
ll=(c)->A; HOST_l2c(ll,(s)); \
ll=(c)->B; HOST_l2c(ll,(s)); \
ll=(c)->C; HOST_l2c(ll,(s)); \
ll=(c)->D; HOST_l2c(ll,(s)); \
} while (0)
#define HASH_BLOCK_DATA_ORDER md5_block_data_order
#define HASH_LONG MD5_LONG
#define HASH_CTX MD5_CTX
#define HASH_CBLOCK MD5_CBLOCK
#define HASH_UPDATE MD5_Update
#define HASH_TRANSFORM MD5_Transform
#define HASH_FINAL MD5_Final
#define HASH_MAKE_STRING(c, s) \
do \
{ \
unsigned long ll; \
ll = (c)->A; \
HOST_l2c(ll, (s)); \
ll = (c)->B; \
HOST_l2c(ll, (s)); \
ll = (c)->C; \
HOST_l2c(ll, (s)); \
ll = (c)->D; \
HOST_l2c(ll, (s)); \
} while (0)
#define HASH_BLOCK_DATA_ORDER md5_block_data_order
#include "md32_common.h"
@ -106,27 +112,35 @@ void md5_block_data_order (MD5_CTX* c, const void* p, size_t num);
* simplified to the code below. Wei attributes these optimizations
* to Peter Gutmann's SHS code, and he attributes it to Rich Schroeppel.
*/
#define F(b,c,d) ((((c) ^ (d)) & (b)) ^ (d))
#define G(b,c,d) ((((b) ^ (c)) & (d)) ^ (c))
#define H(b,c,d) ((b) ^ (c) ^ (d))
#define I(b,c,d) (((~(d)) | (b)) ^ (c))
#define F(b, c, d) ((((c) ^ (d)) & (b)) ^ (d))
#define G(b, c, d) ((((b) ^ (c)) & (d)) ^ (c))
#define H(b, c, d) ((b) ^ (c) ^ (d))
#define I(b, c, d) (((~(d)) | (b)) ^ (c))
#define R0(a,b,c,d,k,s,t) { \
a+=((k)+(t)+F((b),(c),(d))); \
a=ROTATE(a,s); \
a+=b; };\
#define R0(a, b, c, d, k, s, t) \
{ \
a += ((k) + (t) + F((b), (c), (d))); \
a = ROTATE(a, s); \
a += b; \
};
#define R1(a,b,c,d,k,s,t) { \
a+=((k)+(t)+G((b),(c),(d))); \
a=ROTATE(a,s); \
a+=b; };
#define R1(a, b, c, d, k, s, t) \
{ \
a += ((k) + (t) + G((b), (c), (d))); \
a = ROTATE(a, s); \
a += b; \
};
#define R2(a,b,c,d,k,s,t) { \
a+=((k)+(t)+H((b),(c),(d))); \
a=ROTATE(a,s); \
a+=b; };
#define R2(a, b, c, d, k, s, t) \
{ \
a += ((k) + (t) + H((b), (c), (d))); \
a = ROTATE(a, s); \
a += b; \
};
#define R3(a,b,c,d,k,s,t) { \
a+=((k)+(t)+I((b),(c),(d))); \
a=ROTATE(a,s); \
a+=b; };
#define R3(a, b, c, d, k, s, t) \
{ \
a += ((k) + (t) + I((b), (c), (d))); \
a = ROTATE(a, s); \
a += b; \
};

8
utils/configcpp/resource.h
View File

@ -6,9 +6,9 @@
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

625
utils/configcpp/tdriver.cpp
View File

@ -39,340 +39,333 @@ using namespace config;
class ConfigFileTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(ConfigFileTest);
CPPUNIT_TEST_SUITE( ConfigFileTest );
CPPUNIT_TEST(test1);
CPPUNIT_TEST( test1 );
CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error);
CPPUNIT_TEST(test3);
CPPUNIT_TEST(test4);
CPPUNIT_TEST_EXCEPTION(test5, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test6, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test7, std::invalid_argument);
CPPUNIT_TEST_EXCEPTION(test8, std::invalid_argument);
CPPUNIT_TEST(test9);
CPPUNIT_TEST(test10);
CPPUNIT_TEST(test11);
CPPUNIT_TEST(test12);
CPPUNIT_TEST_EXCEPTION(test13_1, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test13_2, std::runtime_error);
CPPUNIT_TEST(test14);
CPPUNIT_TEST_SUITE_END();
CPPUNIT_TEST_EXCEPTION( test2, std::runtime_error );
CPPUNIT_TEST( test3 );
CPPUNIT_TEST( test4 );
CPPUNIT_TEST_EXCEPTION( test5, std::runtime_error );
CPPUNIT_TEST_EXCEPTION( test6, std::runtime_error );
CPPUNIT_TEST_EXCEPTION( test7, std::invalid_argument );
CPPUNIT_TEST_EXCEPTION( test8, std::invalid_argument );
CPPUNIT_TEST( test9 );
CPPUNIT_TEST( test10 );
CPPUNIT_TEST( test11 );
CPPUNIT_TEST( test12 );
CPPUNIT_TEST_EXCEPTION( test13_1, std::runtime_error );
CPPUNIT_TEST_EXCEPTION( test13_2, std::runtime_error );
CPPUNIT_TEST( test14 );
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
}
private:
public:
void setUp()
void tearDown()
{
}
void test1()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string value;
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
value = c1->getConfig("Message", "xName");
CPPUNIT_ASSERT(value.size() == 0);
Config::deleteInstanceMap();
}
void test2()
{
Config* c1 = Config::makeConfig("./yadayada.xml");
string value;
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value.size() == 0);
value = c1->getConfig("Message", "xName");
CPPUNIT_ASSERT(value.size() == 0);
Config::deleteInstanceMap();
}
void test3()
{
Config* c1;
string value;
for (int i = 0; i < 1000; i++)
{
c1 = Config::makeConfig("./Columnstore.xml");
value = c1->getConfig("Message", "Name");
assert(value == "Message");
}
void tearDown()
Config::deleteInstanceMap();
}
void test4()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string value;
value = c1->getConfig("SystemConfig", "SystemVersion");
c1->setConfig("SystemConfig", "SystemVersion", "2.2.versionversionversion");
value = c1->getConfig("SystemConfig", "SystemVersion");
CPPUNIT_ASSERT(value == "2.2.versionversionversion");
::unlink("./Columnstore.xml.new");
c1->write("./Columnstore.xml.new");
value = c1->getConfig("SystemConfig", "SystemVersion");
CPPUNIT_ASSERT(value == "2.2.versionversionversion");
c1->setConfig("SystemConfig", "SystemVersion1", "V1.x");
value = c1->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.x");
c1->setConfig("SystemConfig1", "SystemVersion1", "Vx.x");
value = c1->getConfig("SystemConfig1", "SystemVersion1");
CPPUNIT_ASSERT(value == "Vx.x");
c1->write("./Columnstore.xml.new");
Config* c2 = Config::makeConfig("./Columnstore.xml.new");
value = c2->getConfig("SystemConfig1", "SystemVersion1");
CPPUNIT_ASSERT(value == "Vx.x");
c2->setConfig("SystemConfig", "SystemVersion1", "V1.1");
value = c2->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.1");
c2->write();
Config::deleteInstanceMap();
}
void test5()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
c1->write("/cantwritethis");
Config::deleteInstanceMap();
}
void test6()
{
Config* c1 = Config::makeConfig("./XColumnstore.xml");
// compiler warning...we won't actually get here
c1 = 0;
}
void test7()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string s;
string n;
string v;
c1->setConfig(s, n, v);
Config::deleteInstanceMap();
}
void test8()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string s;
string n;
string v;
v = c1->getConfig(s, n);
Config::deleteInstanceMap();
}
void test9()
{
string value;
Config* c1 = Config::makeConfig("./Columnstore.xml");
Config* c2 = Config::makeConfig("./Columnstore.xml.new");
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
value = c2->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.1");
Config::deleteInstanceMap();
}
void test10()
{
string value;
setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1);
Config* c1 = Config::makeConfig();
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
Config::deleteInstanceMap();
}
void test11()
{
string value;
struct stat stat_buf;
struct utimbuf utime_buf;
CPPUNIT_ASSERT(stat("./Columnstore.xml.new", &stat_buf) == 0);
Config* c1 = Config::makeConfig("./Columnstore.xml.new");
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
utime_buf.actime = utime_buf.modtime = stat_buf.st_mtime + 1;
CPPUNIT_ASSERT(utime("./Columnstore.xml.new", &utime_buf) == 0);
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
Config::deleteInstanceMap();
}
void test12()
{
string value;
int64_t ival;
uint64_t uval;
value = "10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 10);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 10);
value = "0x10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 0x10);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x10);
value = "010";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 010);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 010);
value = "-10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == -10);
value = "10K";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024));
value = "10k";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024));
value = "10M";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "10m";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "10G";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL));
value = "10g";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL));
value = "10MB";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "0x7afafafafafafafa";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 0x7afafafafafafafaLL);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x7afafafafafafafaULL);
value = "-0x7afafafafafafafa";
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x8505050505050506ULL);
value = "-1";
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0xffffffffffffffffULL);
}
void test13_1()
{
string value;
int64_t ival;
value = "2.2MB"; // invalid char causes throw
ival = Config::fromText(value);
}
void test13_2()
{
string value;
int64_t ival;
value = "10,000"; // invalid char causes throw
ival = Config::fromText(value);
}
void test14()
{
ByteStream bs;
ifstream ifs("./Columnstore.xml");
ifs >> bs;
string id(".");
string value;
{
ConfigStream cs(bs, id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
void test1()
string bss(reinterpret_cast<const char*>(bs.buf()), bs.length());
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string value;
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
value = c1->getConfig("Message", "xName");
CPPUNIT_ASSERT(value.size() == 0);
Config::deleteInstanceMap();
ConfigStream cs(bss, id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
void test2()
{
Config* c1 = Config::makeConfig("./yadayada.xml");
string value;
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value.size() == 0);
value = c1->getConfig("Message", "xName");
CPPUNIT_ASSERT(value.size() == 0);
Config::deleteInstanceMap();
ConfigStream cs(bss.c_str(), id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
void test3()
{
Config* c1;
string value;
for (int i = 0; i < 1000; i++)
{
c1 = Config::makeConfig("./Columnstore.xml");
value = c1->getConfig("Message", "Name");
assert(value == "Message");
}
Config::deleteInstanceMap();
}
void test4()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string value;
value = c1->getConfig("SystemConfig", "SystemVersion");
c1->setConfig("SystemConfig", "SystemVersion", "2.2.versionversionversion");
value = c1->getConfig("SystemConfig", "SystemVersion");
CPPUNIT_ASSERT(value == "2.2.versionversionversion");
::unlink("./Columnstore.xml.new");
c1->write("./Columnstore.xml.new");
value = c1->getConfig("SystemConfig", "SystemVersion");
CPPUNIT_ASSERT(value == "2.2.versionversionversion");
c1->setConfig("SystemConfig", "SystemVersion1", "V1.x");
value = c1->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.x");
c1->setConfig("SystemConfig1", "SystemVersion1", "Vx.x");
value = c1->getConfig("SystemConfig1", "SystemVersion1");
CPPUNIT_ASSERT(value == "Vx.x");
c1->write("./Columnstore.xml.new");
Config* c2 = Config::makeConfig("./Columnstore.xml.new");
value = c2->getConfig("SystemConfig1", "SystemVersion1");
CPPUNIT_ASSERT(value == "Vx.x");
c2->setConfig("SystemConfig", "SystemVersion1", "V1.1");
value = c2->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.1");
c2->write();
Config::deleteInstanceMap();
}
void test5()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
c1->write("/cantwritethis");
Config::deleteInstanceMap();
}
void test6()
{
Config* c1 = Config::makeConfig("./XColumnstore.xml");
// compiler warning...we won't actually get here
c1 = 0;
}
void test7()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string s;
string n;
string v;
c1->setConfig(s, n, v);
Config::deleteInstanceMap();
}
void test8()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string s;
string n;
string v;
v = c1->getConfig(s, n);
Config::deleteInstanceMap();
}
void test9()
{
string value;
Config* c1 = Config::makeConfig("./Columnstore.xml");
Config* c2 = Config::makeConfig("./Columnstore.xml.new");
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
value = c2->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.1");
Config::deleteInstanceMap();
}
void test10()
{
string value;
setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1);
Config* c1 = Config::makeConfig();
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
Config::deleteInstanceMap();
}
void test11()
{
string value;
struct stat stat_buf;
struct utimbuf utime_buf;
CPPUNIT_ASSERT(stat("./Columnstore.xml.new", &stat_buf) == 0);
Config* c1 = Config::makeConfig("./Columnstore.xml.new");
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
utime_buf.actime = utime_buf.modtime = stat_buf.st_mtime + 1;
CPPUNIT_ASSERT(utime("./Columnstore.xml.new", &utime_buf) == 0);
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
Config::deleteInstanceMap();
}
void test12()
{
string value;
int64_t ival;
uint64_t uval;
value = "10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 10);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 10);
value = "0x10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 0x10);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x10);
value = "010";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 010);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 010);
value = "-10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == -10);
value = "10K";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024));
value = "10k";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024));
value = "10M";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "10m";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "10G";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL));
value = "10g";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL));
value = "10MB";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "0x7afafafafafafafa";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 0x7afafafafafafafaLL);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x7afafafafafafafaULL);
value = "-0x7afafafafafafafa";
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x8505050505050506ULL);
value = "-1";
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0xffffffffffffffffULL);
}
void test13_1()
{
string value;
int64_t ival;
value = "2.2MB"; //invalid char causes throw
ival = Config::fromText(value);
}
void test13_2()
{
string value;
int64_t ival;
value = "10,000"; //invalid char causes throw
ival = Config::fromText(value);
}
void test14()
{
ByteStream bs;
ifstream ifs("./Columnstore.xml");
ifs >> bs;
string id(".");
string value;
{
ConfigStream cs(bs, id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
string bss(reinterpret_cast<const char*>(bs.buf()), bs.length());
{
ConfigStream cs(bss, id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
{
ConfigStream cs(bss.c_str(), id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
}
}
};
CPPUNIT_TEST_SUITE_REGISTRATION( ConfigFileTest );
CPPUNIT_TEST_SUITE_REGISTRATION(ConfigFileTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main( int argc, char** argv)
int main(int argc, char** argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest( registry.makeTest() );
bool wasSuccessful = runner.run( "", false );
return (wasSuccessful ? 0 : 1);
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

130
utils/configcpp/tdriver2.cpp
View File

@ -40,92 +40,88 @@ const string cf("./woparms.dat");
class WOConfigFileTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(WOConfigFileTest);
CPPUNIT_TEST_SUITE( WOConfigFileTest );
CPPUNIT_TEST(test1);
CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test3, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test4, std::runtime_error);
CPPUNIT_TEST(test5);
CPPUNIT_TEST( test1 );
CPPUNIT_TEST_EXCEPTION( test2, std::runtime_error );
CPPUNIT_TEST_EXCEPTION( test3, std::runtime_error );
CPPUNIT_TEST_EXCEPTION( test4, std::runtime_error );
CPPUNIT_TEST( test5 );
CPPUNIT_TEST_SUITE_END();
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
unlink(cf.c_str());
}
private:
void tearDown()
{
unlink(cf.c_str());
}
public:
void setUp()
{
unlink(cf.c_str());
}
void test1()
{
WriteOnceConfig woc(cf);
CPPUNIT_ASSERT(woc.owns("PrimitiveServers", "LBID_Shift"));
CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot"));
void tearDown()
{
unlink(cf.c_str());
}
CPPUNIT_ASSERT(!woc.owns("dummy", "dummy"));
void test1()
{
WriteOnceConfig woc(cf);
CPPUNIT_ASSERT(woc.owns("PrimitiveServers", "LBID_Shift"));
CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot"));
int vali;
CPPUNIT_ASSERT(!woc.owns("dummy", "dummy"));
vali = Config::fromText(woc.getConfig("PrimitiveServers", "LBID_Shift"));
CPPUNIT_ASSERT(vali == 13);
int vali;
woc.setConfig("SystemConfig", "DBRootCount", "10");
vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(vali == 10);
vali = Config::fromText(woc.getConfig("PrimitiveServers", "LBID_Shift"));
CPPUNIT_ASSERT(vali == 13);
WriteOnceConfig woc2(cf.c_str());
vali = Config::fromText(woc2.getConfig("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(vali == 10);
}
woc.setConfig("SystemConfig", "DBRootCount", "10");
vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(vali == 10);
void test2()
{
WriteOnceConfig woc(cf);
woc.getConfig("dummy", "dummy");
}
WriteOnceConfig woc2(cf.c_str());
vali = Config::fromText(woc2.getConfig("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(vali == 10);
}
void test3()
{
WriteOnceConfig woc(cf);
woc.setConfig("dummy", "dummy", "100");
}
void test2()
{
WriteOnceConfig woc(cf);
woc.getConfig("dummy", "dummy");
}
void test3()
{
WriteOnceConfig woc(cf);
woc.setConfig("dummy", "dummy", "100");
}
void test4()
{
WriteOnceConfig woc(cf);
woc.setConfig("SystemConfig", "DBRootCount", "10");
woc.setConfig("SystemConfig", "DBRootCount", "11");
}
void test5()
{
WriteOnceConfig woc(cf);
woc.setConfig("SystemConfig", "DBRootCount", "10");
woc.setConfig("SystemConfig", "DBRootCount", "11", true);
}
void test4()
{
WriteOnceConfig woc(cf);
woc.setConfig("SystemConfig", "DBRootCount", "10");
woc.setConfig("SystemConfig", "DBRootCount", "11");
}
void test5()
{
WriteOnceConfig woc(cf);
woc.setConfig("SystemConfig", "DBRootCount", "10");
woc.setConfig("SystemConfig", "DBRootCount", "11", true);
}
};
CPPUNIT_TEST_SUITE_REGISTRATION( WOConfigFileTest );
CPPUNIT_TEST_SUITE_REGISTRATION(WOConfigFileTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main( int argc, char** argv)
int main(int argc, char** argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest( registry.makeTest() );
bool wasSuccessful = runner.run( "", false );
return (wasSuccessful ? 0 : 1);
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

160
utils/configcpp/writeonce.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id: writeonce.h 3495 2013-01-21 14:09:51Z rdempsey $
*
******************************************************************************************/
* $Id: writeonce.h 3495 2013-01-21 14:09:51Z rdempsey $
*
******************************************************************************************/
/**
* @file
*/
@ -37,106 +37,104 @@
namespace config
{
/** @brief a write-once config file I/F class
*
* This class handles write-once config items
*/
class WriteOnceConfig
{
public:
/** @brief ctor
*
*/
explicit WriteOnceConfig(const char* cf = 0);
public:
/** @brief ctor
*
*/
explicit WriteOnceConfig(const char* cf = 0);
/** @brief ctor
*
*/
explicit WriteOnceConfig(const std::string& cf) :
fConfigFileName(cf)
{
setup();
}
/** @brief ctor
*
*/
explicit WriteOnceConfig(const std::string& cf) : fConfigFileName(cf)
{
setup();
}
/** @brief dtor
*
*/
virtual ~WriteOnceConfig() {}
/** @brief dtor
*
*/
virtual ~WriteOnceConfig()
{
}
/** @brief check if this class owns parm
*
*/
bool owns(const std::string& section, const std::string& name) const
{
return (fEntryMap.find(std::string(section + "." + name)) != fEntryMap.end());
}
/** @brief check if this class owns parm
*
*/
bool owns(const std::string& section, const std::string& name) const
{
return (fEntryMap.find(std::string(section + "." + name)) != fEntryMap.end());
}
/** @brief set parm to value
*
* If you attempt to set a value more than once, and force is false, this will throw a runtime_error.
*/
void setConfig(const std::string& section, const std::string& name, const std::string& value, bool force = false);
/** @brief set parm to value
*
* If you attempt to set a value more than once, and force is false, this will throw a runtime_error.
*/
void setConfig(const std::string& section, const std::string& name, const std::string& value,
bool force = false);
/** @brief get value of parm
*
*/
const std::string getConfig(const std::string& section, const std::string& name) const;
/** @brief get value of parm
*
*/
const std::string getConfig(const std::string& section, const std::string& name) const;
protected:
/** @brief load from file
*
*/
messageqcpp::ByteStream load();
protected:
/** @brief load from file
*
*/
messageqcpp::ByteStream load();
/** @brief save to file
*
*/
void save(messageqcpp::ByteStream& ibs) const;
/** @brief save to file
*
*/
void save(messageqcpp::ByteStream& ibs) const;
/** @brief serialize to ByteStream
*
*/
virtual void serialize(messageqcpp::ByteStream& obs) const;
/** @brief serialize to ByteStream
*
*/
virtual void serialize(messageqcpp::ByteStream& obs) const;
/** @brief load from ByteStream
*
*/
virtual void unserialize(messageqcpp::ByteStream& ibs);
/** @brief load from ByteStream
*
*/
virtual void unserialize(messageqcpp::ByteStream& ibs);
private:
typedef std::pair<std::string, bool> ConfigItem_t;
typedef std::tr1::unordered_map<std::string, ConfigItem_t*> EntryMap_t;
private:
typedef std::pair<std::string, bool> ConfigItem_t;
typedef std::tr1::unordered_map<std::string, ConfigItem_t*> EntryMap_t;
static const uint32_t WriteOnceConfigVersion = 1;
static const uint32_t WriteOnceConfigVersion = 1;
//defaults okay
//WriteOnceConfig(const WriteOnceConfig& rhs);
//WriteOnceConfig& operator=(const WriteOnceConfig& rhs);
// defaults okay
// WriteOnceConfig(const WriteOnceConfig& rhs);
// WriteOnceConfig& operator=(const WriteOnceConfig& rhs);
/** @brief ctor helper
*
*/
void setup();
/** @brief ctor helper
*
*/
void setup();
/** @brief setup defaults when file doesn't exist
*
*/
void initializeDefaults();
/** @brief setup defaults when file doesn't exist
*
*/
void initializeDefaults();
EntryMap_t fEntryMap;
EntryMap_t fEntryMap;
std::string fConfigFileName;
ConfigItem_t fLBID_Shift;
ConfigItem_t fDBRootCount;
ConfigItem_t fDBRMRoot;
ConfigItem_t fSharedMemoryTmpFile1;
ConfigItem_t fTxnIDFile;
ConfigItem_t fSharedMemoryTmpFile2;
std::string fConfigFileName;
ConfigItem_t fLBID_Shift;
ConfigItem_t fDBRootCount;
ConfigItem_t fDBRMRoot;
ConfigItem_t fSharedMemoryTmpFile1;
ConfigItem_t fTxnIDFile;
ConfigItem_t fSharedMemoryTmpFile2;
};
}
} // namespace config

335
utils/configcpp/xmlparser.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
#include "mcsconfig.h"
#include <string>
@ -33,257 +33,256 @@ using namespace std;
namespace config
{
const string XMLParser::getConfig(const xmlDocPtr doc, const string& section, const string& name) const
{
string res;
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
string res;
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
cur1 = cur1->xmlChildrenNode;
cur1 = cur1->xmlChildrenNode;
while (cur1 != NULL)
while (cur1 != NULL)
{
string cur1name = (const char*)cur1->name;
if ((boost::iequals(cur1name, section)))
{
string cur1name = (const char*)cur1->name;
xmlNodePtr cur2 = cur1->xmlChildrenNode;
if ((boost::iequals(cur1name, section)))
while (cur2 != NULL)
{
string cur2name = (const char*)cur2->name;
if ((boost::iequals(cur2name, name)))
{
xmlNodePtr cur2 = cur1->xmlChildrenNode;
xmlNodePtr cur3 = cur2->xmlChildrenNode;
while (cur2 != NULL)
{
string cur2name = (const char*)cur2->name;
if (cur3)
res = (const char*)cur3->content;
if ((boost::iequals(cur2name, name)))
{
xmlNodePtr cur3 = cur2->xmlChildrenNode;
if (cur3)
res = (const char*)cur3->content;
return res;
}
cur2 = cur2->next;
}
return res;
}
cur1 = cur1->next;
cur2 = cur2->next;
}
}
// maybe nullstr if not found
return res;
cur1 = cur1->next;
}
// maybe nullstr if not found
return res;
}
void XMLParser::getConfig(const xmlDocPtr doc, const string& section, const string& name, vector<string>& values) const
void XMLParser::getConfig(const xmlDocPtr doc, const string& section, const string& name,
vector<string>& values) const
{
string res;
string res;
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
cur1 = cur1->xmlChildrenNode;
cur1 = cur1->xmlChildrenNode;
while (cur1 != NULL)
while (cur1 != NULL)
{
string cur1name = (const char*)cur1->name;
if ((boost::iequals(cur1name, section)))
{
string cur1name = (const char*)cur1->name;
xmlNodePtr cur2 = cur1->xmlChildrenNode;
if ((boost::iequals(cur1name, section)))
while (cur2 != NULL)
{
string cur2name = (const char*)cur2->name;
if ((boost::iequals(cur2name, name)))
{
xmlNodePtr cur2 = cur1->xmlChildrenNode;
res.clear();
xmlNodePtr cur3 = cur2->xmlChildrenNode;
while (cur2 != NULL)
{
string cur2name = (const char*)cur2->name;
if (cur3)
res = (const char*)cur3->content;
if ((boost::iequals(cur2name, name)))
{
res.clear();
xmlNodePtr cur3 = cur2->xmlChildrenNode;
if (cur3)
res = (const char*)cur3->content;
values.push_back(res);
}
cur2 = cur2->next;
}
values.push_back(res);
}
cur1 = cur1->next;
cur2 = cur2->next;
}
}
cur1 = cur1->next;
}
}
void XMLParser::setConfig(xmlDocPtr doc, const string& section, const string& name, const string& value)
{
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
if (cur1 == NULL)
throw runtime_error("XMLParser::setConfig: error accessing XML root");
if (cur1 == NULL)
throw runtime_error("XMLParser::setConfig: error accessing XML root");
xmlNodePtr cur2;
xmlNodePtr cur2;
cur1 = cur1->xmlChildrenNode;
cur1 = cur1->xmlChildrenNode;
while (cur1 != NULL)
while (cur1 != NULL)
{
string cur1name = (const char*)cur1->name;
if (boost::iequals(cur1name, section))
{
string cur1name = (const char*)cur1->name;
cur2 = cur1->xmlChildrenNode;
if (boost::iequals(cur1name, section))
while (cur2 != NULL)
{
string cur2name = (const char*)cur2->name;
if (boost::iequals(cur2name, name))
{
cur2 = cur1->xmlChildrenNode;
xmlNodePtr cur3 = cur2->xmlChildrenNode;
while (cur2 != NULL)
{
string cur2name = (const char*)cur2->name;
if (cur3 == NULL)
{
xmlAddChild(cur2, xmlNewText((const xmlChar*)"\t"));
cur3 = cur2->xmlChildrenNode;
}
else
{
xmlFree(cur3->content);
}
if (boost::iequals(cur2name, name))
{
xmlNodePtr cur3 = cur2->xmlChildrenNode;
if (cur3 == NULL)
{
xmlAddChild(cur2, xmlNewText((const xmlChar*)"\t"));
cur3 = cur2->xmlChildrenNode;
}
else
{
xmlFree(cur3->content);
}
cur3->content = xmlStrdup((const xmlChar*)value.c_str());
return;
}
cur2 = cur2->next;
}
// We found the section, but not the name, so we need to add a new node here
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t"));
xmlNewTextChild(cur1, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str());
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n\t"));
return;
cur3->content = xmlStrdup((const xmlChar*)value.c_str());
return;
}
cur1 = cur1->next;
cur2 = cur2->next;
}
// We found the section, but not the name, so we need to add a new node here
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t"));
xmlNewTextChild(cur1, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str());
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n\t"));
return;
}
// We did not find the section, so we need to add it and the name here
cur1 = xmlDocGetRootElement(doc);
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t"));
cur2 = xmlNewChild(cur1, NULL, (const xmlChar*)section.c_str(), NULL);
xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t\t"));
xmlNewTextChild(cur2, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str());
xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t"));
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n"));
cur1 = cur1->next;
}
return;
// We did not find the section, so we need to add it and the name here
cur1 = xmlDocGetRootElement(doc);
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\t"));
cur2 = xmlNewChild(cur1, NULL, (const xmlChar*)section.c_str(), NULL);
xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t\t"));
xmlNewTextChild(cur2, NULL, (const xmlChar*)name.c_str(), (const xmlChar*)value.c_str());
xmlAddChild(cur2, xmlNewText((const xmlChar*)"\n\t"));
xmlAddChild(cur1, xmlNewText((const xmlChar*)"\n"));
return;
}
void XMLParser::delConfig(xmlDocPtr doc, const string& section, const string& name)
{
string res;
string res;
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
if (cur1 == NULL)
throw runtime_error("XMLParser::delConfig: error accessing XML root");
if (cur1 == NULL)
throw runtime_error("XMLParser::delConfig: error accessing XML root");
cur1 = cur1->xmlChildrenNode;
cur1 = cur1->xmlChildrenNode;
while (cur1 != NULL)
while (cur1 != NULL)
{
string cur1name = (const char*)cur1->name;
if ((boost::iequals(cur1name, section)))
{
string cur1name = (const char*)cur1->name;
xmlNodePtr cur2 = cur1->xmlChildrenNode;
if ((boost::iequals(cur1name, section)))
while (cur2 != NULL)
{
xmlNodePtr tmp = cur2;
cur2 = cur2->next;
string tmpname = (const char*)tmp->name;
if ((boost::iequals(tmpname, name)))
{
xmlNodePtr cur2 = cur1->xmlChildrenNode;
while (cur2 != NULL)
{
xmlNodePtr tmp = cur2;
cur2 = cur2->next;
string tmpname = (const char*)tmp->name;
if ((boost::iequals(tmpname, name)))
{
xmlUnlinkNode(tmp);
xmlFreeNode(tmp);
}
}
xmlUnlinkNode(tmp);
xmlFreeNode(tmp);
}
cur1 = cur1->next;
}
}
return;
cur1 = cur1->next;
}
return;
}
const vector<string> XMLParser::enumConfig(const xmlDocPtr doc) const
{
vector<string> resv;
string res;
vector<string> resv;
string res;
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
cur1 = cur1->xmlChildrenNode;
cur1 = cur1->xmlChildrenNode;
while (cur1 != NULL)
{
res = reinterpret_cast<const char*>(cur1->name);
while (cur1 != NULL)
{
res = reinterpret_cast<const char*>(cur1->name);
if (res != "text" && res != "comment")
resv.push_back(res);
if (res != "text" && res != "comment")
resv.push_back(res);
cur1 = cur1->next;
}
cur1 = cur1->next;
}
return resv;
return resv;
}
const vector<string> XMLParser::enumSection(const xmlDocPtr doc, const string& section) const
{
vector<string> resv;
string res;
vector<string> resv;
string res;
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
xmlNodePtr cur1 = xmlDocGetRootElement(doc);
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
if (cur1 == NULL)
throw runtime_error("XMLParser::getConfig: error accessing XML root");
cur1 = cur1->xmlChildrenNode;
cur1 = cur1->xmlChildrenNode;
while (cur1 != NULL)
while (cur1 != NULL)
{
if ((!xmlStrcmp(cur1->name, (const xmlChar*)section.c_str())))
{
if ((!xmlStrcmp(cur1->name, (const xmlChar*)section.c_str())))
{
xmlNodePtr cur2 = cur1->xmlChildrenNode;
xmlNodePtr cur2 = cur1->xmlChildrenNode;
while (cur2 != NULL)
{
res = reinterpret_cast<const char*>(cur2->name);
while (cur2 != NULL)
{
res = reinterpret_cast<const char*>(cur2->name);
if (res != "text" && res != "comment")
resv.push_back(res);
if (res != "text" && res != "comment")
resv.push_back(res);
cur2 = cur2->next;
}
}
cur1 = cur1->next;
cur2 = cur2->next;
}
}
return resv;
cur1 = cur1->next;
}
return resv;
}
} //namespace
} // namespace config
// vim:ts=4 sw=4:

46
utils/configcpp/xmlparser.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/**
* @file
*/
@ -30,37 +30,37 @@
namespace config
{
/** class XMLParser */
class XMLParser
{
public:
XMLParser() { }
~XMLParser() { }
public:
XMLParser()
{
}
~XMLParser()
{
}
const std::string getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name) const;
const std::string getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name) const;
void getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name,
std::vector<std::string>& values) const;
void getConfig(const xmlDocPtr doc, const std::string& section, const std::string& name,
std::vector<std::string>& values) const;
void setConfig(xmlDocPtr doc, const std::string& section, const std::string& name,
const std::string& value);
void setConfig(xmlDocPtr doc, const std::string& section, const std::string& name,
const std::string& value);
void delConfig(xmlDocPtr doc, const std::string& section, const std::string& name);
void delConfig(xmlDocPtr doc, const std::string& section, const std::string& name);
const std::vector<std::string> enumConfig(const xmlDocPtr doc) const;
const std::vector<std::string> enumSection(const xmlDocPtr doc, const std::string& section) const;
private:
//defaults okay
//XMLParser(const XMLParser& rhs);
//XMLParser& operator=(const XMLParser& rhs);
const std::vector<std::string> enumConfig(const xmlDocPtr doc) const;
const std::vector<std::string> enumSection(const xmlDocPtr doc, const std::string& section) const;
private:
// defaults okay
// XMLParser(const XMLParser& rhs);
// XMLParser& operator=(const XMLParser& rhs);
};
} //namespace
} // namespace config
// vim:ts=4 sw=4:

5638
utils/dataconvert/dataconvert.cpp
View File

File diff suppressed because it is too large Load Diff

2078
utils/dataconvert/dataconvert.h
View File

File diff suppressed because it is too large Load Diff

2298
utils/dataconvert/tdriver.cpp
View File

File diff suppressed because it is too large Load Diff

389
utils/ddlcleanup/ddlcleanuputil.cpp
View File

@ -16,8 +16,8 @@
MA 02110-1301, USA. */
/*
* $Id: ddlcleanuputil.cpp 3904 2013-06-18 12:22:51Z rdempsey $
*/
* $Id: ddlcleanuputil.cpp 3904 2013-06-18 12:22:51Z rdempsey $
*/
#include <unistd.h>
#include <iostream>
@ -65,206 +65,209 @@ using namespace logging;
namespace ddlcleanuputil
{
int ddl_cleanup()
{
scoped_ptr<DBRM> dbrmp(new DBRM());
DDLPackageProcessor ddlpackageprocessor(dbrmp.get());
DDLPackageProcessor::TableLogInfo tableLogs;
int rc = 0;
uint64_t uniqueId = dbrmp->getUnique64();
logging::LoggingID lid(20); // This is running in the DMLProc space, so we use DML's lid
logging::MessageLog ml(lid);
scoped_ptr<DBRM> dbrmp(new DBRM());
DDLPackageProcessor ddlpackageprocessor(dbrmp.get());
DDLPackageProcessor::TableLogInfo tableLogs;
int rc = 0;
uint64_t uniqueId = dbrmp->getUnique64();
logging::LoggingID lid(20); // This is running in the DMLProc space, so we use DML's lid
logging::MessageLog ml(lid);
//Get the logs information back first.
try
// Get the logs information back first.
try
{
ddlpackageprocessor.fetchLogFile(tableLogs, uniqueId);
}
catch (runtime_error& ex)
{
// Log to err.log
ostringstream oss;
oss << "DDLProc cannot get clean up information from DDL log files due to " << ex.what();
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logErrorMessage(message1);
}
DDLPackageProcessor::TableLogInfo::const_iterator it;
for (it = tableLogs.begin(); it != tableLogs.end(); it++)
{
ostringstream oss;
oss << "DDLCleanup is cleaning table with oid " << it->first;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logInfoMessage(message1);
DDLPackageProcessor::LogInfo aLogInfo = it->second;
switch (aLogInfo.fileType)
{
ddlpackageprocessor.fetchLogFile(tableLogs, uniqueId);
}
catch (runtime_error& ex)
{
//Log to err.log
ostringstream oss;
oss << "DDLProc cannot get clean up information from DDL log files due to " << ex.what();
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logErrorMessage( message1 );
}
DDLPackageProcessor::TableLogInfo::const_iterator it ;
for (it = tableLogs.begin(); it != tableLogs.end(); it++)
{
ostringstream oss;
oss << "DDLCleanup is cleaning table with oid " << it->first;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logInfoMessage( message1 );
DDLPackageProcessor::LogInfo aLogInfo = it->second;
switch (aLogInfo.fileType)
case DDLPackageProcessor::DROPTABLE_LOG:
{
try
{
case DDLPackageProcessor::DROPTABLE_LOG:
{
try
{
ddlpackageprocessor.flushPrimprocCache ( aLogInfo.oids );
ddlpackageprocessor.removeExtents( aLogInfo.oids );
ddlpackageprocessor.removeFiles( uniqueId, aLogInfo.oids );
//delete log file
ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPTABLE_LOG, it->first, uniqueId);
ostringstream oss;
oss << "DDLProc has cleaned up drop table left over for table " << it->first;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logInfoMessage( message1 );
}
catch (runtime_error& ex)
{
ostringstream oss;
oss << "DDLProc cannot clean up drop table for table " << it->first << " due to " << ex.what();
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logErrorMessage( message1 );
}
break;
}
case DDLPackageProcessor::DROPPART_LOG:
{
string emsg;
rc = dbrmp->markPartitionForDeletion( aLogInfo.oids, aLogInfo.partitionNums, emsg);
if (( rc != 0 ) && ( rc != BRM::ERR_PARTITION_DISABLED ))
{
continue;
}
try
{
ddlpackageprocessor.removePartitionFiles( aLogInfo.oids, aLogInfo.partitionNums, uniqueId );
cacheutils::flushPartition( aLogInfo.oids, aLogInfo.partitionNums );
emsg.clear();
rc = dbrmp->deletePartition( aLogInfo.oids, aLogInfo.partitionNums, emsg);
ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPPART_LOG, it->first, uniqueId);
ostringstream oss;
oss << "DDLProc has cleaned up drop partitions left over for table " << it->first;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logInfoMessage( message1 );
}
catch (runtime_error& ex)
{
ostringstream oss;
oss << "DDLProc cannot clean up drop partitions for table " << it->first << " due to " << ex.what();
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logErrorMessage( message1 );
}
break;
}
case DDLPackageProcessor::TRUNCATE_LOG:
{
rc = dbrmp->markAllPartitionForDeletion( aLogInfo.oids);
if (rc != 0) //Log a message to err.log
{
string errMsg;
BRM::errString(rc, errMsg);
ostringstream oss;
oss << "DDLProc didn't clean up files for truncate table with oid " << it->first << " due to " << errMsg;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logErrorMessage( message1 );
continue;
}
try
{
ddlpackageprocessor.removeFiles(uniqueId, aLogInfo.oids);
rc = cacheutils::flushOIDsFromCache(aLogInfo.oids);
ddlpackageprocessor.removeExtents( aLogInfo.oids );
//create a new sets of files. Just find a dbroot according th the number of tables in syscat-1.
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr =
CalpontSystemCatalog::makeCalpontSystemCatalog(1);
int tableCount = systemCatalogPtr->getTableCount();
int dbRootCnt = 1;
int useDBRoot = 1;
string DBRootCount = config::Config::makeConfig()->getConfig("SystemConfig", "DBRootCount");
if (DBRootCount.length() != 0)
dbRootCnt = static_cast<int>(config::Config::fromText(DBRootCount));
useDBRoot = ((tableCount - 1) % dbRootCnt) + 1;
//Create all column and dictionary files
CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(it->first);
ddlpackageprocessor.createFiles(aTableName, useDBRoot, uniqueId, static_cast<uint32_t>(aLogInfo.oids.size()));
ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::TRUNCATE_LOG, it->first, uniqueId);
ostringstream oss;
oss << "DDLProc has cleaned up truncate table left over for table " << it->first << " and the table lock is released.";
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format( args1 );
ml.logInfoMessage( message1 );
}
catch (std::exception& ex)
{
logging::Message::Args args1;
logging::Message message1(2);
ostringstream oss;
oss << "DDLProc didn't clean up truncate table left over for table with oid " << it->first << " due to " << ex.what();
args1.add(oss.str());
message1.format( args1 );
ml.logErrorMessage( message1 );
continue;
}
break;
}
default:
break;
ddlpackageprocessor.flushPrimprocCache(aLogInfo.oids);
ddlpackageprocessor.removeExtents(aLogInfo.oids);
ddlpackageprocessor.removeFiles(uniqueId, aLogInfo.oids);
// delete log file
ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPTABLE_LOG, it->first, uniqueId);
ostringstream oss;
oss << "DDLProc has cleaned up drop table left over for table " << it->first;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logInfoMessage(message1);
}
}
//unlock the table
std::vector<BRM::TableLockInfo> tableLocks;
tableLocks = dbrmp->getAllTableLocks();
for (unsigned idx = 0; idx < tableLocks.size(); idx++)
{
if (tableLocks[idx].ownerName == "DDLProc")
catch (runtime_error& ex)
{
try
{
(void)dbrmp->releaseTableLock(tableLocks[idx].id);
}
catch ( ... ) {}
ostringstream oss;
oss << "DDLProc cannot clean up drop table for table " << it->first << " due to " << ex.what();
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logErrorMessage(message1);
}
}
return 0;
break;
}
case DDLPackageProcessor::DROPPART_LOG:
{
string emsg;
rc = dbrmp->markPartitionForDeletion(aLogInfo.oids, aLogInfo.partitionNums, emsg);
if ((rc != 0) && (rc != BRM::ERR_PARTITION_DISABLED))
{
continue;
}
try
{
ddlpackageprocessor.removePartitionFiles(aLogInfo.oids, aLogInfo.partitionNums, uniqueId);
cacheutils::flushPartition(aLogInfo.oids, aLogInfo.partitionNums);
emsg.clear();
rc = dbrmp->deletePartition(aLogInfo.oids, aLogInfo.partitionNums, emsg);
ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::DROPPART_LOG, it->first, uniqueId);
ostringstream oss;
oss << "DDLProc has cleaned up drop partitions left over for table " << it->first;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logInfoMessage(message1);
}
catch (runtime_error& ex)
{
ostringstream oss;
oss << "DDLProc cannot clean up drop partitions for table " << it->first << " due to " << ex.what();
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logErrorMessage(message1);
}
break;
}
case DDLPackageProcessor::TRUNCATE_LOG:
{
rc = dbrmp->markAllPartitionForDeletion(aLogInfo.oids);
if (rc != 0) // Log a message to err.log
{
string errMsg;
BRM::errString(rc, errMsg);
ostringstream oss;
oss << "DDLProc didn't clean up files for truncate table with oid " << it->first << " due to "
<< errMsg;
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logErrorMessage(message1);
continue;
}
try
{
ddlpackageprocessor.removeFiles(uniqueId, aLogInfo.oids);
rc = cacheutils::flushOIDsFromCache(aLogInfo.oids);
ddlpackageprocessor.removeExtents(aLogInfo.oids);
// create a new sets of files. Just find a dbroot according th the number of tables in syscat-1.
boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr =
CalpontSystemCatalog::makeCalpontSystemCatalog(1);
int tableCount = systemCatalogPtr->getTableCount();
int dbRootCnt = 1;
int useDBRoot = 1;
string DBRootCount = config::Config::makeConfig()->getConfig("SystemConfig", "DBRootCount");
if (DBRootCount.length() != 0)
dbRootCnt = static_cast<int>(config::Config::fromText(DBRootCount));
useDBRoot = ((tableCount - 1) % dbRootCnt) + 1;
// Create all column and dictionary files
CalpontSystemCatalog::TableName aTableName = systemCatalogPtr->tableName(it->first);
ddlpackageprocessor.createFiles(aTableName, useDBRoot, uniqueId,
static_cast<uint32_t>(aLogInfo.oids.size()));
ddlpackageprocessor.deleteLogFile(DDLPackageProcessor::TRUNCATE_LOG, it->first, uniqueId);
ostringstream oss;
oss << "DDLProc has cleaned up truncate table left over for table " << it->first
<< " and the table lock is released.";
logging::Message::Args args1;
logging::Message message1(2);
args1.add(oss.str());
message1.format(args1);
ml.logInfoMessage(message1);
}
catch (std::exception& ex)
{
logging::Message::Args args1;
logging::Message message1(2);
ostringstream oss;
oss << "DDLProc didn't clean up truncate table left over for table with oid " << it->first
<< " due to " << ex.what();
args1.add(oss.str());
message1.format(args1);
ml.logErrorMessage(message1);
continue;
}
break;
}
default: break;
}
}
// unlock the table
std::vector<BRM::TableLockInfo> tableLocks;
tableLocks = dbrmp->getAllTableLocks();
for (unsigned idx = 0; idx < tableLocks.size(); idx++)
{
if (tableLocks[idx].ownerName == "DDLProc")
{
try
{
(void)dbrmp->releaseTableLock(tableLocks[idx].id);
}
catch (...)
{
}
}
}
return 0;
}
} //namespace ddlcleanuputil
} // namespace ddlcleanuputil
// vim:ts=4 sw=4:

3
utils/ddlcleanup/ddlcleanuputil.h
View File

@ -21,7 +21,6 @@
#pragma once
/**
* block cache utilities
*/
@ -30,5 +29,3 @@ namespace ddlcleanuputil
int ddl_cleanup();
}
// vim:ts=4 sw=4:

68
utils/funcexp/func_abs.cpp
View File

@ -17,10 +17,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_abs.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*
****************************************************************************/
* $Id: func_abs.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -36,61 +36,47 @@ using namespace rowgroup;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_abs::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType)
CalpontSystemCatalog::ColType Func_abs::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
// operation type is not used by this functor
return fp[0]->data()->resultType();
// operation type is not used by this functor
return fp[0]->data()->resultType();
}
int64_t Func_abs::getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType&)
int64_t Func_abs::getIntVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&)
{
// null value is indicated by isNull
return llabs(parm[0]->data()->getIntVal(row, isNull));
// null value is indicated by isNull
return llabs(parm[0]->data()->getIntVal(row, isNull));
}
uint64_t Func_abs::getUintVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType&)
uint64_t Func_abs::getUintVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&)
{
// null value is indicated by isNull
return parm[0]->data()->getIntVal(row, isNull);
// null value is indicated by isNull
return parm[0]->data()->getIntVal(row, isNull);
}
IDB_Decimal Func_abs::getDecimalVal(Row& row,
FunctionParm& parm,
bool& isNull,
IDB_Decimal Func_abs::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType&)
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
if (parm[0]->data()->resultType().isWideDecimalType())
d.s128Value = (d.s128Value < 0) ? -d.s128Value : d.s128Value;
else
d.value = llabs(d.value);
return d;
if (parm[0]->data()->resultType().isWideDecimalType())
d.s128Value = (d.s128Value < 0) ? -d.s128Value : d.s128Value;
else
d.value = llabs(d.value);
return d;
}
double Func_abs::getDoubleVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType&)
double Func_abs::getDoubleVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&)
{
return fabs(parm[0]->data()->getDoubleVal(row, isNull));
return fabs(parm[0]->data()->getDoubleVal(row, isNull));
}
long double Func_abs::getLongDoubleVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType&)
long double Func_abs::getLongDoubleVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType&)
{
return fabsl(parm[0]->data()->getLongDoubleVal(row, isNull));
return fabsl(parm[0]->data()->getLongDoubleVal(row, isNull));
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

424
utils/funcexp/func_add_time.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_add_time.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_add_time.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -38,283 +38,269 @@ using namespace dataconvert;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_add_time::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_add_time::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_add_time::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_add_time::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIME)
{
return getTimeIntVal(row, parm, isNull, op_ct);
}
else
{
return getDatetimeIntVal(row, parm, isNull, op_ct);
}
if (parm[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::TIME)
{
return getTimeIntVal(row, parm, isNull, op_ct);
}
else
{
return getDatetimeIntVal(row, parm, isNull, op_ct);
}
}
string Func_add_time::getStrVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
string Func_add_time::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return intToString(getIntVal(row, parm, isNull, ct));
return intToString(getIntVal(row, parm, isNull, ct));
}
int32_t Func_add_time::getDateIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int32_t Func_add_time::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E);
return (((getDatetimeIntVal(row, parm, isNull, ct) >> 32) & 0xFFFFFFC0) | 0x3E);
}
int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
int64_t Func_add_time::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
if (parm[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP)
return getTimestampIntVal(row, parm, isNull, ct);
if (parm[0]->data()->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP)
return getTimestampIntVal(row, parm, isNull, ct);
int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull);
int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull);
if (isNull)
return -1;
if (isNull)
return -1;
// Adding a zero date to a time is always NULL
if (val1 == 0)
{
isNull = true;
return -1;
}
// Adding a zero date to a time is always NULL
if (val1 == 0)
{
isNull = true;
return -1;
}
const string& val2 = parm[1]->data()->getStrVal(row, isNull);
int sign = parm[2]->data()->getIntVal(row, isNull);
DateTime dt1;
dt1.year = (val1 >> 48) & 0xffff;
dt1.month = (val1 >> 44) & 0xf;
dt1.day = (val1 >> 38) & 0x3f;
dt1.hour = (val1 >> 32) & 0x3f;
dt1.minute = (val1 >> 26) & 0x3f;
dt1.second = (val1 >> 20) & 0x3f;
dt1.msecond = val1 & 0xfffff;
const string& val2 = parm[1]->data()->getStrVal(row, isNull);
int sign = parm[2]->data()->getIntVal(row, isNull);
DateTime dt1;
dt1.year = (val1 >> 48) & 0xffff;
dt1.month = (val1 >> 44) & 0xf;
dt1.day = (val1 >> 38) & 0x3f;
dt1.hour = (val1 >> 32) & 0x3f;
dt1.minute = (val1 >> 26) & 0x3f;
dt1.second = (val1 >> 20) & 0x3f;
dt1.msecond = val1 & 0xfffff;
int64_t time = DataConvert::stringToTime(val2);
int64_t time = DataConvert::stringToTime(val2);
if (time == -1)
{
isNull = true;
return -1;
}
if (time == -1)
{
isNull = true;
return -1;
}
Time t2 = *(reinterpret_cast<Time*>(&time));
Time t2 = *(reinterpret_cast<Time*>(&time));
// MySQL TIME type range '-838:59:59' and '838:59:59'
if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999)
{
isNull = true;
return -1;
}
// MySQL TIME type range '-838:59:59' and '838:59:59'
if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999)
{
isNull = true;
return -1;
}
int val_sign = 1;
int val_sign = 1;
if (t2.hour < 0)
{
val_sign = -1;
}
if (t2.hour < 0)
{
val_sign = -1;
}
if (abs(t2.hour) > 838)
{
t2.hour = 838;
t2.minute = 59;
t2.second = 59;
t2.msecond = 999999;
}
if (abs(t2.hour) > 838)
{
t2.hour = 838;
t2.minute = 59;
t2.second = 59;
t2.msecond = 999999;
}
if (val_sign * sign < 0)
{
t2.hour = -abs(t2.hour);
t2.minute = -abs(t2.minute);
t2.second = -abs(t2.second);
t2.msecond = -abs(t2.msecond);
}
else
{
t2.hour = abs(t2.hour);
t2.minute = abs(t2.minute);
t2.second = abs(t2.second);
t2.msecond = abs(t2.msecond);
}
if (val_sign * sign < 0)
{
t2.hour = -abs(t2.hour);
t2.minute = -abs(t2.minute);
t2.second = -abs(t2.second);
t2.msecond = -abs(t2.msecond);
}
else
{
t2.hour = abs(t2.hour);
t2.minute = abs(t2.minute);
t2.second = abs(t2.second);
t2.msecond = abs(t2.msecond);
}
t2.day = 0;
t2.day = 0;
return addTime(dt1, t2);
return addTime(dt1, t2);
}
int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
int64_t Func_add_time::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull);
int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull);
if (isNull)
return -1;
if (isNull)
return -1;
// Adding a zero date to a time is always NULL
if (val1 == 0)
{
isNull = true;
return -1;
}
// Adding a zero date to a time is always NULL
if (val1 == 0)
{
isNull = true;
return -1;
}
const string& val2 = parm[1]->data()->getStrVal(row, isNull);
int sign = parm[2]->data()->getIntVal(row, isNull);
DateTime dt1;
TimeStamp timestamp(val1);
int64_t seconds = timestamp.second;
MySQLTime m_time;
gmtSecToMySQLTime(seconds, m_time, timeZone());
dt1.year = m_time.year;
dt1.month = m_time.month;
dt1.day = m_time.day;
dt1.hour = m_time.hour;
dt1.minute = m_time.minute;
dt1.second = m_time.second;
dt1.msecond = timestamp.msecond;
const string& val2 = parm[1]->data()->getStrVal(row, isNull);
int sign = parm[2]->data()->getIntVal(row, isNull);
DateTime dt1;
TimeStamp timestamp(val1);
int64_t seconds = timestamp.second;
MySQLTime m_time;
gmtSecToMySQLTime(seconds, m_time, timeZone());
dt1.year = m_time.year;
dt1.month = m_time.month;
dt1.day = m_time.day;
dt1.hour = m_time.hour;
dt1.minute = m_time.minute;
dt1.second = m_time.second;
dt1.msecond = timestamp.msecond;
int64_t time = DataConvert::stringToTime(val2);
int64_t time = DataConvert::stringToTime(val2);
if (time == -1)
{
isNull = true;
return -1;
}
if (time == -1)
{
isNull = true;
return -1;
}
Time t2 = *(reinterpret_cast<Time*>(&time));
Time t2 = *(reinterpret_cast<Time*>(&time));
// MySQL TIME type range '-838:59:59' and '838:59:59'
if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999)
{
isNull = true;
return -1;
}
// MySQL TIME type range '-838:59:59' and '838:59:59'
if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999)
{
isNull = true;
return -1;
}
int val_sign = 1;
int val_sign = 1;
if (t2.hour < 0)
{
val_sign = -1;
}
if (t2.hour < 0)
{
val_sign = -1;
}
if (abs(t2.hour) > 838)
{
t2.hour = 838;
t2.minute = 59;
t2.second = 59;
t2.msecond = 999999;
}
if (abs(t2.hour) > 838)
{
t2.hour = 838;
t2.minute = 59;
t2.second = 59;
t2.msecond = 999999;
}
if (val_sign * sign < 0)
{
t2.hour = -abs(t2.hour);
t2.minute = -abs(t2.minute);
t2.second = -abs(t2.second);
t2.msecond = -abs(t2.msecond);
}
else
{
t2.hour = abs(t2.hour);
t2.minute = abs(t2.minute);
t2.second = abs(t2.second);
t2.msecond = abs(t2.msecond);
}
if (val_sign * sign < 0)
{
t2.hour = -abs(t2.hour);
t2.minute = -abs(t2.minute);
t2.second = -abs(t2.second);
t2.msecond = -abs(t2.msecond);
}
else
{
t2.hour = abs(t2.hour);
t2.minute = abs(t2.minute);
t2.second = abs(t2.second);
t2.msecond = abs(t2.msecond);
}
t2.day = 0;
t2.day = 0;
return addTime(dt1, t2);
return addTime(dt1, t2);
}
int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_add_time::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull);
int64_t val1 = parm[0]->data()->getTimeIntVal(row, isNull);
if (isNull)
return -1;
if (isNull)
return -1;
const string& val2 = parm[1]->data()->getStrVal(row, isNull);
int sign = parm[2]->data()->getIntVal(row, isNull);
Time dt1;
dt1.day = 0;
dt1.is_neg = val1 >> 63;
dt1.hour = (val1 >> 40) & 0xfff;
dt1.minute = (val1 >> 32) & 0xff;
dt1.second = (val1 >> 24) & 0xff;
dt1.msecond = val1 & 0xffffff;
const string& val2 = parm[1]->data()->getStrVal(row, isNull);
int sign = parm[2]->data()->getIntVal(row, isNull);
Time dt1;
dt1.day = 0;
dt1.is_neg = val1 >> 63;
dt1.hour = (val1 >> 40) & 0xfff;
dt1.minute = (val1 >> 32) & 0xff;
dt1.second = (val1 >> 24) & 0xff;
dt1.msecond = val1 & 0xffffff;
int64_t time = DataConvert::stringToTime(val2);
int64_t time = DataConvert::stringToTime(val2);
if (time == -1)
{
isNull = true;
return -1;
}
if (time == -1)
{
isNull = true;
return -1;
}
Time t2 = *(reinterpret_cast<Time*>(&time));
Time t2 = *(reinterpret_cast<Time*>(&time));
// MySQL TIME type range '-838:59:59' and '838:59:59'
if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999)
{
isNull = true;
return -1;
}
// MySQL TIME type range '-838:59:59' and '838:59:59'
if (t2.minute > 59 || t2.second > 59 || t2.msecond > 999999)
{
isNull = true;
return -1;
}
int val_sign = 1;
int val_sign = 1;
if (t2.hour < 0)
{
val_sign = -1;
}
if (t2.hour < 0)
{
val_sign = -1;
}
if (abs(t2.hour) > 838)
{
t2.hour = 838;
t2.minute = 59;
t2.second = 59;
t2.msecond = 999999;
}
if (abs(t2.hour) > 838)
{
t2.hour = 838;
t2.minute = 59;
t2.second = 59;
t2.msecond = 999999;
}
t2.day = 0;
t2.day = 0;
if (val_sign * sign < 0)
{
t2.hour = -abs(t2.hour);
t2.minute = -abs(t2.minute);
t2.second = -abs(t2.second);
t2.msecond = -abs(t2.msecond);
}
else
{
t2.hour = abs(t2.hour);
t2.minute = abs(t2.minute);
t2.second = abs(t2.second);
t2.msecond = abs(t2.msecond);
}
if (val_sign * sign < 0)
{
t2.hour = -abs(t2.hour);
t2.minute = -abs(t2.minute);
t2.second = -abs(t2.second);
t2.msecond = -abs(t2.msecond);
}
else
{
t2.hour = abs(t2.hour);
t2.minute = abs(t2.minute);
t2.second = abs(t2.second);
t2.msecond = abs(t2.msecond);
}
return addTime(dt1, t2);
return addTime(dt1, t2);
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

28
utils/funcexp/func_ascii.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_ascii.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_ascii.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -38,24 +38,22 @@ using namespace dataconvert;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_ascii::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_ascii::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_ascii::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_ascii::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
const string& str = parm[0]->data()->getStrVal(row, isNull);
const string& str = parm[0]->data()->getStrVal(row, isNull);
if (str.empty())
return 0;
if (str.empty())
return 0;
return (unsigned char)str[0];
return (unsigned char)str[0];
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

598
utils/funcexp/func_between.cpp
View File

@ -17,10 +17,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_between.cpp 3954 2013-07-08 16:30:15Z bpaul $
*
*
****************************************************************************/
* $Id: func_between.cpp 3954 2013-07-08 16:30:15Z bpaul $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -38,366 +38,342 @@ using namespace execplan;
#include "idberrorinfo.h"
#include "errorids.h"
using namespace logging;
namespace
{
template<typename result_t>
template <typename result_t>
inline bool numericGE(result_t op1, result_t op2)
{
return op1 >= op2;
return op1 >= op2;
}
template<typename result_t>
template <typename result_t>
inline bool numericLE(result_t op1, result_t op2)
{
return op1 <= op2;
return op1 <= op2;
}
inline bool strGE(CHARSET_INFO &cs, const string& op1, const string& op2)
inline bool strGE(CHARSET_INFO& cs, const string& op1, const string& op2)
{
return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) >= 0;
return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) >= 0;
}
inline bool strLE(CHARSET_INFO &cs, const string& op1, const string& op2)
inline bool strLE(CHARSET_INFO& cs, const string& op1, const string& op2)
{
return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) <= 0;
return cs.strnncoll(op1.c_str(), op1.length(), op2.c_str(), op2.length()) <= 0;
}
inline bool getBool(rowgroup::Row& row,
funcexp::FunctionParm& pm,
bool& isNull,
CalpontSystemCatalog::ColType& ct,
bool notBetween)
inline bool getBool(rowgroup::Row& row, funcexp::FunctionParm& pm, bool& isNull,
CalpontSystemCatalog::ColType& ct, bool notBetween)
{
switch (ct.colDataType)
switch (ct.colDataType)
{
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
{
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
{
int64_t val = pm[0]->data()->getIntVal(row, isNull);
int64_t val = pm[0]->data()->getIntVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getIntVal(row, isNull)) && !isNull)
return true;
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getIntVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull);
}
isNull = false;
return (!numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull;
}
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
{
uint64_t val = pm[0]->data()->getUintVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getUintVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getUintVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull;
}
case execplan::CalpontSystemCatalog::DATE:
{
int32_t val = pm[0]->data()->getDateIntVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDateIntVal(row, isNull)) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDateIntVal(row, isNull));
}
case execplan::CalpontSystemCatalog::DATETIME:
{
int64_t val = pm[0]->data()->getDatetimeIntVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull));
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull));
}
case execplan::CalpontSystemCatalog::TIME:
{
// Shift out unused day for compare
int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12;
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) &&
numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12);
}
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
{
double val = pm[0]->data()->getDoubleVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDoubleVal(row, isNull)) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDoubleVal(row, isNull));
}
case execplan::CalpontSystemCatalog::LONGDOUBLE:
{
long double val = pm[0]->data()->getLongDoubleVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull)) && !isNull);
}
return !isNull &&
numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull));
}
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull);
if (isNull)
return false;
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)) && !isNull);
}
return numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDecimalVal(row, isNull));
}
case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR'
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
{
const string& val = pm[0]->data()->getStrVal(row, isNull);
CHARSET_INFO &cs = datatypes::Charset(ct.charsetNumber).getCharset();
if (notBetween)
{
if (!strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!strLE(cs, val, pm[2]->data()->getStrVal(row, isNull)) && !isNull);
}
return !isNull &&
strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) &&
strLE(cs, val, pm[2]->data()->getStrVal(row, isNull));
}
default:
{
std::ostringstream oss;
oss << "between: datatype of " << execplan::colDataTypeToString(ct.colDataType);
throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT);
}
return !isNull && numericGE(val, pm[1]->data()->getIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getIntVal(row, isNull)) && !isNull;
}
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
{
uint64_t val = pm[0]->data()->getUintVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getUintVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull);
}
return !isNull && numericGE(val, pm[1]->data()->getUintVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getUintVal(row, isNull)) && !isNull;
}
case execplan::CalpontSystemCatalog::DATE:
{
int32_t val = pm[0]->data()->getDateIntVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDateIntVal(row, isNull)) && !isNull);
}
return !isNull && numericGE(val, pm[1]->data()->getDateIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDateIntVal(row, isNull));
}
case execplan::CalpontSystemCatalog::DATETIME:
{
int64_t val = pm[0]->data()->getDatetimeIntVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull)) && !isNull);
}
return !isNull && numericGE(val, pm[1]->data()->getDatetimeIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDatetimeIntVal(row, isNull));
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
int64_t val = pm[0]->data()->getTimestampIntVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull)) && !isNull);
}
return !isNull && numericGE(val, pm[1]->data()->getTimestampIntVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getTimestampIntVal(row, isNull));
}
case execplan::CalpontSystemCatalog::TIME:
{
// Shift out unused day for compare
int64_t val = pm[0]->data()->getTimeIntVal(row, isNull) << 12;
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12) && !isNull);
}
return !isNull && numericGE(val, pm[1]->data()->getTimeIntVal(row, isNull) << 12) &&
numericLE(val, pm[2]->data()->getTimeIntVal(row, isNull) << 12);
}
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
{
double val = pm[0]->data()->getDoubleVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDoubleVal(row, isNull)) && !isNull);
}
return !isNull && numericGE(val, pm[1]->data()->getDoubleVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDoubleVal(row, isNull));
}
case execplan::CalpontSystemCatalog::LONGDOUBLE:
{
long double val = pm[0]->data()->getLongDoubleVal(row, isNull);
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull)) && !isNull);
}
return !isNull && numericGE(val, pm[1]->data()->getLongDoubleVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getLongDoubleVal(row, isNull));
}
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull);
if (isNull)
return false;
if (notBetween)
{
if (!numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)) && !isNull);
}
return numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) &&
numericLE(val, pm[2]->data()->getDecimalVal(row, isNull));
}
case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR'
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
{
const string& val = pm[0]->data()->getStrVal(row, isNull);
CHARSET_INFO& cs = datatypes::Charset(ct.charsetNumber).getCharset();
if (notBetween)
{
if (!strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) && !isNull)
return true;
isNull = false;
return (!strLE(cs, val, pm[2]->data()->getStrVal(row, isNull)) && !isNull);
}
return !isNull && strGE(cs, val, pm[1]->data()->getStrVal(row, isNull)) &&
strLE(cs, val, pm[2]->data()->getStrVal(row, isNull));
}
default:
{
std::ostringstream oss;
oss << "between: datatype of " << execplan::colDataTypeToString(ct.colDataType);
throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT);
}
}
}
}
} // namespace
namespace funcexp
{
CalpontSystemCatalog::ColType Func_between::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_between::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
PredicateOperator op;
CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType();
//op.operationType(fp[0]->data()->resultType());
bool allString = true;
PredicateOperator op;
CalpontSystemCatalog::ColType ct = fp[0]->data()->resultType();
// op.operationType(fp[0]->data()->resultType());
bool allString = true;
for (uint32_t i = 1; i < fp.size(); i++)
{
// op.setOpType(op.operationType(), fp[i]->data()->resultType());
op.setOpType(ct, fp[i]->data()->resultType());
ct = op.operationType();
if ((fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) ||
ct.colDataType == CalpontSystemCatalog::DATE || ct.colDataType == CalpontSystemCatalog::DATETIME ||
ct.colDataType == CalpontSystemCatalog::TIMESTAMP || ct.colDataType == CalpontSystemCatalog::TIME)
{
allString = false;
}
}
if (allString)
{
ct.colDataType = CalpontSystemCatalog::VARCHAR;
ct.colWidth = 255;
}
else if (op.operationType().colDataType == CalpontSystemCatalog::DATETIME)
{
ConstantColumn* cc = NULL;
for (uint32_t i = 1; i < fp.size(); i++)
{
//op.setOpType(op.operationType(), fp[i]->data()->resultType());
op.setOpType(ct, fp[i]->data()->resultType());
ct = op.operationType();
cc = dynamic_cast<ConstantColumn*>(fp[i]->data());
if ((fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT &&
fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) ||
ct.colDataType == CalpontSystemCatalog::DATE ||
ct.colDataType == CalpontSystemCatalog::DATETIME ||
ct.colDataType == CalpontSystemCatalog::TIMESTAMP ||
ct.colDataType == CalpontSystemCatalog::TIME)
{
allString = false;
}
if (cc)
{
Result result = cc->result();
result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal);
cc->result(result);
}
}
}
else if (op.operationType().colDataType == CalpontSystemCatalog::TIMESTAMP)
{
ConstantColumn* cc = NULL;
if (allString)
for (uint32_t i = 1; i < fp.size(); i++)
{
ct.colDataType = CalpontSystemCatalog::VARCHAR;
ct.colWidth = 255;
}
cc = dynamic_cast<ConstantColumn*>(fp[i]->data());
else if (op.operationType().colDataType == CalpontSystemCatalog::DATETIME)
if (cc)
{
Result result = cc->result();
result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone());
cc->result(result);
}
}
}
else if (op.operationType().colDataType == CalpontSystemCatalog::TIME)
{
ConstantColumn* cc = NULL;
for (uint32_t i = 1; i < fp.size(); i++)
{
ConstantColumn* cc = NULL;
cc = dynamic_cast<ConstantColumn*>(fp[i]->data());
for (uint32_t i = 1; i < fp.size(); i++)
{
cc = dynamic_cast<ConstantColumn*>(fp[i]->data());
if (cc)
{
Result result = cc->result();
result.intVal = dataconvert::DataConvert::datetimeToInt(result.strVal);
cc->result(result);
}
}
if (cc)
{
Result result = cc->result();
result.intVal = dataconvert::DataConvert::timeToInt(result.strVal);
cc->result(result);
}
}
else if (op.operationType().colDataType == CalpontSystemCatalog::TIMESTAMP)
{
ConstantColumn* cc = NULL;
}
for (uint32_t i = 1; i < fp.size(); i++)
{
cc = dynamic_cast<ConstantColumn*>(fp[i]->data());
if (cc)
{
Result result = cc->result();
result.intVal = dataconvert::DataConvert::timestampToInt(result.strVal, timeZone());
cc->result(result);
}
}
}
else if (op.operationType().colDataType == CalpontSystemCatalog::TIME)
{
ConstantColumn* cc = NULL;
for (uint32_t i = 1; i < fp.size(); i++)
{
cc = dynamic_cast<ConstantColumn*>(fp[i]->data());
if (cc)
{
Result result = cc->result();
result.intVal = dataconvert::DataConvert::timeToInt(result.strVal);
cc->result(result);
}
}
}
return ct;
return ct;
}
bool Func_between::getBoolVal(rowgroup::Row& row,
FunctionParm& pm,
bool& isNull,
bool Func_between::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return getBool(row, pm, isNull, ct, false) && !isNull;
return getBool(row, pm, isNull, ct, false) && !isNull;
}
CalpontSystemCatalog::ColType Func_notbetween::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_notbetween::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
PredicateOperator* op = new PredicateOperator();
CalpontSystemCatalog::ColType ct;
op->setOpType(fp[0]->data()->resultType(), fp[1]->data()->resultType());
op->setOpType(op->resultType(), fp[2]->data()->resultType());
return op->operationType();
PredicateOperator* op = new PredicateOperator();
CalpontSystemCatalog::ColType ct;
op->setOpType(fp[0]->data()->resultType(), fp[1]->data()->resultType());
op->setOpType(op->resultType(), fp[2]->data()->resultType());
return op->operationType();
}
bool Func_notbetween::getBoolVal(rowgroup::Row& row,
FunctionParm& pm,
bool& isNull,
bool Func_notbetween::getBoolVal(rowgroup::Row& row, FunctionParm& pm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return getBool(row, pm, isNull, ct, true) && !isNull;
return getBool(row, pm, isNull, ct, true) && !isNull;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

714
utils/funcexp/func_bitwise.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_bitwise.cpp 3616 2013-03-04 14:56:29Z rdempsey $
*
*
****************************************************************************/
* $Id: func_bitwise.cpp 3616 2013-03-04 14:56:29Z rdempsey $
*
*
****************************************************************************/
#include <string>
using namespace std;
@ -49,52 +49,44 @@ namespace
{
using namespace funcexp;
bool validateBitOperandTypeOrError(execplan::FunctionColumn &col,
const Func & func,
uint argno)
bool validateBitOperandTypeOrError(execplan::FunctionColumn& col, const Func& func, uint argno)
{
auto & type = col.functionParms()[argno]->data()->resultType();
if (type.canReturnXInt64())
return false;
func.raiseIllegalParameterDataTypeError(type);
return true;
auto& type = col.functionParms()[argno]->data()->resultType();
if (type.canReturnXInt64())
return false;
func.raiseIllegalParameterDataTypeError(type);
return true;
}
template<typename T>
datatypes::TUInt64Null ConvertToBitOperand(const T &val)
template <typename T>
datatypes::TUInt64Null ConvertToBitOperand(const T& val)
{
if (val > static_cast<T>(UINT64_MAX))
return datatypes::TUInt64Null(UINT64_MAX);
if (val >= 0)
return datatypes::TUInt64Null(static_cast<uint64_t>(val));
if (val < static_cast<T>(INT64_MIN))
return datatypes::TUInt64Null(static_cast<uint64_t>(INT64_MAX)+1);
return datatypes::TUInt64Null((uint64_t) (int64_t) val);
if (val > static_cast<T>(UINT64_MAX))
return datatypes::TUInt64Null(UINT64_MAX);
if (val >= 0)
return datatypes::TUInt64Null(static_cast<uint64_t>(val));
if (val < static_cast<T>(INT64_MIN))
return datatypes::TUInt64Null(static_cast<uint64_t>(INT64_MAX) + 1);
return datatypes::TUInt64Null((uint64_t)(int64_t)val);
}
static
datatypes::TUInt64Null DecimalToBitOperand(Row& row,
const execplan::SPTP& parm,
const funcexp::Func& thisFunc)
static datatypes::TUInt64Null DecimalToBitOperand(Row& row, const execplan::SPTP& parm,
const funcexp::Func& thisFunc)
{
bool tmpIsNull = false;
datatypes::Decimal d = parm->data()->getDecimalVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
bool tmpIsNull = false;
datatypes::Decimal d = parm->data()->getDecimalVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH)
{
int128_t val = d.getPosNegRoundedIntegralPart(0).getValue();
return ConvertToBitOperand<int128_t>(val);
}
if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH)
{
int128_t val = d.getPosNegRoundedIntegralPart(0).getValue();
return ConvertToBitOperand<int128_t>(val);
}
return datatypes::TUInt64Null((uint64_t) d.decimal64ToSInt64Round());
return datatypes::TUInt64Null((uint64_t)d.decimal64ToSInt64Round());
}
// Functions TypeHolderStd::canReturnXInt64() and GenericToBitOperand()
// should be splitted eventually to virtual methods in TypeHandler.
//
@ -112,460 +104,392 @@ datatypes::TUInt64Null DecimalToBitOperand(Row& row,
// here. This same method is potentially useful in other methods
// and could be extracted into a utility class with its own header
// if that is the case - this is left as future exercise
datatypes::TUInt64Null GenericToBitOperand(
Row& row,
const execplan::SPTP& parm,
const funcexp::Func& thisFunc,
bool temporalRounding)
datatypes::TUInt64Null GenericToBitOperand(Row& row, const execplan::SPTP& parm,
const funcexp::Func& thisFunc, bool temporalRounding)
{
switch (parm->data()->resultType().colDataType)
switch (parm->data()->resultType().colDataType)
{
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
{
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
{
datatypes::TSInt64Null tmp= parm->data()->toTSInt64Null(row);
return tmp.isNull() ? datatypes::TUInt64Null() :
datatypes::TUInt64Null((uint64_t) (int64_t) tmp);
}
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::UFLOAT:
{
bool tmpIsNull = false;
double val = parm->data()->getDoubleVal(row, tmpIsNull);
return tmpIsNull ? datatypes::TUInt64Null() :
ConvertToBitOperand<double>(round(val));
}
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
return parm->data()->toTUInt64Null(row);
case execplan::CalpontSystemCatalog::VARCHAR:
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
{
bool tmpIsNull = false;
const string& str = parm->data()->getStrVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
datatypes::DataCondition cnverr;
literal::Converter<literal::SignedNumericLiteral> cnv(str, cnverr);
cnv.normalize();
return cnv.negative() ?
datatypes::TUInt64Null((uint64_t)cnv.toPackedSDecimal<int64_t>(0, cnverr)) :
datatypes::TUInt64Null(cnv.toPackedUDecimal<uint64_t>(0, cnverr));
}
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
return DecimalToBitOperand(row, parm, thisFunc);
case execplan::CalpontSystemCatalog::DATE:
{
bool tmpIsNull = false;
int32_t time = parm->data()->getDateIntVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
int64_t value = Date(time).convertToMySQLint();
return datatypes::TUInt64Null((uint64_t) value);
}
case execplan::CalpontSystemCatalog::DATETIME:
{
bool tmpIsNull = false;
int64_t time = parm->data()->getDatetimeIntVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
// @bug 4703 - missing year when convering to int
DateTime dt(time);
int64_t value = dt.convertToMySQLint();
if (temporalRounding && dt.msecond >= 500000)
value++;
return datatypes::TUInt64Null((uint64_t) value);
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
bool tmpIsNull = false;
int64_t time = parm->data()->getTimestampIntVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
TimeStamp dt(time);
int64_t value = dt.convertToMySQLint(thisFunc.timeZone());
if (temporalRounding && dt.msecond >= 500000)
value++;
return datatypes::TUInt64Null((uint64_t) value);
}
case execplan::CalpontSystemCatalog::TIME:
{
bool tmpIsNull = false;
int64_t time = parm->data()->getTimeIntVal(row, tmpIsNull);
Time dt(time);
int64_t value = dt.convertToMySQLint();
if (temporalRounding && dt.msecond >= 500000)
value < 0 ? value-- : value++;
return datatypes::TUInt64Null((uint64_t) value);
}
default:
idbassert(0); // Not possible: checked during the preparation stage.
break;
datatypes::TSInt64Null tmp = parm->data()->toTSInt64Null(row);
return tmp.isNull() ? datatypes::TUInt64Null() : datatypes::TUInt64Null((uint64_t)(int64_t)tmp);
}
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::UFLOAT:
{
bool tmpIsNull = false;
double val = parm->data()->getDoubleVal(row, tmpIsNull);
return tmpIsNull ? datatypes::TUInt64Null() : ConvertToBitOperand<double>(round(val));
}
return datatypes::TUInt64Null();
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT: return parm->data()->toTUInt64Null(row);
case execplan::CalpontSystemCatalog::VARCHAR:
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
{
bool tmpIsNull = false;
const string& str = parm->data()->getStrVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
datatypes::DataCondition cnverr;
literal::Converter<literal::SignedNumericLiteral> cnv(str, cnverr);
cnv.normalize();
return cnv.negative() ? datatypes::TUInt64Null((uint64_t)cnv.toPackedSDecimal<int64_t>(0, cnverr))
: datatypes::TUInt64Null(cnv.toPackedUDecimal<uint64_t>(0, cnverr));
}
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL: return DecimalToBitOperand(row, parm, thisFunc);
case execplan::CalpontSystemCatalog::DATE:
{
bool tmpIsNull = false;
int32_t time = parm->data()->getDateIntVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
int64_t value = Date(time).convertToMySQLint();
return datatypes::TUInt64Null((uint64_t)value);
}
case execplan::CalpontSystemCatalog::DATETIME:
{
bool tmpIsNull = false;
int64_t time = parm->data()->getDatetimeIntVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
// @bug 4703 - missing year when convering to int
DateTime dt(time);
int64_t value = dt.convertToMySQLint();
if (temporalRounding && dt.msecond >= 500000)
value++;
return datatypes::TUInt64Null((uint64_t)value);
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
bool tmpIsNull = false;
int64_t time = parm->data()->getTimestampIntVal(row, tmpIsNull);
if (tmpIsNull)
return datatypes::TUInt64Null();
TimeStamp dt(time);
int64_t value = dt.convertToMySQLint(thisFunc.timeZone());
if (temporalRounding && dt.msecond >= 500000)
value++;
return datatypes::TUInt64Null((uint64_t)value);
}
case execplan::CalpontSystemCatalog::TIME:
{
bool tmpIsNull = false;
int64_t time = parm->data()->getTimeIntVal(row, tmpIsNull);
Time dt(time);
int64_t value = dt.convertToMySQLint();
if (temporalRounding && dt.msecond >= 500000)
value < 0 ? value-- : value++;
return datatypes::TUInt64Null((uint64_t)value);
}
default:
idbassert(0); // Not possible: checked during the preparation stage.
break;
}
return datatypes::TUInt64Null();
}
}
} // namespace
namespace funcexp
{
class BitOperandGeneric: public datatypes::TUInt64Null
class BitOperandGeneric : public datatypes::TUInt64Null
{
public:
BitOperandGeneric() { }
BitOperandGeneric(Row& row,
const execplan::SPTP& parm,
const funcexp::Func& thisFunc)
:TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true))
{ }
public:
BitOperandGeneric()
{
}
BitOperandGeneric(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc)
: TUInt64Null(GenericToBitOperand(row, parm, thisFunc, true))
{
}
};
// The shift amount operand in MariaDB does not round temporal values
// when sql_mode=TIME_FRAC_ROUND is not set.
class BitOperandGenericShiftAmount: public datatypes::TUInt64Null
class BitOperandGenericShiftAmount : public datatypes::TUInt64Null
{
public:
BitOperandGenericShiftAmount() { }
BitOperandGenericShiftAmount(Row& row,
const execplan::SPTP& parm,
const funcexp::Func& thisFunc)
:TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false))
{ }
public:
BitOperandGenericShiftAmount()
{
}
BitOperandGenericShiftAmount(Row& row, const execplan::SPTP& parm, const funcexp::Func& thisFunc)
: TUInt64Null(GenericToBitOperand(row, parm, thisFunc, false))
{
}
};
// A functor to return NULL as a bitwise operation result.
// Used when an unexpected argument count
// is encounteded during the preparation step.
class Func_bitwise_null: public Func_BitOp
class Func_bitwise_null : public Func_BitOp
{
public:
Func_bitwise_null(): Func_BitOp("bitwise") { }
int64_t getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
isNull = true;
return 0;
}
public:
Func_bitwise_null() : Func_BitOp("bitwise")
{
}
int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
isNull = true;
return 0;
}
};
bool Func_BitOp::validateArgCount(execplan::FunctionColumn &col, uint expected) const
bool Func_BitOp::validateArgCount(execplan::FunctionColumn& col, uint expected) const
{
static Func_bitwise_null return_null;
if (col.functionParms().size() == expected)
return false;
col.setFunctor(&return_null);
return true;
static Func_bitwise_null return_null;
if (col.functionParms().size() == expected)
return false;
col.setFunctor(&return_null);
return true;
}
void Func_BitOp::setFunctorByParm(execplan::FunctionColumn &col,
const execplan::SPTP& parm,
Func_Int & return_uint64_from_uint64,
Func_Int & return_uint64_from_sint64,
Func_Int & return_uint64_generic) const
void Func_BitOp::setFunctorByParm(execplan::FunctionColumn& col, const execplan::SPTP& parm,
Func_Int& return_uint64_from_uint64, Func_Int& return_uint64_from_sint64,
Func_Int& return_uint64_generic) const
{
if (parm->data()->resultType().isUnsignedInteger())
col.setFunctor(&return_uint64_from_uint64);
else if (parm->data()->resultType().isSignedInteger())
col.setFunctor(&return_uint64_from_sint64);
else
col.setFunctor(&return_uint64_generic);
}
bool Func_BitOp::fixForBitShift(execplan::FunctionColumn &col,
Func_Int & return_uint64_from_uint64,
Func_Int & return_uint64_from_sint64,
Func_Int & return_uint64_generic) const
{
if (validateArgCount(col, 2))
return false;
// The functor detection is done using functionParms()[0] only.
// This is how MariaDB performs it.
setFunctorByParm(col, col.functionParms()[0],
return_uint64_from_uint64,
return_uint64_from_sint64,
return_uint64_generic);
return validateBitOperandTypeOrError(col, *this, 0) ||
validateBitOperandTypeOrError(col, *this, 1);
}
bool Func_BitOp::fixForBitOp2(execplan::FunctionColumn &col,
Func_Int & return_uint64_from_uint64_uint64,
Func_Int & return_uint64_from_sint64_sint64,
Func_Int & return_uint64_generic) const
{
if (validateArgCount(col, 2))
return false;
if (col.functionParms()[0]->data()->resultType().isUnsignedInteger() &&
col.functionParms()[1]->data()->resultType().isUnsignedInteger())
{
col.setFunctor(&return_uint64_from_uint64_uint64);
return false;
}
if (col.functionParms()[0]->data()->resultType().isSignedInteger() &&
col.functionParms()[1]->data()->resultType().isSignedInteger())
{
col.setFunctor(&return_uint64_from_sint64_sint64);
return false;
}
if (parm->data()->resultType().isUnsignedInteger())
col.setFunctor(&return_uint64_from_uint64);
else if (parm->data()->resultType().isSignedInteger())
col.setFunctor(&return_uint64_from_sint64);
else
col.setFunctor(&return_uint64_generic);
return validateBitOperandTypeOrError(col, *this, 0) ||
validateBitOperandTypeOrError(col, *this, 1);
}
bool Func_BitOp::fixForBitShift(execplan::FunctionColumn& col, Func_Int& return_uint64_from_uint64,
Func_Int& return_uint64_from_sint64, Func_Int& return_uint64_generic) const
{
if (validateArgCount(col, 2))
return false;
// The functor detection is done using functionParms()[0] only.
// This is how MariaDB performs it.
setFunctorByParm(col, col.functionParms()[0], return_uint64_from_uint64, return_uint64_from_sint64,
return_uint64_generic);
return validateBitOperandTypeOrError(col, *this, 0) || validateBitOperandTypeOrError(col, *this, 1);
}
bool Func_BitOp::fixForBitOp2(execplan::FunctionColumn& col, Func_Int& return_uint64_from_uint64_uint64,
Func_Int& return_uint64_from_sint64_sint64,
Func_Int& return_uint64_generic) const
{
if (validateArgCount(col, 2))
return false;
if (col.functionParms()[0]->data()->resultType().isUnsignedInteger() &&
col.functionParms()[1]->data()->resultType().isUnsignedInteger())
{
col.setFunctor(&return_uint64_from_uint64_uint64);
return false;
}
if (col.functionParms()[0]->data()->resultType().isSignedInteger() &&
col.functionParms()[1]->data()->resultType().isSignedInteger())
{
col.setFunctor(&return_uint64_from_sint64_sint64);
return false;
}
col.setFunctor(&return_uint64_generic);
return validateBitOperandTypeOrError(col, *this, 0) || validateBitOperandTypeOrError(col, *this, 1);
}
//
// BITAND
//
template<class TA, class TB>
class Func_bitand_return_uint64: public Func_bitand
template <class TA, class TB>
class Func_bitand_return_uint64 : public Func_bitand
{
public:
int64_t getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Lazy<TA, TB> args(row, parm, *this);
return (int64_t) (args.a & args.b).nullSafeValue(isNull);
}
public:
int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Lazy<TA, TB> args(row, parm, *this);
return (int64_t)(args.a & args.b).nullSafeValue(isNull);
}
};
bool Func_bitand::fix(execplan::FunctionColumn &col) const
bool Func_bitand::fix(execplan::FunctionColumn& col) const
{
static Func_bitand_return_uint64<ParmTUInt64, ParmTUInt64> return_uint64_from_uint64_uint64;
static Func_bitand_return_uint64<ParmTSInt64, ParmTSInt64> return_uint64_from_sint64_sint64;
static Func_bitand_return_uint64<BitOperandGeneric, BitOperandGeneric> return_uint64_generic;
return fixForBitOp2(col, return_uint64_from_uint64_uint64,
return_uint64_from_sint64_sint64,
return_uint64_generic);
static Func_bitand_return_uint64<ParmTUInt64, ParmTUInt64> return_uint64_from_uint64_uint64;
static Func_bitand_return_uint64<ParmTSInt64, ParmTSInt64> return_uint64_from_sint64_sint64;
static Func_bitand_return_uint64<BitOperandGeneric, BitOperandGeneric> return_uint64_generic;
return fixForBitOp2(col, return_uint64_from_uint64_uint64, return_uint64_from_sint64_sint64,
return_uint64_generic);
}
//
// LEFT SHIFT
//
template<class TA>
class Func_leftshift_return_uint64: public Func_leftshift
template <class TA>
class Func_leftshift_return_uint64 : public Func_leftshift
{
public:
int64_t getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Eager<TA, BitOperandGenericShiftAmount> args(row, parm, *this);
return (int64_t) args.a.MariaDBShiftLeft(args.b).nullSafeValue(isNull);
}
public:
int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Eager<TA, BitOperandGenericShiftAmount> args(row, parm, *this);
return (int64_t)args.a.MariaDBShiftLeft(args.b).nullSafeValue(isNull);
}
};
bool Func_leftshift::fix(execplan::FunctionColumn &col) const
bool Func_leftshift::fix(execplan::FunctionColumn& col) const
{
static Func_leftshift_return_uint64<ParmTUInt64> return_uint64_from_uint64;
static Func_leftshift_return_uint64<ParmTSInt64> return_uint64_from_sint64;
static Func_leftshift_return_uint64<BitOperandGeneric> return_uint64_generic;
return fixForBitShift(col, return_uint64_from_uint64,
return_uint64_from_sint64,
return_uint64_generic);
static Func_leftshift_return_uint64<ParmTUInt64> return_uint64_from_uint64;
static Func_leftshift_return_uint64<ParmTSInt64> return_uint64_from_sint64;
static Func_leftshift_return_uint64<BitOperandGeneric> return_uint64_generic;
return fixForBitShift(col, return_uint64_from_uint64, return_uint64_from_sint64, return_uint64_generic);
}
//
// RIGHT SHIFT
//
template<class TA>
class Func_rightshift_return_uint64: public Func_rightshift
template <class TA>
class Func_rightshift_return_uint64 : public Func_rightshift
{
public:
int64_t getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Eager<TA, BitOperandGenericShiftAmount> args(row, parm, *this);
return (int64_t) args.a.MariaDBShiftRight(args.b).nullSafeValue(isNull);
}
public:
int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Eager<TA, BitOperandGenericShiftAmount> args(row, parm, *this);
return (int64_t)args.a.MariaDBShiftRight(args.b).nullSafeValue(isNull);
}
};
bool Func_rightshift::fix(execplan::FunctionColumn &col) const
bool Func_rightshift::fix(execplan::FunctionColumn& col) const
{
static Func_rightshift_return_uint64<ParmTUInt64> return_uint64_from_uint64;
static Func_rightshift_return_uint64<ParmTSInt64> return_uint64_from_sint64;
static Func_rightshift_return_uint64<BitOperandGeneric> return_uint64_generic;
return fixForBitShift(col, return_uint64_from_uint64,
return_uint64_from_sint64,
return_uint64_generic);
static Func_rightshift_return_uint64<ParmTUInt64> return_uint64_from_uint64;
static Func_rightshift_return_uint64<ParmTSInt64> return_uint64_from_sint64;
static Func_rightshift_return_uint64<BitOperandGeneric> return_uint64_generic;
return fixForBitShift(col, return_uint64_from_uint64, return_uint64_from_sint64, return_uint64_generic);
}
//
// BIT OR
//
uint64_t Func_bitor::getUintVal(rowgroup::Row& row,
FunctionParm& fp,
bool& isNull,
uint64_t Func_bitor::getUintVal(rowgroup::Row& row, FunctionParm& fp, bool& isNull,
execplan::CalpontSystemCatalog::ColType& op_ct)
{
return static_cast<uint64_t>(getIntVal(row, fp, isNull, op_ct));
return static_cast<uint64_t>(getIntVal(row, fp, isNull, op_ct));
}
template<class TA, class TB>
class Func_bitor_return_uint64: public Func_bitor
template <class TA, class TB>
class Func_bitor_return_uint64 : public Func_bitor
{
public:
int64_t getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Lazy<TA, TB> args(row, parm, *this);
return (int64_t) (args.a | args.b).nullSafeValue(isNull);
}
public:
int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Lazy<TA, TB> args(row, parm, *this);
return (int64_t)(args.a | args.b).nullSafeValue(isNull);
}
};
bool Func_bitor::fix(execplan::FunctionColumn &col) const
bool Func_bitor::fix(execplan::FunctionColumn& col) const
{
static Func_bitor_return_uint64<ParmTUInt64, ParmTUInt64> return_uint64_from_uint64_uint64;
static Func_bitor_return_uint64<ParmTSInt64, ParmTSInt64> return_uint64_from_sint64_sint64;
static Func_bitor_return_uint64<BitOperandGeneric, BitOperandGeneric> return_uint64_generic;
return fixForBitOp2(col, return_uint64_from_uint64_uint64,
return_uint64_from_sint64_sint64,
return_uint64_generic);
static Func_bitor_return_uint64<ParmTUInt64, ParmTUInt64> return_uint64_from_uint64_uint64;
static Func_bitor_return_uint64<ParmTSInt64, ParmTSInt64> return_uint64_from_sint64_sint64;
static Func_bitor_return_uint64<BitOperandGeneric, BitOperandGeneric> return_uint64_generic;
return fixForBitOp2(col, return_uint64_from_uint64_uint64, return_uint64_from_sint64_sint64,
return_uint64_generic);
}
//
// BIT XOR
//
template<class TA, class TB>
class Func_bitxor_return_uint64: public Func_bitxor
template <class TA, class TB>
class Func_bitxor_return_uint64 : public Func_bitxor
{
public:
int64_t getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Eager<TA, TB> args(row, parm, *this);
return (int64_t) (args.a ^ args.b).nullSafeValue(isNull);
}
public:
int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 2);
Arg2Eager<TA, TB> args(row, parm, *this);
return (int64_t)(args.a ^ args.b).nullSafeValue(isNull);
}
};
bool Func_bitxor::fix(execplan::FunctionColumn &col) const
bool Func_bitxor::fix(execplan::FunctionColumn& col) const
{
static Func_bitxor_return_uint64<ParmTUInt64, ParmTUInt64> return_uint64_from_uint64_uint64;
static Func_bitxor_return_uint64<ParmTSInt64, ParmTSInt64> return_uint64_from_sint64_sint64;
static Func_bitxor_return_uint64<BitOperandGeneric, BitOperandGeneric> return_uint64_generic;
return fixForBitOp2(col, return_uint64_from_uint64_uint64,
return_uint64_from_sint64_sint64,
return_uint64_generic);
static Func_bitxor_return_uint64<ParmTUInt64, ParmTUInt64> return_uint64_from_uint64_uint64;
static Func_bitxor_return_uint64<ParmTSInt64, ParmTSInt64> return_uint64_from_sint64_sint64;
static Func_bitxor_return_uint64<BitOperandGeneric, BitOperandGeneric> return_uint64_generic;
return fixForBitOp2(col, return_uint64_from_uint64_uint64, return_uint64_from_sint64_sint64,
return_uint64_generic);
}
//
// BIT COUNT
//
inline int64_t bitCount(uint64_t val)
{
// Refer to Hacker's Delight Chapter 5
// for the bit counting algo used here
val = val - ((val >> 1) & 0x5555555555555555);
val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333);
val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F;
val = val + (val >> 8);
val = val + (val >> 16);
val = val + (val >> 32);
// Refer to Hacker's Delight Chapter 5
// for the bit counting algo used here
val = val - ((val >> 1) & 0x5555555555555555);
val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333);
val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F;
val = val + (val >> 8);
val = val + (val >> 16);
val = val + (val >> 32);
return (int64_t)(val & 0x000000000000007F);
return (int64_t)(val & 0x000000000000007F);
}
template<class TA>
class Func_bit_count_return_uint64: public Func_bit_count
template <class TA>
class Func_bit_count_return_uint64 : public Func_bit_count
{
public:
int64_t getIntVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 1);
return bitCount((uint64_t) TA(row, parm[0], *this).nullSafeValue(isNull));
}
public:
int64_t getIntVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& operationColType) override
{
idbassert(parm.size() == 1);
return bitCount((uint64_t)TA(row, parm[0], *this).nullSafeValue(isNull));
}
};
bool Func_bit_count::fix(execplan::FunctionColumn &col) const
bool Func_bit_count::fix(execplan::FunctionColumn& col) const
{
static Func_bit_count_return_uint64<ParmTUInt64> return_uint64_from_uint64;
static Func_bit_count_return_uint64<ParmTSInt64> return_uint64_from_sint64;
static Func_bit_count_return_uint64<BitOperandGeneric> return_uint64_generic;
if (validateArgCount(col, 1))
return false;
setFunctorByParm(col, col.functionParms()[0],
return_uint64_from_uint64,
return_uint64_from_sint64,
return_uint64_generic);
return validateBitOperandTypeOrError(col, *this, 0);
static Func_bit_count_return_uint64<ParmTUInt64> return_uint64_from_uint64;
static Func_bit_count_return_uint64<ParmTSInt64> return_uint64_from_sint64;
static Func_bit_count_return_uint64<BitOperandGeneric> return_uint64_generic;
if (validateArgCount(col, 1))
return false;
setFunctorByParm(col, col.functionParms()[0], return_uint64_from_uint64, return_uint64_from_sint64,
return_uint64_generic);
return validateBitOperandTypeOrError(col, *this, 0);
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

1087
utils/funcexp/func_case.cpp
View File

File diff suppressed because it is too large Load Diff

2663
utils/funcexp/func_cast.cpp
View File

File diff suppressed because it is too large Load Diff

1021
utils/funcexp/func_ceil.cpp
View File

File diff suppressed because it is too large Load Diff

240
utils/funcexp/func_char.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_char.cpp 3931 2013-06-21 20:17:28Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_char.cpp 3931 2013-06-21 20:17:28Z bwilkinson $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -38,148 +38,144 @@ using namespace rowgroup;
#include "errorids.h"
using namespace logging;
#include <myisampack.h> // min_intXstore()
#include <myisampack.h> // min_intXstore()
#include "vlarray.h"
namespace
{
inline size_t getChar(int32_t num, char*& buf)
{
char tmp[4];
size_t numBytes = 0;
if (num & 0xFF000000L)
{
mi_int4store(tmp, num);
numBytes = 4;
}
else if (num & 0xFF0000L)
{
mi_int3store(tmp, num);
numBytes = 3;
}
else if (num & 0xFF00L)
{
mi_int2store(tmp, num);
numBytes = 2;
}
else
{
*((int8_t*)buf) = num;
++ buf;
return 1;
}
memcpy(buf, tmp, numBytes);
buf += numBytes;
return numBytes;
char tmp[4];
size_t numBytes = 0;
if (num & 0xFF000000L)
{
mi_int4store(tmp, num);
numBytes = 4;
}
else if (num & 0xFF0000L)
{
mi_int3store(tmp, num);
numBytes = 3;
}
else if (num & 0xFF00L)
{
mi_int2store(tmp, num);
numBytes = 2;
}
else
{
*((int8_t*)buf) = num;
++buf;
return 1;
}
memcpy(buf, tmp, numBytes);
buf += numBytes;
return numBytes;
}
}
} // namespace
namespace funcexp
{
CalpontSystemCatalog::ColType Func_char::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType)
CalpontSystemCatalog::ColType Func_char::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
// operation type is not used by this functor
return fp[0]->data()->resultType();
// operation type is not used by this functor
return fp[0]->data()->resultType();
}
string Func_char::getStrVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
string Func_char::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType& ct)
{
const int BUF_SIZE = 4 * parm.size();
utils::VLArray<char, 1024> buf(BUF_SIZE);
buf[0]= 0;
char* pBuf = buf;
CHARSET_INFO* cs = ct.getCharset();
int32_t value = 0;
int32_t numBytes = 0;
for (uint32_t i = 0; i < parm.size(); ++i)
const int BUF_SIZE = 4 * parm.size();
utils::VLArray<char, 1024> buf(BUF_SIZE);
buf[0] = 0;
char* pBuf = buf;
CHARSET_INFO* cs = ct.getCharset();
int32_t value = 0;
int32_t numBytes = 0;
for (uint32_t i = 0; i < parm.size(); ++i)
{
ReturnedColumn* rc = (ReturnedColumn*)parm[i]->data();
switch (rc->resultType().colDataType)
{
ReturnedColumn* rc = (ReturnedColumn*)parm[i]->data();
switch (rc->resultType().colDataType)
{
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
{
value = rc->getIntVal(row, isNull);
}
break;
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
{
value = rc->getIntVal(row, isNull);
}
break;
case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR'
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
{
double vf = std::round(rc->getDoubleVal(row, isNull));
value = (int32_t)vf;
}
break;
case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR'
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
{
double vf = std::round(rc->getDoubleVal(row, isNull));
value = (int32_t)vf;
}
break;
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
value = static_cast<int32_t>(rc->getDecimalVal(row, isNull).toSInt64Round());
}
break;
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
value = static_cast<int32_t>(rc->getDecimalVal(row, isNull).toSInt64Round());
}
break;
case execplan::CalpontSystemCatalog::DATE:
case execplan::CalpontSystemCatalog::DATETIME:
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
continue; // Dates are ignored
}
break;
case execplan::CalpontSystemCatalog::DATE:
case execplan::CalpontSystemCatalog::DATETIME:
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
continue; // Dates are ignored
}
break;
default:
{
value = 0;
}
}
if (isNull)
continue;
numBytes += getChar(value, pBuf);
default:
{
value = 0;
}
}
isNull = false;
/* Check whether we got a well-formed string */
MY_STRCOPY_STATUS status;
int32_t actualBytes = cs->well_formed_char_length(buf, buf + numBytes, numBytes, &status);
if (isNull)
continue;
if (UNLIKELY(actualBytes < numBytes))
{
numBytes = actualBytes;
ostringstream os;
os << "Invalid character string for " << cs->cs_name.str << ": value = " << hex << buf + actualBytes;
logging::Message::Args args;
logging::Message message(9);
args.add(os.str());
logging::LoggingID logid(28); // Shows as PrimProc, which may not be correct in all cases
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_WARNING, message, logid);
// TODO: push warning to client
}
std::string ret(buf, numBytes);
return ret;
numBytes += getChar(value, pBuf);
}
isNull = false;
/* Check whether we got a well-formed string */
MY_STRCOPY_STATUS status;
int32_t actualBytes = cs->well_formed_char_length(buf, buf + numBytes, numBytes, &status);
if (UNLIKELY(actualBytes < numBytes))
{
numBytes = actualBytes;
ostringstream os;
os << "Invalid character string for " << cs->cs_name.str << ": value = " << hex << buf + actualBytes;
logging::Message::Args args;
logging::Message message(9);
args.add(os.str());
logging::LoggingID logid(28); // Shows as PrimProc, which may not be correct in all cases
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_WARNING, message, logid);
// TODO: push warning to client
}
std::string ret(buf, numBytes);
return ret;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

146
utils/funcexp/func_char_length.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_char_length.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_char_length.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -38,89 +38,87 @@ using namespace execplan;
#include "idberrorinfo.h"
#include "errorids.h"
using namespace logging;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_char_length::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_char_length::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return fp[0]->data()->resultType();
return fp[0]->data()->resultType();
}
int64_t Func_char_length::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_char_length::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType;
CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType;
switch (type)
switch (type)
{
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
case execplan::CalpontSystemCatalog::VARCHAR: // including CHAR
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
const string& tstr = parm[0]->data()->getStrVal(row, isNull);
if (isNull)
return 0;
const char* b = tstr.c_str();
const char* e = tstr.c_str() + tstr.length();
return (int64_t)parm[0]->data()->resultType().getCharset()->numchars(b, e);
}
case execplan::CalpontSystemCatalog::DATE:
{
string date = dataconvert::DataConvert::dateToString(parm[0]->data()->getDateIntVal(row, isNull));
return (int64_t)date.size();
}
case execplan::CalpontSystemCatalog::DATETIME:
{
string date = dataconvert::DataConvert::datetimeToString(parm[0]->data()->getDatetimeIntVal(row, isNull));
return (int64_t)date.size();
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
string date = dataconvert::DataConvert::timestampToString(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone());
return (int64_t)date.size();
}
case execplan::CalpontSystemCatalog::TIME:
{
string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull));
return (int64_t)date.size();
}
default:
{
std::ostringstream oss;
oss << "char_length: datatype of " << execplan::colDataTypeToString(type);
throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT);
}
const string& tstr = parm[0]->data()->getStrVal(row, isNull);
if (isNull)
return 0;
const char* b = tstr.c_str();
const char* e = tstr.c_str() + tstr.length();
return (int64_t)parm[0]->data()->resultType().getCharset()->numchars(b, e);
}
return 0;
case execplan::CalpontSystemCatalog::DATE:
{
string date = dataconvert::DataConvert::dateToString(parm[0]->data()->getDateIntVal(row, isNull));
return (int64_t)date.size();
}
case execplan::CalpontSystemCatalog::DATETIME:
{
string date =
dataconvert::DataConvert::datetimeToString(parm[0]->data()->getDatetimeIntVal(row, isNull));
return (int64_t)date.size();
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
string date = dataconvert::DataConvert::timestampToString(
parm[0]->data()->getTimestampIntVal(row, isNull), timeZone());
return (int64_t)date.size();
}
case execplan::CalpontSystemCatalog::TIME:
{
string date = dataconvert::DataConvert::timeToString(parm[0]->data()->getTimeIntVal(row, isNull));
return (int64_t)date.size();
}
default:
{
std::ostringstream oss;
oss << "char_length: datatype of " << execplan::colDataTypeToString(type);
throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT);
}
}
return 0;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

280
utils/funcexp/func_coalesce.cpp
View File

@ -17,10 +17,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_coalesce.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*
****************************************************************************/
* $Id: func_coalesce.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -39,229 +39,209 @@ using namespace dataconvert;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_coalesce::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_coalesce::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_coalesce::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_coalesce::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
int64_t val = 0;
int64_t val = 0;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
val = parm[i]->data()->getIntVal(row, isNull);
if (isNull)
{
val = parm[i]->data()->getIntVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return val;
isNull = false;
continue;
}
isNull = true;
return val;
}
isNull = true;
return val;
}
string Func_coalesce::getStrVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
string Func_coalesce::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
string val;
string val;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
val = parm[i]->data()->getStrVal(row, isNull);
if (isNull)
{
val = parm[i]->data()->getStrVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return val;
isNull = false;
continue;
}
isNull = true;
return "";
return val;
}
isNull = true;
return "";
}
int32_t Func_coalesce::getDateIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int32_t Func_coalesce::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
int64_t val = 0;
int64_t val = 0;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
val = parm[i]->data()->getDateIntVal(row, isNull);
if (isNull)
{
val = parm[i]->data()->getDateIntVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return val;
isNull = false;
continue;
}
isNull = true;
return val;
}
isNull = true;
return val;
}
int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
int64_t Func_coalesce::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
int64_t val = 0;
int64_t val = 0;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
val = parm[i]->data()->getDatetimeIntVal(row, isNull);
if (isNull)
{
val = parm[i]->data()->getDatetimeIntVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return val;
isNull = false;
continue;
}
isNull = true;
return val;
}
isNull = true;
return val;
}
int64_t Func_coalesce::getTimestampIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
int64_t Func_coalesce::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
int64_t val = 0;
int64_t val = 0;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
val = parm[i]->data()->getTimestampIntVal(row, isNull);
if (isNull)
{
val = parm[i]->data()->getTimestampIntVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return val;
isNull = false;
continue;
}
isNull = true;
return val;
}
isNull = true;
return val;
}
int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_coalesce::getTimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
int64_t val = 0;
int64_t val = 0;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
val = parm[i]->data()->getTimeIntVal(row, isNull);
if (isNull)
{
val = parm[i]->data()->getTimeIntVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return val;
isNull = false;
continue;
}
isNull = true;
return val;
}
isNull = true;
return val;
}
double Func_coalesce::getDoubleVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
double Func_coalesce::getDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
execplan::CalpontSystemCatalog::ColType& ct)
{
double d = 0.0;
double d = 0.0;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
d = parm[i]->data()->getDoubleVal(row, isNull);
if (isNull)
{
d = parm[i]->data()->getDoubleVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return d;
isNull = false;
continue;
}
isNull = true;
return d;
}
isNull = true;
return d;
}
long double Func_coalesce::getLongDoubleVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
execplan::CalpontSystemCatalog::ColType& ct)
long double Func_coalesce::getLongDoubleVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
execplan::CalpontSystemCatalog::ColType& ct)
{
long double d = 0.0;
long double d = 0.0;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
d = parm[i]->data()->getLongDoubleVal(row, isNull);
if (isNull)
{
d = parm[i]->data()->getLongDoubleVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return d;
isNull = false;
continue;
}
isNull = true;
return d;
}
isNull = true;
return d;
}
execplan::IDB_Decimal Func_coalesce::getDecimalVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
execplan::CalpontSystemCatalog::ColType& ct)
execplan::IDB_Decimal Func_coalesce::getDecimalVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
execplan::CalpontSystemCatalog::ColType& ct)
{
IDB_Decimal d;
IDB_Decimal d;
for (uint32_t i = 0; i < parm.size(); i++)
for (uint32_t i = 0; i < parm.size(); i++)
{
d = parm[i]->data()->getDecimalVal(row, isNull);
if (isNull)
{
d = parm[i]->data()->getDecimalVal(row, isNull);
if (isNull)
{
isNull = false;
continue;
}
return d;
isNull = false;
continue;
}
isNull = true;
return d;
}
isNull = true;
return d;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

50
utils/funcexp/func_concat.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_concat.cpp 3918 2013-06-19 13:54:34Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_concat.cpp 3918 2013-06-19 13:54:34Z bwilkinson $
*
*
****************************************************************************/
#include <string>
using namespace std;
@ -36,37 +36,33 @@ using namespace dataconvert;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_concat::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_concat::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
// operation type is not used by this functor
return fp[0]->data()->resultType();
// operation type is not used by this functor
return fp[0]->data()->resultType();
}
// Returns the string that results from concatenating the arguments.
// concat() returns NULL if any argument is NULL.
//
string Func_concat::getStrVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType&)
string Func_concat::getStrVal(Row& row, FunctionParm& parm, bool& isNull, CalpontSystemCatalog::ColType&)
{
string ret;
string tmp;
stringValue(parm[0], row, isNull, ret);
// TODO: do a better job of cutting down the number re-allocations.
// look at Item_func_concat::realloc_result for ideas and use
// std::string:resize() appropriatly.
for ( unsigned int id = 1 ; id < parm.size() ; id++)
{
stringValue(parm[id], row, isNull, tmp);
ret.append(tmp);
}
string ret;
string tmp;
stringValue(parm[0], row, isNull, ret);
return ret;
// TODO: do a better job of cutting down the number re-allocations.
// look at Item_func_concat::realloc_result for ideas and use
// std::string:resize() appropriatly.
for (unsigned int id = 1; id < parm.size(); id++)
{
stringValue(parm[id], row, isNull, tmp);
ret.append(tmp);
}
return ret;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

57
utils/funcexp/func_concat_oracle.cpp
View File

@ -30,48 +30,45 @@ using namespace dataconvert;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_concat_oracle::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_concat_oracle::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
// operation type is not used by this functor
return fp[0]->data()->resultType();
// operation type is not used by this functor
return fp[0]->data()->resultType();
}
// Returns the string that results from concatenating the arguments.
// concat_oracle() returns NULL all arguments are NULL.
// single arguments null is replaced by "".
//
string Func_concat_oracle::getStrVal(Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType&)
string Func_concat_oracle::getStrVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType&)
{
string ret;
string tmp;
stringValue(parm[0], row, isNull, ret);
// Oracle Mode should replace NULL with "" unless all values are NULL
string ret;
string tmp;
stringValue(parm[0], row, isNull, ret);
// Oracle Mode should replace NULL with "" unless all values are NULL
if (isNull)
{
ret = "";
isNull = false;
}
// TODO: do a better job of cutting down the number re-allocations.
// look at Item_func_concat::realloc_result for ideas and use
// std::string:resize() appropriatly.
for (unsigned int id = 1; id < parm.size(); id++)
{
stringValue(parm[id], row, isNull, tmp);
if (isNull)
{
ret = "";
isNull = false;
}
// TODO: do a better job of cutting down the number re-allocations.
// look at Item_func_concat::realloc_result for ideas and use
// std::string:resize() appropriatly.
for ( unsigned int id = 1 ; id < parm.size() ; id++)
{
stringValue(parm[id], row, isNull, tmp);
if (isNull)
{
tmp = "";
isNull = false;
}
ret.append(tmp);
tmp = "";
isNull = false;
}
ret.append(tmp);
}
return ret;
return ret;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

77
utils/funcexp/func_concat_ws.cpp
View File

@ -16,42 +16,38 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_concat_ws.cpp 3714 2013-04-18 16:29:25Z bpaul $
*
*
****************************************************************************/
* $Id: func_concat_ws.cpp 3714 2013-04-18 16:29:25Z bpaul $
*
*
****************************************************************************/
#include <string>
using namespace std;
#include "functor_str.h"
#include "functioncolumn.h"
#include "utils_utf8.h" // idb_mbstowcs()
#include "utils_utf8.h" // idb_mbstowcs()
using namespace execplan;
#include "rowgroup.h"
using namespace rowgroup;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_concat_ws::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_concat_ws::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
// operation type is not used by this functor
return fp[0]->data()->resultType();
// operation type is not used by this functor
return fp[0]->data()->resultType();
}
string Func_concat_ws::getStrVal(Row& row,
FunctionParm& parm,
bool& isNull,
string Func_concat_ws::getStrVal(Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& type)
{
string delim;
stringValue(parm[0], row, isNull, delim);
if (isNull)
return "";
string delim;
stringValue(parm[0], row, isNull, delim);
if (isNull)
return "";
// TODO: I don't think we need wide chars here.
// Concatenation works without see Server implementation.
@ -97,33 +93,32 @@ string Func_concat_ws::getStrVal(Row& row,
delete [] wcbuf;
return ret;
#endif
string str;
string tmp;
for ( uint32_t i = 1 ; i < parm.size() ; i++)
string str;
string tmp;
for (uint32_t i = 1; i < parm.size(); i++)
{
stringValue(parm[i], row, isNull, tmp);
if (isNull)
{
stringValue(parm[i], row, isNull, tmp);
if (isNull)
{
isNull = false;
continue;
}
if (!str.empty())
str += delim;
// TODO: Work on string reallocation. Use std::string::resize() to
// grab larger chunks in some intellegent manner.
str += tmp;
isNull = false;
continue;
}
if (str.empty())
isNull = true;
else
isNull = false;
if (!str.empty())
str += delim;
return str;
// TODO: Work on string reallocation. Use std::string::resize() to
// grab larger chunks in some intellegent manner.
str += tmp;
}
if (str.empty())
isNull = true;
else
isNull = false;
return str;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

344
utils/funcexp/func_conv.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_conv.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_conv.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -38,92 +38,93 @@ namespace
{
int64_t convStrToNum(const string& str, int base, bool unsignedFlag)
{
int negative;
uint64_t cutoff, cutlim, i, j, save;
int overflow;
int negative;
uint64_t cutoff, cutlim, i, j, save;
int overflow;
// to skip the leading spaces.
for (i = 0; i < str.length() && str.c_str()[i] == ' '; i++)
{}
// to skip the leading spaces.
for (i = 0; i < str.length() && str.c_str()[i] == ' '; i++)
{
}
if (i == str.length())
{
return 0L;
}
if (i == str.length())
{
return 0L;
}
if (str.c_str()[i] == '-')
{
negative = 1;
++i;
}
else if (str.c_str()[i] == '+')
{
negative = 0;
++i;
}
if (str.c_str()[i] == '-')
{
negative = 1;
++i;
}
else if (str.c_str()[i] == '+')
{
negative = 0;
++i;
}
else
negative = 0;
save = i;
cutoff = (~(uint64_t)0) / (uint64_t)base;
cutlim = (uint32_t)((~(uint64_t)0) % (uint64_t)base);
overflow = 0;
j = 0;
for (; i < str.length(); i++)
{
unsigned char c = str.c_str()[i];
if (c >= '0' && c <= '9')
c -= '0';
else if (c >= 'A' && c <= 'Z')
c = c - 'A' + 10;
else if (c >= 'a' && c <= 'z')
c = c - 'a' + 10;
else
negative = 0;
break;
save = i;
if (c >= base)
break;
cutoff = (~(uint64_t) 0) / (uint64_t) base;
cutlim = (uint32_t) ((~(uint64_t) 0) % (uint64_t) base);
overflow = 0;
j = 0;
for (; i < str.length(); i++)
if (j > cutoff || (j == cutoff && c > cutlim))
overflow = 1;
else
{
unsigned char c = str.c_str()[i];
if (c >= '0' && c <= '9')
c -= '0';
else if (c >= 'A' && c <= 'Z')
c = c - 'A' + 10;
else if (c >= 'a' && c <= 'z')
c = c - 'a' + 10;
else
break;
if (c >= base)
break;
if (j > cutoff || (j == cutoff && c > cutlim))
overflow = 1;
else
{
j *= (uint64_t) base;
j += c;
}
j *= (uint64_t)base;
j += c;
}
}
if (i == save)
return 0L;
if (i == save)
return 0L;
if (!unsignedFlag)
if (!unsignedFlag)
{
if (negative)
{
if (negative)
{
if (j > (uint64_t) numeric_limits<int64_t>::min())
overflow = 1;
}
else if (j > (uint64_t) numeric_limits<int64_t>::max())
{
overflow = 1;
}
if (j > (uint64_t)numeric_limits<int64_t>::min())
overflow = 1;
}
if (overflow)
else if (j > (uint64_t)numeric_limits<int64_t>::max())
{
if (unsignedFlag)
return (~(uint64_t) 0);
return negative ? numeric_limits<int64_t>::min() : numeric_limits<int64_t>::max();
overflow = 1;
}
}
return (negative ? -((int64_t) j) : (int64_t) j);
}
if (overflow)
{
if (unsignedFlag)
return (~(uint64_t)0);
return negative ? numeric_limits<int64_t>::min() : numeric_limits<int64_t>::max();
}
return (negative ? -((int64_t)j) : (int64_t)j);
}
} // namespace
namespace funcexp
{
@ -131,61 +132,61 @@ namespace helpers
{
const char* convNumToStr(int64_t val, char* dst, int radix)
{
if (radix == 16 || radix == -16)
sprintf(dst, "%llX", (long long)val);
if (radix == 16 || radix == -16)
sprintf(dst, "%llX", (long long)val);
else if (radix == 8 || radix == -8)
sprintf(dst, "%llo", (long long)val);
else if (radix == 8 || radix == -8)
sprintf(dst, "%llo", (long long)val);
else if (radix == 10)
else if (radix == 10)
{
sprintf(dst, "%llu", (unsigned long long)val);
}
else if (radix == -10)
sprintf(dst, "%lld", (long long)val);
else if (radix == 2 || radix == -2)
{
char tmp[65];
char* ptr = &tmp[64];
*ptr-- = 0;
for (int i = 0; i < 64; i++)
{
sprintf(dst, "%llu", (unsigned long long)val);
}
else if (radix == -10)
sprintf(dst, "%lld", (long long)val);
if (val & 1)
*ptr-- = '1';
else
*ptr-- = '0';
else if (radix == 2 || radix == -2)
val >>= 1;
}
ptr = strchr(tmp, '1');
if (ptr == 0)
strcpy(dst, &tmp[63]);
else
strcpy(dst, ptr);
}
else if (radix == 4 || radix == -4)
{
char tmp[33];
char* ptr = &tmp[32];
*ptr-- = 0;
for (int i = 0; i < 32; i++)
{
char tmp[65];
char* ptr = &tmp[64];
*ptr-- = 0;
for (int i = 0; i < 64; i++)
{
if (val & 1)
*ptr-- = '1';
else
*ptr-- = '0';
val >>= 1;
}
ptr = strchr(tmp, '1');
if (ptr == 0)
strcpy(dst, &tmp[63]);
else
strcpy(dst, ptr);
*ptr-- = '0' + (val & 3);
val >>= 2;
}
else if (radix == 4 || radix == -4)
{
char tmp[33];
char* ptr = &tmp[32];
*ptr-- = 0;
for (int i = 0; i < 32; i++)
{
*ptr-- = '0' + (val & 3);
val >>= 2;
}
ptr = strpbrk(tmp, "123");
ptr = strpbrk(tmp, "123");
if (ptr == 0)
strcpy(dst, &tmp[31]);
else
strcpy(dst, ptr);
}
if (ptr == 0)
strcpy(dst, &tmp[31]);
else
strcpy(dst, ptr);
}
#if 0
else if (radix == 8 || radix == -8)
@ -234,76 +235,73 @@ const char* convNumToStr(int64_t val, char* dst, int radix)
}
#endif
else if (radix == 32 || radix == -32)
else if (radix == 32 || radix == -32)
{
char tmp[14];
char* ptr = &tmp[13];
*ptr-- = 0;
for (int i = 0; i < 13; i++)
{
char tmp[14];
char* ptr = &tmp[13];
*ptr-- = 0;
int v = val & 0x1f;
for (int i = 0; i < 13; i++)
{
int v = val & 0x1f;
if (v > 9)
*ptr-- = 'A' + v - 10;
else
*ptr-- = '0' + v;
if (v > 9)
*ptr-- = 'A' + v - 10;
else
*ptr-- = '0' + v;
val >>= 5;
}
ptr = strpbrk(tmp, "123456789ABCDEFGHIJKLMNOPQRSTUV");
if (ptr == 0)
strcpy(dst, &tmp[12]);
else
strcpy(dst, ptr);
val >>= 5;
}
ptr = strpbrk(tmp, "123456789ABCDEFGHIJKLMNOPQRSTUV");
if (ptr == 0)
strcpy(dst, &tmp[12]);
else
*dst = 0;
strcpy(dst, ptr);
}
else
*dst = 0;
return dst;
return dst;
}
} //namespace funcexp::helpers
} // namespace helpers
CalpontSystemCatalog::ColType Func_conv::operationType(FunctionParm& fp, CalpontSystemCatalog::ColType& resultType)
CalpontSystemCatalog::ColType Func_conv::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
// operation type is not used by this functor
return fp[0]->data()->resultType();
// operation type is not used by this functor
return fp[0]->data()->resultType();
}
string Func_conv::getStrVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
string Func_conv::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
const string& res = parm[0]->data()->getStrVal(row, isNull);
string str;
char ans[65];
int64_t dec;
int64_t from_base = parm[1]->data()->getIntVal(row, isNull);
int64_t to_base = parm[2]->data()->getIntVal(row, isNull);
const string& res = parm[0]->data()->getStrVal(row, isNull);
string str;
char ans[65];
int64_t dec;
int64_t from_base = parm[1]->data()->getIntVal(row, isNull);
int64_t to_base = parm[2]->data()->getIntVal(row, isNull);
if (isNull || abs(static_cast<int>(to_base)) > 36 || abs(static_cast<int>(to_base)) < 2 ||
abs(static_cast<int>(from_base)) > 36 || abs(static_cast<int>(from_base)) < 2 || !(res.length()))
{
isNull = true;
return "";
}
if (isNull || abs(static_cast<int>(to_base)) > 36 || abs(static_cast<int>(to_base)) < 2 ||
abs(static_cast<int>(from_base)) > 36 || abs(static_cast<int>(from_base)) < 2 || !(res.length()))
{
isNull = true;
return "";
}
if (from_base < 0)
dec = convStrToNum(res, -from_base, false);
else
dec = (int64_t) convStrToNum( res, from_base, true);
if (from_base < 0)
dec = convStrToNum(res, -from_base, false);
else
dec = (int64_t)convStrToNum(res, from_base, true);
str = helpers::convNumToStr(dec, ans, to_base);
str = helpers::convNumToStr(dec, ans, to_base);
isNull = str.empty();
isNull = str.empty();
return str;
return str;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

311
utils/funcexp/func_convert_tz.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <cstdlib>
#include <string>
using namespace std;
@ -41,192 +40,180 @@ using namespace logging;
namespace funcexp
{
// need my_tz_find in dataconvert
CalpontSystemCatalog::ColType Func_convert_tz::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_convert_tz::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_convert_tz::getDatetimeIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
execplan::CalpontSystemCatalog::ColType& ct)
int64_t Func_convert_tz::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
execplan::CalpontSystemCatalog::ColType& ct)
{
return dataconvert::DataConvert::intToDatetime(getIntVal(row, parm, isNull, ct));
return dataconvert::DataConvert::intToDatetime(getIntVal(row, parm, isNull, ct));
}
int64_t Func_convert_tz::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
int64_t Func_convert_tz::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
messageqcpp::ByteStream bs;
bool bFromTz = false;
bool bToTz = false;
dataconvert::TIME_ZONE_INFO tzinfo;
std::vector<int64_t> ats;
std::vector<unsigned char> types;
std::vector<TRAN_TYPE_INFO> ttis;
messageqcpp::ByteStream bs;
bool bFromTz = false;
bool bToTz = false;
dataconvert::TIME_ZONE_INFO tzinfo;
std::vector<int64_t> ats;
std::vector<unsigned char> types;
std::vector<TRAN_TYPE_INFO> ttis;
#ifdef ABBR_ARE_USED
std::vector<char> chars;
std::vector<char> chars;
#endif
std::vector<LS_INFO> lsis;
std::vector<int64_t> revts;
std::vector<REVT_INFO> revtis;
std::vector<TRAN_TYPE_INFO> fallback_tti;
std::vector<LS_INFO> lsis;
std::vector<int64_t> revts;
std::vector<REVT_INFO> revtis;
std::vector<TRAN_TYPE_INFO> fallback_tti;
int64_t seconds = 0;
uint32_t err_code = 0;
int64_t datetime_start = parm[0]->data()->getDatetimeIntVal(row, isNull);
int64_t rtn = 0;
int64_t seconds = 0;
uint32_t err_code = 0;
int64_t datetime_start = parm[0]->data()->getDatetimeIntVal(row, isNull);
int64_t rtn = 0;
if (isNull)
return 0;
// Adding a zero date to a time is always NULL
if (datetime_start == 0)
if (isNull)
return 0;
// Adding a zero date to a time is always NULL
if (datetime_start == 0)
{
isNull = true;
return -1;
}
const string& from_tz = parm[1]->data()->getStrVal(row, isNull);
if (isNull)
{
return 0;
}
const string& to_tz = parm[2]->data()->getStrVal(row, isNull);
if (isNull)
{
return 0;
}
cout << "from " << from_tz << endl;
cout << "to " << to_tz << endl;
const string& serialized_from_tzinfo = parm[3]->data()->getStrVal(row, isNull);
const string& serialized_to_tzinfo = parm[4]->data()->getStrVal(row, isNull);
if (!serialized_from_tzinfo.empty())
{
bs.append((uint8_t*)serialized_from_tzinfo.c_str(), serialized_from_tzinfo.length());
dataconvert::unserializeTimezoneInfo(bs, &tzinfo);
deserializeInlineVector<int64_t>(bs, ats);
tzinfo.ats = ats.data();
deserializeInlineVector<unsigned char>(bs, types);
tzinfo.types = types.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, ttis);
tzinfo.ttis = ttis.data();
#ifdef ABBR_ARE_USED
deserializeInlineVector<char>(bs_fromTZ, chars);
tzinfo.chars = chars.data();
#endif
deserializeInlineVector<LS_INFO>(bs, lsis);
tzinfo.lsis = lsis.data();
deserializeInlineVector<int64_t>(bs, revts);
tzinfo.revts = revts.data();
deserializeInlineVector<REVT_INFO>(bs, revtis);
tzinfo.revtis = revtis.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, fallback_tti);
tzinfo.fallback_tti = fallback_tti.data();
bFromTz = true;
}
dataconvert::DateTime datetime(datetime_start);
if (!dataconvert::DataConvert::isColumnDateTimeValid(datetime_start))
return datetime_start;
// int64_t result_datetime;
bool valid = true;
MySQLTime my_time_tmp, my_start_time;
my_time_tmp.reset();
my_start_time.year = datetime.year;
my_start_time.month = datetime.month;
my_start_time.day = datetime.day;
my_start_time.hour = datetime.hour;
my_start_time.minute = datetime.minute;
my_start_time.second = datetime.second;
my_start_time.second_part = datetime.msecond;
if (bFromTz)
{
seconds = dataconvert::TIME_to_gmt_sec(my_start_time, &tzinfo, &err_code);
if (err_code)
{
return datetime.convertToMySQLint();
}
}
else
{
seconds = dataconvert::mySQLTimeToGmtSec(my_start_time, from_tz, valid);
if (!valid)
{
if (seconds != 0)
isNull = true;
return -1;
return datetime.convertToMySQLint();
}
}
const string& from_tz = parm[1]->data()->getStrVal(row, isNull);
if (isNull)
{
return 0;
}
const string& to_tz = parm[2]->data()->getStrVal(row, isNull);
if (isNull)
{
return 0;
}
cout << "from " << from_tz << endl;
cout << "to " << to_tz << endl;
const string& serialized_from_tzinfo = parm[3]->data()->getStrVal(row, isNull);
const string& serialized_to_tzinfo = parm[4]->data()->getStrVal(row, isNull);
if (!serialized_from_tzinfo.empty())
{
bs.append((uint8_t*)serialized_from_tzinfo.c_str(),serialized_from_tzinfo.length());
dataconvert::unserializeTimezoneInfo(bs,&tzinfo);
deserializeInlineVector<int64_t>(bs, ats);
tzinfo.ats = ats.data();
deserializeInlineVector<unsigned char>(bs, types);
tzinfo.types = types.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, ttis);
tzinfo.ttis = ttis.data();
if (!serialized_to_tzinfo.empty())
{
bs.reset();
bs.append((uint8_t*)serialized_to_tzinfo.c_str(), serialized_to_tzinfo.length());
dataconvert::unserializeTimezoneInfo(bs, &tzinfo);
deserializeInlineVector<int64_t>(bs, ats);
tzinfo.ats = ats.data();
deserializeInlineVector<unsigned char>(bs, types);
tzinfo.types = types.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, ttis);
tzinfo.ttis = ttis.data();
#ifdef ABBR_ARE_USED
deserializeInlineVector<char>(bs_fromTZ, chars);
tzinfo.chars = chars.data();
deserializeInlineVector<char>(bs_fromTZ, chars);
tzinfo.chars = chars.data();
#endif
deserializeInlineVector<LS_INFO>(bs, lsis);
tzinfo.lsis = lsis.data();
deserializeInlineVector<int64_t>(bs, revts);
tzinfo.revts = revts.data();
deserializeInlineVector<REVT_INFO>(bs, revtis);
tzinfo.revtis = revtis.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, fallback_tti);
tzinfo.fallback_tti = fallback_tti.data();
bFromTz = true;
}
deserializeInlineVector<LS_INFO>(bs, lsis);
tzinfo.lsis = lsis.data();
deserializeInlineVector<int64_t>(bs, revts);
tzinfo.revts = revts.data();
deserializeInlineVector<REVT_INFO>(bs, revtis);
tzinfo.revtis = revtis.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, fallback_tti);
tzinfo.fallback_tti = fallback_tti.data();
bToTz = true;
}
if (bToTz)
{
dataconvert::gmt_sec_to_TIME(&my_time_tmp, seconds, &tzinfo);
if (my_time_tmp.second == 60 || my_time_tmp.second == 61)
my_time_tmp.second = 59;
}
else
{
dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz);
}
dataconvert::DateTime datetime(datetime_start);
if (!dataconvert::DataConvert::isColumnDateTimeValid( datetime_start ))
return datetime_start;
dataconvert::DateTime result_datetime(my_time_tmp.year, my_time_tmp.month, my_time_tmp.day,
my_time_tmp.hour, my_time_tmp.minute, my_time_tmp.second,
my_time_tmp.second_part);
//int64_t result_datetime;
bool valid = true;
MySQLTime my_time_tmp,my_start_time;
my_time_tmp.reset();
my_start_time.year = datetime.year;
my_start_time.month = datetime.month;
my_start_time.day = datetime.day;
my_start_time.hour = datetime.hour;
my_start_time.minute = datetime.minute;
my_start_time.second = datetime.second;
my_start_time.second_part = datetime.msecond;
if ((rtn = result_datetime.convertToMySQLint()) == 0)
isNull = true;
if (bFromTz)
{
seconds = dataconvert::TIME_to_gmt_sec(my_start_time,&tzinfo,&err_code);
if (err_code)
{
return datetime.convertToMySQLint();
}
}
else
{
seconds = dataconvert::mySQLTimeToGmtSec(my_start_time,from_tz,valid);
if (!valid)
{
if (seconds != 0)
isNull = true;
return datetime.convertToMySQLint();
}
}
if (!serialized_to_tzinfo.empty())
{
bs.reset();
bs.append((uint8_t*)serialized_to_tzinfo.c_str(),serialized_to_tzinfo.length());
dataconvert::unserializeTimezoneInfo(bs,&tzinfo);
deserializeInlineVector<int64_t>(bs, ats);
tzinfo.ats = ats.data();
deserializeInlineVector<unsigned char>(bs, types);
tzinfo.types = types.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, ttis);
tzinfo.ttis = ttis.data();
#ifdef ABBR_ARE_USED
deserializeInlineVector<char>(bs_fromTZ, chars);
tzinfo.chars = chars.data();
#endif
deserializeInlineVector<LS_INFO>(bs, lsis);
tzinfo.lsis = lsis.data();
deserializeInlineVector<int64_t>(bs, revts);
tzinfo.revts = revts.data();
deserializeInlineVector<REVT_INFO>(bs, revtis);
tzinfo.revtis = revtis.data();
deserializeInlineVector<TRAN_TYPE_INFO>(bs, fallback_tti);
tzinfo.fallback_tti = fallback_tti.data();
bToTz = true;
}
if (bToTz)
{
dataconvert::gmt_sec_to_TIME(&my_time_tmp, seconds, &tzinfo);
if (my_time_tmp.second == 60 || my_time_tmp.second == 61)
my_time_tmp.second= 59;
}
else
{
dataconvert::gmtSecToMySQLTime(seconds, my_time_tmp, to_tz);
}
dataconvert::DateTime result_datetime(my_time_tmp.year,
my_time_tmp.month,
my_time_tmp.day,
my_time_tmp.hour,
my_time_tmp.minute,
my_time_tmp.second,
my_time_tmp.second_part);
if ((rtn = result_datetime.convertToMySQLint()) == 0)
isNull = true;
return rtn;
return rtn;
}
string Func_convert_tz::getStrVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
string Func_convert_tz::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct));
return dataconvert::DataConvert::datetimeToString(getIntVal(row, parm, isNull, ct));
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

49
utils/funcexp/func_crc32.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_crc32.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_crc32.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
#include <unistd.h>
#include <cstdlib>
@ -38,37 +38,30 @@ using namespace execplan;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_crc32::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_crc32::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_crc32::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_crc32::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
unsigned crc;
switch (parm.size()) {
default:
isNull = true;
return 0;
case 1:
crc = 0;
break;
unsigned crc;
switch (parm.size())
{
default: isNull = true; return 0;
case 1: crc = 0; break;
case 2:
crc = static_cast<unsigned>(parm[0]->data()->getIntVal(row, isNull));
if (isNull)
return 0;
}
const string& b = parm[parm.size() - 1]->data()->getStrVal(row, isNull);
if (isNull)
crc = static_cast<unsigned>(parm[0]->data()->getIntVal(row, isNull));
if (isNull)
return 0;
return crc32(crc, reinterpret_cast<const uint8_t*>(b.data()), b.size());
}
const string& b = parm[parm.size() - 1]->data()->getStrVal(row, isNull);
if (isNull)
return 0;
return crc32(crc, reinterpret_cast<const uint8_t*>(b.data()), b.size());
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

197
utils/funcexp/func_date.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_date.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_date.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -38,122 +38,115 @@ using namespace execplan;
#include "errorids.h"
using namespace logging;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_date::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_date::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_date::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_date::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType&)
{
CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType;
CalpontSystemCatalog::ColDataType type = parm[0]->data()->resultType().colDataType;
string value = "";
string value = "";
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
switch (type)
switch (type)
{
case execplan::CalpontSystemCatalog::DATE:
{
case execplan::CalpontSystemCatalog::DATE:
{
return parm[0]->data()->getDatetimeIntVal(row, isNull);
break;
}
case execplan::CalpontSystemCatalog::DATETIME:
{
int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull);
value = dataconvert::DataConvert::datetimeToString(val1);
value = value.substr(0, 10);
break;
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull);
value = dataconvert::DataConvert::timestampToString(val1, timeZone());
value = value.substr(0, 10);
break;
}
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
{
int64_t val;
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
aDateTime.hour = 0;
aDateTime.minute = 0;
aDateTime.second = 0;
aDateTime.msecond = 0;
if ((aTime.hour < 0) || (aTime.is_neg))
{
aTime.hour = -abs(aTime.hour);
aTime.minute = -abs(aTime.minute);
aTime.second = -abs(aTime.second);
aTime.msecond = -abs(aTime.msecond);
}
val = addTime(aDateTime, aTime);
value = dataconvert::DataConvert::datetimeToString(val);
value = value.substr(0, 10);
break;
}
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
case execplan::CalpontSystemCatalog::VARCHAR:
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
isNull = true;
return 0;
}
break;
default:
{
std::ostringstream oss;
oss << "date: datatype of " << execplan::colDataTypeToString(type);
throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT);
}
return parm[0]->data()->getDatetimeIntVal(row, isNull);
break;
}
return dataconvert::DataConvert::datetimeToInt(value);
case execplan::CalpontSystemCatalog::DATETIME:
{
int64_t val1 = parm[0]->data()->getDatetimeIntVal(row, isNull);
value = dataconvert::DataConvert::datetimeToString(val1);
value = value.substr(0, 10);
break;
}
case execplan::CalpontSystemCatalog::TIMESTAMP:
{
int64_t val1 = parm[0]->data()->getTimestampIntVal(row, isNull);
value = dataconvert::DataConvert::timestampToString(val1, timeZone());
value = value.substr(0, 10);
break;
}
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
{
int64_t val;
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
aDateTime.hour = 0;
aDateTime.minute = 0;
aDateTime.second = 0;
aDateTime.msecond = 0;
if ((aTime.hour < 0) || (aTime.is_neg))
{
aTime.hour = -abs(aTime.hour);
aTime.minute = -abs(aTime.minute);
aTime.second = -abs(aTime.second);
aTime.msecond = -abs(aTime.msecond);
}
val = addTime(aDateTime, aTime);
value = dataconvert::DataConvert::datetimeToString(val);
value = value.substr(0, 10);
break;
}
case execplan::CalpontSystemCatalog::BIGINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::TINYINT:
case execplan::CalpontSystemCatalog::SMALLINT:
case execplan::CalpontSystemCatalog::UBIGINT:
case execplan::CalpontSystemCatalog::UINT:
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UTINYINT:
case execplan::CalpontSystemCatalog::USMALLINT:
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
case execplan::CalpontSystemCatalog::VARCHAR:
case execplan::CalpontSystemCatalog::CHAR:
case execplan::CalpontSystemCatalog::TEXT:
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
isNull = true;
return 0;
}
break;
default:
{
std::ostringstream oss;
oss << "date: datatype of " << execplan::colDataTypeToString(type);
throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT);
}
}
return dataconvert::DataConvert::datetimeToInt(value);
}
string Func_date::getStrVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
string Func_date::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType&)
{
const string& val = parm[0]->data()->getStrVal(row, isNull);
const string& val = parm[0]->data()->getStrVal(row, isNull);
return val.substr(0, 10);
return val.substr(0, 10);
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

1445
utils/funcexp/func_date_add.cpp
View File

File diff suppressed because it is too large Load Diff

658
utils/funcexp/func_date_format.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_date_format.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
* $Id: func_date_format.cpp 3923 2013-06-19 21:43:06Z bwilkinson $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -40,394 +40,380 @@ namespace helpers
{
const string IDB_date_format(const DateTime& dt, const string& format)
{
// assume 256 is enough. assume not allowing incomplete date
char buf[256];
char* ptr = buf;
uint32_t weekday = 0;
uint32_t dayval = 0;
uint32_t weekval = 0;
uint32_t weekyear = 0;
// assume 256 is enough. assume not allowing incomplete date
char buf[256];
char* ptr = buf;
uint32_t weekday = 0;
uint32_t dayval = 0;
uint32_t weekval = 0;
uint32_t weekyear = 0;
for (uint32_t i = 0; i < format.length(); i++)
for (uint32_t i = 0; i < format.length(); i++)
{
if (format[i] != '%')
*ptr++ = format[i];
else
{
if (format[i] != '%')
*ptr++ = format[i];
else
{
i++;
i++;
switch (format[i])
{
case 'M':
sprintf(ptr, "%s", helpers::monthFullNames[dt.month].c_str());
ptr += helpers::monthFullNames[dt.month].length();
break;
switch (format[i])
{
case 'M':
sprintf(ptr, "%s", helpers::monthFullNames[dt.month].c_str());
ptr += helpers::monthFullNames[dt.month].length();
break;
case 'b':
sprintf(ptr, "%s", helpers::monthAbNames[dt.month].c_str());
ptr += helpers::monthAbNames[dt.month].length();
break;
case 'b':
sprintf(ptr, "%s", helpers::monthAbNames[dt.month].c_str());
ptr += helpers::monthAbNames[dt.month].length();
break;
case 'W':
weekday = helpers::calc_mysql_weekday( dt.year, dt.month, dt.day, false);
sprintf(ptr, "%s", helpers::weekdayFullNames[weekday].c_str());
ptr += helpers::weekdayFullNames[weekday].length();
break;
case 'W':
weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false);
sprintf(ptr, "%s", helpers::weekdayFullNames[weekday].c_str());
ptr += helpers::weekdayFullNames[weekday].length();
break;
case 'w':
weekday = helpers::calc_mysql_weekday( dt.year, dt.month, dt.day, true);
sprintf(ptr, "%01d", weekday);
ptr += 1;
break;
case 'w':
weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, true);
sprintf(ptr, "%01d", weekday);
ptr += 1;
break;
case 'a':
weekday = helpers::calc_mysql_weekday( dt.year, dt.month, dt.day, false);
sprintf(ptr, "%s", helpers::weekdayAbNames[weekday].c_str());
ptr += helpers::weekdayAbNames[weekday].length();
break;
case 'a':
weekday = helpers::calc_mysql_weekday(dt.year, dt.month, dt.day, false);
sprintf(ptr, "%s", helpers::weekdayAbNames[weekday].c_str());
ptr += helpers::weekdayAbNames[weekday].length();
break;
case 'D':
sprintf(ptr, "%s", helpers::dayOfMonth[dt.day].c_str());
ptr += helpers::dayOfMonth[dt.day].length();
break;
case 'D':
sprintf(ptr, "%s", helpers::dayOfMonth[dt.day].c_str());
ptr += helpers::dayOfMonth[dt.day].length();
break;
case 'Y':
sprintf(ptr, "%04d", dt.year);
ptr += 4;
break;
case 'Y':
sprintf(ptr, "%04d", dt.year);
ptr += 4;
break;
case 'y':
sprintf(ptr, "%02d", dt.year % 100);
ptr += 2;
break;
case 'y':
sprintf(ptr, "%02d", dt.year % 100);
ptr += 2;
break;
case 'm':
sprintf(ptr, "%02d", dt.month);
ptr += 2;
break;
case 'm':
sprintf(ptr, "%02d", dt.month);
ptr += 2;
break;
case 'c':
sprintf(ptr, "%d", dt.month);
ptr = ptr + (dt.month >= 10 ? 2 : 1);
break;
case 'c':
sprintf(ptr, "%d", dt.month);
ptr = ptr + (dt.month >= 10 ? 2 : 1);
break;
case 'd':
sprintf(ptr, "%02d", dt.day);
ptr += 2;
break;
case 'd':
sprintf(ptr, "%02d", dt.day);
ptr += 2;
break;
case 'e':
sprintf(ptr, "%d", dt.day);
ptr = ptr + (dt.day >= 10 ? 2 : 1);
break;
case 'e':
sprintf(ptr, "%d", dt.day);
ptr = ptr + (dt.day >= 10 ? 2 : 1);
break;
case 'f':
sprintf(ptr, "%06d", dt.msecond);
ptr += 6;
break;
case 'f':
sprintf(ptr, "%06d", dt.msecond);
ptr += 6;
break;
case 'H':
sprintf(ptr, "%02d", dt.hour);
ptr += 2;
break;
case 'H':
sprintf(ptr, "%02d", dt.hour);
ptr += 2;
break;
case 'h':
case 'I':
sprintf(ptr, "%02d", (dt.hour % 24 + 11) % 12 + 1);
ptr += 2;
break;
case 'h':
case 'I':
sprintf(ptr, "%02d", (dt.hour % 24 + 11) % 12 + 1);
ptr += 2;
break;
case 'i': /* minutes */
sprintf(ptr, "%02d", dt.minute);
ptr += 2;
break;
case 'i': /* minutes */
sprintf(ptr, "%02d", dt.minute);
ptr += 2;
break;
case 'j':
dayval = helpers::calc_mysql_daynr( dt.year, dt.month, dt.day ) -
helpers::calc_mysql_daynr( dt.year, 1, 1 ) + 1;
sprintf(ptr, "%03d", dayval);
ptr += 3;
break;
case 'j':
dayval = helpers::calc_mysql_daynr(dt.year, dt.month, dt.day) -
helpers::calc_mysql_daynr(dt.year, 1, 1) + 1;
sprintf(ptr, "%03d", dayval);
ptr += 3;
break;
case 'k':
sprintf(ptr, "%d", dt.hour);
ptr += (dt.hour >= 10 ? 2 : 1);
break;
case 'k':
sprintf(ptr, "%d", dt.hour);
ptr += (dt.hour >= 10 ? 2 : 1);
break;
case 'l':
sprintf(ptr, "%d", (dt.hour % 24 + 11) % 12 + 1);
ptr += ((dt.hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1);
break;
case 'l':
sprintf(ptr, "%d", (dt.hour % 24 + 11) % 12 + 1);
ptr += ((dt.hour % 24 + 11) % 12 + 1 >= 10 ? 2 : 1);
break;
case 'p':
sprintf(ptr, "%s", (dt.hour % 24 < 12 ? "AM" : "PM"));
ptr += 2;
break;
case 'p':
sprintf(ptr, "%s", (dt.hour % 24 < 12 ? "AM" : "PM"));
ptr += 2;
break;
case 'r':
sprintf(ptr, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"),
(dt.hour + 11) % 12 + 1, dt.minute, dt.second);
ptr += 11;
break;
case 'r':
sprintf(ptr, (dt.hour % 24 < 12 ? "%02d:%02d:%02d AM" : "%02d:%02d:%02d PM"),
(dt.hour + 11) % 12 + 1, dt.minute, dt.second);
ptr += 11;
break;
case 'S':
case 's':
sprintf(ptr, "%02d", dt.second);
ptr += 2;
break;
case 'S':
case 's':
sprintf(ptr, "%02d", dt.second);
ptr += 2;
break;
case 'T':
sprintf (ptr, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second);
ptr += 8;
break;
case 'T':
sprintf(ptr, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second);
ptr += 8;
break;
case 'U':
weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day, 0);
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'U':
weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, 0);
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'V':
weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day,
helpers::WEEK_NO_ZERO );
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'V':
weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO);
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'u':
weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day,
helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS);
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'u':
weekval = helpers::calc_mysql_week(dt.year, dt.month, dt.day,
helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS);
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'v':
weekval = helpers::calc_mysql_week( dt.year, dt.month, dt.day,
helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS);
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'v':
weekval = helpers::calc_mysql_week(
dt.year, dt.month, dt.day,
helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS);
sprintf(ptr, "%02d", weekval);
ptr += 2;
break;
case 'x':
helpers::calc_mysql_week( dt.year, dt.month, dt.day,
helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS, &weekyear);
sprintf(ptr, "%04d", weekyear);
ptr += 4;
break;
case 'x':
helpers::calc_mysql_week(
dt.year, dt.month, dt.day,
helpers::WEEK_NO_ZERO | helpers::WEEK_MONDAY_FIRST | helpers::WEEK_GT_THREE_DAYS, &weekyear);
sprintf(ptr, "%04d", weekyear);
ptr += 4;
break;
case 'X':
helpers::calc_mysql_week( dt.year, dt.month, dt.day,
helpers::WEEK_NO_ZERO, &weekyear);
sprintf(ptr, "%04d", weekyear);
ptr += 4;
break;
case 'X':
helpers::calc_mysql_week(dt.year, dt.month, dt.day, helpers::WEEK_NO_ZERO, &weekyear);
sprintf(ptr, "%04d", weekyear);
ptr += 4;
break;
default:
*ptr++ = format[i];
}
}
default: *ptr++ = format[i];
}
}
}
*ptr = 0;
return string(buf);
}
*ptr = 0;
return string(buf);
}
} // namespace helpers
CalpontSystemCatalog::ColType Func_date_format::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_date_format::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::VARCHAR;
ct.colWidth = 255;
return ct;
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::VARCHAR;
ct.colWidth = 255;
return ct;
}
string Func_date_format::getStrVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
string Func_date_format::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType&)
{
int64_t val = 0;
DateTime dt = 0;
int64_t val = 0;
DateTime dt = 0;
switch (parm[0]->data()->resultType().colDataType)
switch (parm[0]->data()->resultType().colDataType)
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
dt.year = (uint32_t)((val >> 16) & 0xffff);
dt.month = (uint32_t)((val >> 12) & 0xf);
dt.day = (uint32_t)((val >> 6) & 0x3f);
break;
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getDatetimeIntVal(row, isNull);
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
break;
case CalpontSystemCatalog::TIMESTAMP:
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
dt.year = (uint32_t)((val >> 16) & 0xffff);
dt.month = (uint32_t)((val >> 12) & 0xf);
dt.day = (uint32_t)((val >> 6) & 0x3f);
break;
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getDatetimeIntVal(row, isNull);
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
break;
case CalpontSystemCatalog::TIMESTAMP:
{
val = parm[0]->data()->getTimestampIntVal(row, isNull);
TimeStamp timestamp(val);
int64_t seconds = timestamp.second;
MySQLTime time;
gmtSecToMySQLTime(seconds, time, timeZone());
dt.year = time.year;
dt.month = time.month;
dt.day = time.day;
dt.hour = time.hour;
dt.minute = time.minute;
dt.second = time.second;
dt.msecond = timestamp.msecond;
break;
}
case CalpontSystemCatalog::TIME:
{
DateTime aDateTime = static_cast<DateTime>(nowDatetime());
Time aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
aDateTime.hour = 0;
aDateTime.minute = 0;
aDateTime.second = 0;
aDateTime.msecond = 0;
if ((aTime.hour < 0) || (aTime.is_neg))
{
aTime.hour = -abs(aTime.hour);
aTime.minute = -abs(aTime.minute);
aTime.second = -abs(aTime.second);
aTime.msecond = -abs(aTime.msecond);
}
val = addTime(aDateTime, aTime);
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
break;
}
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::TEXT:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return "";
}
else
{
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return "";
}
else
{
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return "";
}
else
{
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
}
}
else
{
isNull = true;
return "";
}
break;
default:
isNull = true;
return "";
val = parm[0]->data()->getTimestampIntVal(row, isNull);
TimeStamp timestamp(val);
int64_t seconds = timestamp.second;
MySQLTime time;
gmtSecToMySQLTime(seconds, time, timeZone());
dt.year = time.year;
dt.month = time.month;
dt.day = time.day;
dt.hour = time.hour;
dt.minute = time.minute;
dt.second = time.second;
dt.msecond = timestamp.msecond;
break;
}
const string& format = parm[1]->data()->getStrVal(row, isNull);
case CalpontSystemCatalog::TIME:
{
DateTime aDateTime = static_cast<DateTime>(nowDatetime());
Time aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
aDateTime.hour = 0;
aDateTime.minute = 0;
aDateTime.second = 0;
aDateTime.msecond = 0;
if ((aTime.hour < 0) || (aTime.is_neg))
{
aTime.hour = -abs(aTime.hour);
aTime.minute = -abs(aTime.minute);
aTime.second = -abs(aTime.second);
aTime.msecond = -abs(aTime.msecond);
}
val = addTime(aDateTime, aTime);
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
break;
}
return helpers::IDB_date_format(dt, format);
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::TEXT:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return "";
}
else
{
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return "";
}
else
{
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return "";
}
else
{
dt.year = (uint32_t)((val >> 48) & 0xffff);
dt.month = (uint32_t)((val >> 44) & 0xf);
dt.day = (uint32_t)((val >> 38) & 0x3f);
dt.hour = (uint32_t)((val >> 32) & 0x3f);
dt.minute = (uint32_t)((val >> 26) & 0x3f);
dt.second = (uint32_t)((val >> 20) & 0x3f);
dt.msecond = (uint32_t)((val & 0xfffff));
}
}
else
{
isNull = true;
return "";
}
break;
default: isNull = true; return "";
}
const string& format = parm[1]->data()->getStrVal(row, isNull);
return helpers::IDB_date_format(dt, format);
}
int32_t Func_date_format::getDateIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int32_t Func_date_format::getDateIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return dataconvert::DataConvert::dateToInt(getStrVal(row, parm, isNull, ct));
return dataconvert::DataConvert::dateToInt(getStrVal(row, parm, isNull, ct));
}
int64_t Func_date_format::getDatetimeIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
int64_t Func_date_format::getDatetimeIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct));
return dataconvert::DataConvert::datetimeToInt(getStrVal(row, parm, isNull, ct));
}
int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
CalpontSystemCatalog::ColType& ct)
int64_t Func_date_format::getTimestampIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone());
return dataconvert::DataConvert::timestampToInt(getStrVal(row, parm, isNull, ct), timeZone());
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

207
utils/funcexp/func_day.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_day.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*
****************************************************************************/
* $Id: func_day.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -35,119 +35,112 @@ using namespace execplan;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_day::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_day::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_day::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_day::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
int64_t val = 0;
int64_t val = 0;
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
switch (parm[0]->data()->resultType().colDataType)
switch (parm[0]->data()->resultType().colDataType)
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
return (uint32_t)((val >> 6) & 0x3f);
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getIntVal(row, isNull);
return (uint32_t)((val >> 38) & 0x3f);
case CalpontSystemCatalog::TIMESTAMP:
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
return (uint32_t)((val >> 6) & 0x3f);
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getIntVal(row, isNull);
return (uint32_t)((val >> 38) & 0x3f);
case CalpontSystemCatalog::TIMESTAMP:
{
dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull));
int64_t seconds = timestamp.second;
dataconvert::MySQLTime m_time;
dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone());
return m_time.day;
}
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
val = addTime(aDateTime, aTime);
return (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
return (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
return (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
return (uint32_t)((val >> 38) & 0x3f);
}
}
else
{
isNull = true;
}
break;
default:
isNull = true;
return -1;
dataconvert::TimeStamp timestamp(parm[0]->data()->getTimestampIntVal(row, isNull));
int64_t seconds = timestamp.second;
dataconvert::MySQLTime m_time;
dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone());
return m_time.day;
}
return -1;
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
val = addTime(aDateTime, aTime);
return (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
return (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
return (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
return (uint32_t)((val >> 38) & 0x3f);
}
}
else
{
isNull = true;
}
break;
default: isNull = true; return -1;
}
return -1;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

267
utils/funcexp/func_dayname.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_dayname.cpp 2477 2011-04-01 16:07:35Z rdempsey $
*
*
****************************************************************************/
* $Id: func_dayname.cpp 2477 2011-04-01 16:07:35Z rdempsey $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -36,156 +36,147 @@ using namespace execplan;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_dayname::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_dayname::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_dayname::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_dayname::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
uint32_t year = 0;
uint32_t month = 0;
uint32_t day = 0;
int64_t val = 0;
int32_t dayofweek = 0;
uint32_t year = 0;
uint32_t month = 0;
uint32_t day = 0;
int64_t val = 0;
int32_t dayofweek = 0;
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
switch (parm[0]->data()->resultType().colDataType)
switch (parm[0]->data()->resultType().colDataType)
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 16) & 0xffff);
month = (uint32_t)((val >> 12) & 0xf);
day = (uint32_t)((val >> 6) & 0x3f);
break;
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::TIMESTAMP:
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 16) & 0xffff);
month = (uint32_t)((val >> 12) & 0xf);
day = (uint32_t)((val >> 6) & 0x3f);
break;
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::TIMESTAMP:
{
val = parm[0]->data()->getIntVal(row, isNull);
dataconvert::TimeStamp timestamp(val);
int64_t seconds = timestamp.second;
dataconvert::MySQLTime time;
dataconvert::gmtSecToMySQLTime(seconds, time, timeZone());
year = time.year;
month = time.month;
day = time.day;
break;
}
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
val = addTime(aDateTime, aTime);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
}
else
{
isNull = true;
return -1;
}
break;
default:
isNull = true;
return -1;
val = parm[0]->data()->getIntVal(row, isNull);
dataconvert::TimeStamp timestamp(val);
int64_t seconds = timestamp.second;
dataconvert::MySQLTime time;
dataconvert::gmtSecToMySQLTime(seconds, time, timeZone());
year = time.year;
month = time.month;
day = time.day;
break;
}
dayofweek = helpers::calc_mysql_weekday( year, month, day, false);
return dayofweek;
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
val = addTime(aDateTime, aTime);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
}
else
{
isNull = true;
return -1;
}
break;
default: isNull = true; return -1;
}
dayofweek = helpers::calc_mysql_weekday(year, month, day, false);
return dayofweek;
}
string Func_dayname::getStrVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
string Func_dayname::getStrVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& op_ct)
{
int32_t weekday = getIntVal(row, parm, isNull, op_ct);
int32_t weekday = getIntVal(row, parm, isNull, op_ct);
if (weekday == -1)
return "";
if (weekday == -1)
return "";
return helpers::weekdayFullNames[weekday];
return helpers::weekdayFullNames[weekday];
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

249
utils/funcexp/func_dayofweek.cpp
View File

@ -16,10 +16,10 @@
MA 02110-1301, USA. */
/****************************************************************************
* $Id: func_dayofweek.cpp 3616 2013-03-04 14:56:29Z rdempsey $
*
*
****************************************************************************/
* $Id: func_dayofweek.cpp 3616 2013-03-04 14:56:29Z rdempsey $
*
*
****************************************************************************/
#include <cstdlib>
#include <string>
@ -36,139 +36,134 @@ using namespace execplan;
namespace funcexp
{
CalpontSystemCatalog::ColType Func_dayofweek::operationType( FunctionParm& fp, CalpontSystemCatalog::ColType& resultType )
CalpontSystemCatalog::ColType Func_dayofweek::operationType(FunctionParm& fp,
CalpontSystemCatalog::ColType& resultType)
{
return resultType;
return resultType;
}
int64_t Func_dayofweek::getIntVal(rowgroup::Row& row,
FunctionParm& parm,
bool& isNull,
int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, FunctionParm& parm, bool& isNull,
CalpontSystemCatalog::ColType& ct)
{
uint32_t year = 0;
uint32_t month = 0;
uint32_t day = 0;
int64_t val = 0;
uint32_t year = 0;
uint32_t month = 0;
uint32_t day = 0;
int64_t val = 0;
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
dataconvert::DateTime aDateTime;
dataconvert::Time aTime;
switch (parm[0]->data()->resultType().colDataType)
switch (parm[0]->data()->resultType().colDataType)
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 16) & 0xffff);
month = (uint32_t)((val >> 12) & 0xf);
day = (uint32_t)((val >> 6) & 0x3f);
break;
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::TIMESTAMP:
{
case CalpontSystemCatalog::DATE:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 16) & 0xffff);
month = (uint32_t)((val >> 12) & 0xf);
day = (uint32_t)((val >> 6) & 0x3f);
break;
case CalpontSystemCatalog::DATETIME:
val = parm[0]->data()->getIntVal(row, isNull);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::TIMESTAMP:
{
val = parm[0]->data()->getIntVal(row, isNull);
dataconvert::TimeStamp timestamp(val);
int64_t seconds = timestamp.second;
dataconvert::MySQLTime time;
dataconvert::gmtSecToMySQLTime(seconds, time, timeZone());
year = time.year;
month = time.month;
day = time.day;
break;
}
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
val = addTime(aDateTime, aTime);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
}
else
{
isNull = true;
return -1;
}
break;
default:
isNull = true;
return -1;
val = parm[0]->data()->getIntVal(row, isNull);
dataconvert::TimeStamp timestamp(val);
int64_t seconds = timestamp.second;
dataconvert::MySQLTime time;
dataconvert::gmtSecToMySQLTime(seconds, time, timeZone());
year = time.year;
month = time.month;
day = time.day;
break;
}
return helpers::calc_mysql_weekday(year, month, day, true) + 1;
// Time adds to now() and then gets value
case CalpontSystemCatalog::TIME:
aDateTime = static_cast<dataconvert::DateTime>(nowDatetime());
aTime = parm[0]->data()->getTimeIntVal(row, isNull);
aTime.day = 0;
val = addTime(aDateTime, aTime);
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
val = dataconvert::DataConvert::stringToDatetime(parm[0]->data()->getStrVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::INT:
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (parm[0]->data()->resultType().scale == 0)
{
val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull));
if (val == -1)
{
isNull = true;
return -1;
}
else
{
year = (uint32_t)((val >> 48) & 0xffff);
month = (uint32_t)((val >> 44) & 0xf);
day = (uint32_t)((val >> 38) & 0x3f);
}
}
else
{
isNull = true;
return -1;
}
break;
default: isNull = true; return -1;
}
return helpers::calc_mysql_weekday(year, month, day, true) + 1;
}
} // namespace funcexp
} // namespace funcexp
// vim:ts=4 sw=4:

Some files were not shown because too many files have changed in this diff Show More