database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-08 14:22:09 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
utils/common/MonitorProcMem.cpp
View File

@@ -57,11 +57,12 @@ int MonitorProcMem::fMemPctCheck = 0;
//------------------------------------------------------------------------------
void MonitorProcMem::operator()() const
{
while (1)
{
while (1)
{
if (fMaxPct > 0)
{
size_t pct = rss() * 100 / fMemTotal;
if (pct > fMaxPct)
{
cerr << "PrimProc: Too much memory allocated!" << endl;
@@ -78,8 +79,8 @@ void MonitorProcMem::operator()() const
fMemFree = cg.getFreeMemory();
//calculateFreeMem();
pause_();
}
pause_();
}
}
//------------------------------------------------------------------------------
@@ -97,48 +98,49 @@ size_t MonitorProcMem::memTotal() const
//------------------------------------------------------------------------------
size_t MonitorProcMem::rss() const
{
uint64_t rss;
uint64_t rss;
#if defined(_MSC_VER)
HANDLE hProcess;
PROCESS_MEMORY_COUNTERS pmc;
hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |
PROCESS_VM_READ,
FALSE, fPid );
if (NULL == hProcess)
return 0;
HANDLE hProcess;
PROCESS_MEMORY_COUNTERS pmc;
hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |
PROCESS_VM_READ,
FALSE, fPid );
if ( GetProcessMemoryInfo( hProcess, &pmc, sizeof(pmc)) )
rss = pmc.WorkingSetSize;
else
rss = 0;
if (NULL == hProcess)
return 0;
CloseHandle( hProcess );
if ( GetProcessMemoryInfo( hProcess, &pmc, sizeof(pmc)) )
rss = pmc.WorkingSetSize;
else
rss = 0;
CloseHandle( hProcess );
#elif defined(__FreeBSD__)
ostringstream cmd;
cmd << "ps -a -o rss -p " << getpid() << " | tail +2";
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.str().c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
rss = atoi(input) * 1024LL;
ostringstream cmd;
cmd << "ps -a -o rss -p " << getpid() << " | tail +2";
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.str().c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
rss = atoi(input) * 1024LL;
#else
ostringstream pstat;
pstat << "/proc/" << fPid << "/statm";
ifstream in(pstat.str().c_str());
size_t x;
ostringstream pstat;
pstat << "/proc/" << fPid << "/statm";
ifstream in(pstat.str().c_str());
size_t x;
in >> x;
in >> rss;
in >> x;
in >> rss;
//rss is now in pages, convert to bytes
rss *= fPageSize;
//rss is now in pages, convert to bytes
rss *= fPageSize;
#endif
return static_cast<size_t>(rss);
return static_cast<size_t>(rss);
}
//------------------------------------------------------------------------------
@@ -146,31 +148,33 @@ size_t MonitorProcMem::rss() const
//------------------------------------------------------------------------------
void MonitorProcMem::pause_( ) const
{
struct timespec req;
struct timespec rem;
struct timespec req;
struct timespec rem;
req.tv_sec = fSleepSec;
req.tv_nsec = 0;
req.tv_sec = fSleepSec;
req.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
while (1)
{
while (1)
{
#ifdef _MSC_VER
Sleep(req.tv_sec * 1000);
Sleep(req.tv_sec * 1000);
#else
if (nanosleep(&req, &rem) != 0)
{
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
continue;
}
}
if (nanosleep(&req, &rem) != 0)
{
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
continue;
}
}
#endif
break;
}
break;
}
}
//------------------------------------------------------------------------------
@@ -178,7 +182,7 @@ void MonitorProcMem::pause_( ) const
//------------------------------------------------------------------------------
unsigned MonitorProcMem::memUsedPct()
{
return ((100 * (fMemTotal- fMemFree)) / fMemTotal);
return ((100 * (fMemTotal - fMemFree)) / fMemTotal);
}
//------------------------------------------------------------------------------
@@ -187,7 +191,7 @@ unsigned MonitorProcMem::memUsedPct()
bool MonitorProcMem::isMemAvailable(size_t memRequest)
{
int memAvailPct = ((100 * (fMemFree - memRequest)) / fMemTotal);
return (memAvailPct < fMemPctCheck);
return (memAvailPct < fMemPctCheck);
}
} // end of namespace

127
utils/common/MonitorProcMem.h
View File

@@ -44,78 +44,85 @@ namespace utils
*/
class MonitorProcMem
{
public:
/** @brief MonitorProcMem constructor
*
* @param maxPct (in) maximum allowable memory usage
* @param memChk (in) monitor total system physical memory usage
* @param msgLog (in) message logger to log msg to
* @param sec (in) number of seconds between memory checks
*/
explicit MonitorProcMem(size_t maxPct,
size_t memChk,
uint32_t subsystemID,
unsigned sec=1) :
fPid ( getpid() ),
fMaxPct ( maxPct ),
fSleepSec ( sec ),
fSubsystemID ( subsystemID ),
fPageSize ( getpagesize() ) { fMemPctCheck = memChk; fMemTotal = memTotal(); }
public:
/** @brief MonitorProcMem constructor
*
* @param maxPct (in) maximum allowable memory usage
* @param memChk (in) monitor total system physical memory usage
* @param msgLog (in) message logger to log msg to
* @param sec (in) number of seconds between memory checks
*/
explicit MonitorProcMem(size_t maxPct,
size_t memChk,
uint32_t subsystemID,
unsigned sec = 1) :
fPid ( getpid() ),
fMaxPct ( maxPct ),
fSleepSec ( sec ),
fSubsystemID ( subsystemID ),
fPageSize ( getpagesize() )
{
fMemPctCheck = memChk;
fMemTotal = memTotal();
}
virtual ~MonitorProcMem() {};
virtual ~MonitorProcMem() {};
/** @brief Thread entry point
*
* Entry point for this thread that monitors memory usage.
*/
virtual void operator()() const;
/** @brief Thread entry point
*
* Entry point for this thread that monitors memory usage.
*/
virtual void operator()() const;
/* Accessor to return the free memory
*
*/
size_t getFreeMem() const {return fMemFree;}
/* Accessor to return the free memory
*
*/
size_t getFreeMem() const
{
return fMemFree;
}
/* return the % of total memory used
*
*/
static unsigned memUsedPct();
/* return the % of total memory used
*
*/
static unsigned memUsedPct();
/* return true if memory usage is below configured limit
*
*/
static bool isMemAvailable(size_t memRequest = 0);
/* return true if memory usage is below configured limit
*
*/
static bool isMemAvailable(size_t memRequest = 0);
protected:
//Defaults are okay
//MonitorProcMem (const MonitorProcMem& rhs);
//MonitorProcMem& operator=(const MonitorProcMem& rhs);
//Defaults are okay
//MonitorProcMem (const MonitorProcMem& rhs);
//MonitorProcMem& operator=(const MonitorProcMem& rhs);
/** return the current process RSS size in MB
*
*/
size_t rss() const;
/** return the current process RSS size in MB
*
*/
size_t rss() const;
/* return the system RAM size in MB
*
*/
size_t memTotal() const;
/* return the system RAM size in MB
*
*/
size_t memTotal() const;
/* pause for fSleepSec seconds between memory usage checks
*
*/
void pause_() const;
/* pause for fSleepSec seconds between memory usage checks
*
*/
void pause_() const;
pid_t fPid; // process pid
size_t fMaxPct; // max allowable % memory use
unsigned fSleepSec; // sleep interval in seconds
uint32_t fSubsystemID; // Subsystem ID for Logger
int fPageSize; // page size for this host (in bytes)
pid_t fPid; // process pid
size_t fMaxPct; // max allowable % memory use
unsigned fSleepSec; // sleep interval in seconds
uint32_t fSubsystemID; // Subsystem ID for Logger
int fPageSize; // page size for this host (in bytes)
// @bug4507, monitor % of total used system memory
static uint64_t fMemTotal;
static uint64_t fMemFree;
static int fMemPctCheck;
mutable CGroupConfigurator cg;
// @bug4507, monitor % of total used system memory
static uint64_t fMemTotal;
static uint64_t fMemFree;
static int fMemPctCheck;
mutable CGroupConfigurator cg;
};
}

125
utils/common/atomicops.h
View File

@@ -41,17 +41,19 @@ template <typename T>
inline T atomicInc(volatile T* mem)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
return InterlockedIncrement(reinterpret_cast<volatile LONG*>(mem));
case 8:
return InterlockedIncrement64(reinterpret_cast<volatile LONGLONG*>(mem));
}
switch (sizeof(T))
{
case 4:
default:
return InterlockedIncrement(reinterpret_cast<volatile LONG*>(mem));
case 8:
return InterlockedIncrement64(reinterpret_cast<volatile LONGLONG*>(mem));
}
#else
return __sync_add_and_fetch(mem, 1);
return __sync_add_and_fetch(mem, 1);
#endif
}
@@ -60,17 +62,19 @@ template <typename T>
inline T atomicDec(volatile T* mem)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
return InterlockedDecrement(reinterpret_cast<volatile LONG*>(mem))+1;
case 8:
return InterlockedDecrement64(reinterpret_cast<volatile LONGLONG*>(mem))+1;
}
switch (sizeof(T))
{
case 4:
default:
return InterlockedDecrement(reinterpret_cast<volatile LONG*>(mem)) + 1;
case 8:
return InterlockedDecrement64(reinterpret_cast<volatile LONGLONG*>(mem)) + 1;
}
#else
return __sync_fetch_and_add(mem, -1);
return __sync_fetch_and_add(mem, -1);
#endif
}
@@ -79,20 +83,22 @@ template <typename T>
inline T atomicAdd(volatile T* mem, T val)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), val);
break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), val);
break;
}
return *mem;
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), val);
break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), val);
break;
}
return *mem;
#else
return __sync_add_and_fetch(mem, val);
return __sync_add_and_fetch(mem, val);
#endif
}
@@ -101,19 +107,22 @@ template <typename T>
inline T atomicSub(volatile T* mem, T val)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), -(static_cast<LONG>(val)));
break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), -(static_cast<LONGLONG>(val)));
break;
}
return *mem;
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), -(static_cast<LONG>(val)));
break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), -(static_cast<LONGLONG>(val)));
break;
}
return *mem;
#else
return __sync_sub_and_fetch(mem, val);
return __sync_sub_and_fetch(mem, val);
#endif
}
@@ -121,9 +130,9 @@ inline T atomicSub(volatile T* mem, T val)
inline void atomicMb()
{
#ifdef _MSC_VER
MemoryBarrier();
MemoryBarrier();
#else
__sync_synchronize();
__sync_synchronize();
#endif
}
@@ -138,19 +147,21 @@ template <typename T>
inline bool atomicCAS(volatile T* mem, T comp, T swap)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
//The function returns the initial value of the mem parameter
return (InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(mem), swap, comp) == comp);
case 8:
return (InterlockedCompareExchange64(reinterpret_cast<volatile LONGLONG*>(mem), swap, comp) == comp);
}
switch (sizeof(T))
{
case 4:
default:
//The function returns the initial value of the mem parameter
return (InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(mem), swap, comp) == comp);
case 8:
return (InterlockedCompareExchange64(reinterpret_cast<volatile LONGLONG*>(mem), swap, comp) == comp);
}
#else
//If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is successful and swap was written.
return __sync_bool_compare_and_swap(mem, comp, swap);
//If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is successful and swap was written.
return __sync_bool_compare_and_swap(mem, comp, swap);
#endif
}
@@ -158,9 +169,9 @@ inline bool atomicCAS(volatile T* mem, T comp, T swap)
inline void atomicYield()
{
#ifdef _MSC_VER
SwitchToThread();
SwitchToThread();
#else
sched_yield();
sched_yield();
#endif
}

271
utils/common/cgroupconfigurator.cpp
View File

@@ -53,17 +53,19 @@ using namespace std;
return err; \
} while (0)
namespace {
void log(logging::LOG_TYPE whichLogFile, const string &msg)
namespace
{
logging::Logger logger(12); //12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
void log(logging::LOG_TYPE whichLogFile, const string& msg)
{
logging::Logger logger(12); //12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
}
}
namespace utils {
namespace utils
{
CGroupConfigurator::CGroupConfigurator()
{
@@ -75,6 +77,7 @@ CGroupConfigurator::CGroupConfigurator()
cGroupDefined = false;
else
cGroupDefined = true;
totalMemory = 0;
totalSwap = 0;
printedWarning = false;
@@ -96,24 +99,31 @@ uint32_t CGroupConfigurator::getNumCoresFromCGroup()
if (!in)
RETURN_NO_GROUP(0);
try {
try
{
// Need to parse & count how many CPUs we have access to
in >> cpusString;
}
catch (...) {
catch (...)
{
RETURN_READ_ERROR(0);
}
// the file has comma-deliminted CPU ranges like "0-7,9,11-12".
size_t first = 0, last;
bool lastRange = false;
while (!lastRange) {
while (!lastRange)
{
size_t dash;
string oneRange;
last = cpusString.find(',', first);
if (last == string::npos) {
if (last == string::npos)
{
lastRange = true;
oneRange = cpusString.substr(first);
}
@@ -122,14 +132,17 @@ uint32_t CGroupConfigurator::getNumCoresFromCGroup()
if ((dash = oneRange.find('-')) == string::npos) // single-cpu range
cpus++;
else {
const char *data = oneRange.c_str();
else
{
const char* data = oneRange.c_str();
uint32_t firstCPU = strtol(data, NULL, 10);
uint32_t lastCPU = strtol(&data[dash+1], NULL, 10);
uint32_t lastCPU = strtol(&data[dash + 1], NULL, 10);
cpus += lastCPU - firstCPU + 1;
}
first = last + 1;
}
//cout << "found " << cpus << " CPUS in the string " << cpusString << endl;
return cpus;
}
@@ -137,13 +150,13 @@ uint32_t CGroupConfigurator::getNumCoresFromCGroup()
uint32_t CGroupConfigurator::getNumCoresFromProc()
{
#ifdef _MSC_VER
SYSTEM_INFO siSysInfo;
GetSystemInfo(&siSysInfo);
return siSysInfo.dwNumberOfProcessors;
SYSTEM_INFO siSysInfo;
GetSystemInfo(&siSysInfo);
return siSysInfo.dwNumberOfProcessors;
#else
uint32_t nc = sysconf(_SC_NPROCESSORS_ONLN);
return nc;
return nc;
#endif
}
@@ -160,11 +173,14 @@ uint32_t CGroupConfigurator::getNumCores()
if (!cGroupDefined)
ret = getNumCoresFromProc();
else {
else
{
ret = getNumCoresFromCGroup();
if (ret == 0)
ret = getNumCoresFromProc();
}
//cout << "There are " << ret << " cores available" << endl;
return ret;
}
@@ -178,11 +194,14 @@ uint64_t CGroupConfigurator::getTotalMemory()
if (!cGroupDefined)
ret = getTotalMemoryFromProc();
else {
else
{
ret = getTotalMemoryFromCGroup();
if (ret == 0)
ret = getTotalMemoryFromProc();
}
//cout << "Total mem available is " << ret << endl;
totalMemory = ret;
return totalMemory;
@@ -190,44 +209,46 @@ uint64_t CGroupConfigurator::getTotalMemory()
uint64_t CGroupConfigurator::getTotalMemoryFromProc()
{
size_t memTot;
size_t memTot;
#if defined(_MSC_VER)
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat) == 0)
//FIXME: Assume 2GB?
memTot = 2 * 1024 * 1024;
else
{
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat) == 0)
//FIXME: Assume 2GB?
memTot = 2 * 1024 * 1024;
else
{
#ifndef _WIN64
memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys);
memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys);
#endif
//We now have the total phys mem in bytes
memTot = memStat.ullTotalPhys / 1024;
}
//We now have the total phys mem in bytes
memTot = memStat.ullTotalPhys / 1024;
}
#elif defined(__FreeBSD__)
string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'");
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
memTot = atoi(input);
string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'");
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
memTot = atoi(input);
#else
ifstream in("/proc/meminfo");
string x;
ifstream in("/proc/meminfo");
string x;
in >> x;
in >> memTot;
in >> x;
in >> memTot;
#endif
//memTot is now in KB, convert to bytes
memTot *= 1024;
//memTot is now in KB, convert to bytes
memTot *= 1024;
return memTot;
return memTot;
}
uint64_t CGroupConfigurator::getTotalMemoryFromCGroup()
@@ -241,15 +262,19 @@ uint64_t CGroupConfigurator::getTotalMemoryFromCGroup()
filename = os.str();
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(0);
try {
try
{
in >> ret;
}
catch (...) {
catch (...)
{
RETURN_READ_ERROR(0);
}
return ret;
}
@@ -259,13 +284,16 @@ uint64_t CGroupConfigurator::getFreeMemory()
if (!cGroupDefined)
ret = getFreeMemoryFromProc();
else {
else
{
uint64_t usage = getMemUsageFromCGroup();
if (usage == 0)
ret = getFreeMemoryFromProc();
else
ret = getTotalMemory() - usage;
}
//cout << "free memory = " << ret << endl;
return ret;
}
@@ -276,28 +304,34 @@ uint64_t CGroupConfigurator::getMemUsageFromCGroup()
bool found = false;
char oneline[80];
if (memUsageFilename.empty()) {
if (memUsageFilename.empty())
{
ostringstream filename;
filename << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
memUsageFilename = filename.str();
}
ifstream in(memUsageFilename.c_str());
string &filename = memUsageFilename;
string& filename = memUsageFilename;
if (!in)
RETURN_NO_GROUP(0);
try {
while (in && !found) {
try
{
while (in && !found)
{
in.getline(oneline, 80);
if (strncmp(oneline, "rss", 2) == 0) {
if (strncmp(oneline, "rss", 2) == 0)
{
ret = atoll(&oneline[3]);
found = true;
}
}
}
catch(...) {
catch (...)
{
RETURN_READ_ERROR(0);
}
@@ -306,59 +340,66 @@ uint64_t CGroupConfigurator::getMemUsageFromCGroup()
uint64_t CGroupConfigurator::getFreeMemoryFromProc()
{
uint64_t memFree = 0;
uint64_t buffers = 0;
uint64_t cached = 0;
uint64_t memTotal = 0;
uint64_t memAvailable = 0;
uint64_t memFree = 0;
uint64_t buffers = 0;
uint64_t cached = 0;
uint64_t memTotal = 0;
uint64_t memAvailable = 0;
#if defined(_MSC_VER)
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat))
{
memAvailable = memStat.ullAvailPhys;
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat))
{
memAvailable = memStat.ullAvailPhys;
#ifndef _WIN64
uint64_t tmp = getTotalMemoryFromProc();
if (memFree > tmp)
memAvailable = tmp;
if (memFree > tmp)
memAvailable = tmp;
#endif
}
}
#elif defined(__FreeBSD__)
// FreeBSD is not supported, no optimization.
memAvailable = 0;
// FreeBSD is not supported, no optimization.
memAvailable = 0;
#else
ifstream in("/proc/meminfo");
string x;
ifstream in("/proc/meminfo");
string x;
in >> x; // MemTotal:
in >> memTotal;
in >> x; // kB
in >> x; // MemTotal:
in >> memTotal;
in >> x; // kB
in >> x; // MemFree:
in >> memFree;
in >> x; // kB
in >> x; // MemFree:
in >> memFree;
in >> x; // kB
//check if available or buffers is passed
in >> x;
if ( x == "MemAvailable:")
{
in >> memAvailable; // MemAvailable
}
else
{ // centos 6 and older OSs
in >> buffers;
in >> x; // kB
//check if available or buffers is passed
in >> x;
if ( x == "MemAvailable:")
{
in >> memAvailable; // MemAvailable
}
else
{
// centos 6 and older OSs
in >> buffers;
in >> x; // kB
in >> x; // Cached:
in >> cached;
memAvailable = memFree + buffers + cached;
}
in >> x; // Cached:
in >> cached;
memAvailable = memFree + buffers + cached;
}
#endif
// amount available for application
memAvailable *= 1024;
// amount available for application
memAvailable *= 1024;
return memAvailable;
}
@@ -371,14 +412,17 @@ uint64_t CGroupConfigurator::getTotalSwapSpace()
if (!cGroupDefined)
ret = getTotalSwapFromSysinfo();
else {
else
{
ret = getTotalMemAndSwapFromCGroup();
// if no limit is set in the cgroup, the file contains maxint64. Use sysinfo in that case.
if (ret == -1 || ret == numeric_limits<int64_t>::max())
ret = getTotalSwapFromSysinfo();
else
ret -= getTotalMemory();
}
//cout << "total swap=" << ret << endl;
totalSwap = ret;
return ret;
@@ -402,15 +446,19 @@ int64_t CGroupConfigurator::getTotalMemAndSwapFromCGroup()
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.memsw.limit_in_bytes";
filename = os.str();
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
try {
try
{
in >> ret;
}
catch(...) {
catch (...)
{
RETURN_READ_ERROR(-1);
}
return ret;
}
@@ -420,11 +468,14 @@ uint64_t CGroupConfigurator::getSwapInUse()
if (!cGroupDefined)
ret = getSwapInUseFromSysinfo();
else {
else
{
ret = getSwapInUseFromCGroup();
if (ret == -1)
ret = getSwapInUseFromSysinfo();
}
//cout << "current swap in use=" << ret << endl;
return ret;
}
@@ -436,29 +487,37 @@ int64_t CGroupConfigurator::getSwapInUseFromCGroup()
bool found = false;
char oneline[80];
if (usedSwapFilename.empty()) {
if (usedSwapFilename.empty())
{
ostringstream os;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
usedSwapFilename = os.str();
}
string &filename = usedSwapFilename;
string& filename = usedSwapFilename;
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
try {
while (in && !found) {
try
{
while (in && !found)
{
in.getline(oneline, 80);
if (strncmp(oneline, "swap", 4) == 0) {
if (strncmp(oneline, "swap", 4) == 0)
{
ret = atoll(&oneline[5]);
found = true;
}
}
}
catch(...) {
catch (...)
{
RETURN_READ_ERROR(-1);
}
return ret;
}

13
utils/common/cgroupconfigurator.h
View File

@@ -24,12 +24,14 @@
#include "configcpp.h"
namespace utils {
namespace utils
{
/* This class wraps a few methods for getting configuration variables that potentially
come from a cgroup. Might move it to utils/configcpp, and/or change the name. */
class CGroupConfigurator {
class CGroupConfigurator
{
public:
CGroupConfigurator();
virtual ~CGroupConfigurator();
@@ -40,7 +42,10 @@ public:
uint64_t getTotalSwapSpace();
uint64_t getSwapInUse();
bool usingCGroup() { return cGroupDefined; }
bool usingCGroup()
{
return cGroupDefined;
}
private:
uint32_t getNumCoresFromProc();
@@ -59,7 +64,7 @@ private:
std::string cGroupName;
bool cGroupDefined;
config::Config *config;
config::Config* config;
uint64_t totalMemory;
uint64_t totalSwap;
bool printedWarning;

110
utils/common/fixedallocator.cpp
View File

@@ -40,83 +40,91 @@
using namespace std;
using namespace boost;
namespace utils {
FixedAllocator::FixedAllocator(const FixedAllocator &f)
namespace utils
{
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
capacityRemaining = 0;
currentlyStored = 0;
FixedAllocator::FixedAllocator(const FixedAllocator& f)
{
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
capacityRemaining = 0;
currentlyStored = 0;
}
FixedAllocator & FixedAllocator::operator=(const FixedAllocator &f)
FixedAllocator& FixedAllocator::operator=(const FixedAllocator& f)
{
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
deallocateAll();
return *this;
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
deallocateAll();
return *this;
}
void FixedAllocator::newBlock()
{
shared_array<uint8_t> next;
shared_array<uint8_t> next;
capacityRemaining = elementCount * elementSize;
if (!tmpSpace || mem.size() == 0) {
next.reset(new uint8_t[elementCount * elementSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else {
currentlyStored = 0;
nextAlloc = mem.front().get();
}
capacityRemaining = elementCount * elementSize;
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[elementCount * elementSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
{
currentlyStored = 0;
nextAlloc = mem.front().get();
}
}
void * FixedAllocator::allocate()
void* FixedAllocator::allocate()
{
void *ret;
if (capacityRemaining < elementSize)
newBlock();
ret = nextAlloc;
nextAlloc += elementSize;
capacityRemaining -= elementSize;
currentlyStored += elementSize;
return ret;
void* ret;
if (capacityRemaining < elementSize)
newBlock();
ret = nextAlloc;
nextAlloc += elementSize;
capacityRemaining -= elementSize;
currentlyStored += elementSize;
return ret;
}
void * FixedAllocator::allocate(uint32_t len)
void* FixedAllocator::allocate(uint32_t len)
{
void *ret;
if (capacityRemaining < len)
newBlock();
ret = nextAlloc;
nextAlloc += len;
capacityRemaining -= len;
currentlyStored += len;
return ret;
void* ret;
if (capacityRemaining < len)
newBlock();
ret = nextAlloc;
nextAlloc += len;
capacityRemaining -= len;
currentlyStored += len;
return ret;
}
void FixedAllocator::truncateBy(uint32_t amt)
{
nextAlloc -= amt;
capacityRemaining += amt;
currentlyStored -= amt;
nextAlloc -= amt;
capacityRemaining += amt;
currentlyStored -= amt;
}
void FixedAllocator::deallocateAll()
{
mem.clear();
currentlyStored = 0;
capacityRemaining = 0;
mem.clear();
currentlyStored = 0;
capacityRemaining = 0;
}
uint64_t FixedAllocator::getMemUsage() const
uint64_t FixedAllocator::getMemUsage() const
{
return (mem.size() * elementCount * elementSize);
return (mem.size() * elementCount * elementSize);
}
}

73
utils/common/fixedallocator.h
View File

@@ -22,7 +22,7 @@
/* This allocator is for frequent small allocations that all get deallocated at once.
It allocates large blocks of memory from the system and distributes 'allocsize'
units to the caller. When the large allocation is used up, it will allocate
units to the caller. When the large allocation is used up, it will allocate
more unless the 'tmpspace' flag is set. If it is, it will reuse the memory it
already allocated. This is useful for short-lived vars that are guaranteed to be
out of scope by the time the allocator wraps around.
@@ -45,49 +45,50 @@
#define EXPORT
#endif
namespace utils {
namespace utils
{
class FixedAllocator
{
public:
EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS=(4096 * 4); // should be a multiple of pagesize
EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS = (4096 * 4); // should be a multiple of pagesize
EXPORT FixedAllocator() :
capacityRemaining(0),
elementCount(std::numeric_limits<unsigned long>::max()),
elementSize(0),
currentlyStored(0),
tmpSpace(false),
nextAlloc(0) {}
EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false,
unsigned long numElements = DEFAULT_NUM_ELEMENTS) :
capacityRemaining(0),
elementCount(numElements),
elementSize(allocSize),
currentlyStored(0),
tmpSpace(isTmpSpace),
nextAlloc(0) {}
EXPORT FixedAllocator(const FixedAllocator &);
EXPORT FixedAllocator & operator=(const FixedAllocator &);
virtual ~FixedAllocator() {}
EXPORT FixedAllocator() :
capacityRemaining(0),
elementCount(std::numeric_limits<unsigned long>::max()),
elementSize(0),
currentlyStored(0),
tmpSpace(false),
nextAlloc(0) {}
EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false,
unsigned long numElements = DEFAULT_NUM_ELEMENTS) :
capacityRemaining(0),
elementCount(numElements),
elementSize(allocSize),
currentlyStored(0),
tmpSpace(isTmpSpace),
nextAlloc(0) {}
EXPORT FixedAllocator(const FixedAllocator&);
EXPORT FixedAllocator& operator=(const FixedAllocator&);
virtual ~FixedAllocator() {}
EXPORT void* allocate();
EXPORT void* allocate(uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead)
EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution
void deallocate() { } // does nothing
EXPORT void deallocateAll(); // drops all memory in use
EXPORT uint64_t getMemUsage() const;
EXPORT void * allocate();
EXPORT void * allocate(uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead)
EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution
void deallocate() { } // does nothing
EXPORT void deallocateAll(); // drops all memory in use
EXPORT uint64_t getMemUsage() const;
private:
void newBlock();
void newBlock();
std::vector<boost::shared_array<uint8_t> > mem;
unsigned long capacityRemaining;
uint64_t elementCount;
unsigned long elementSize;
uint64_t currentlyStored;
bool tmpSpace;
uint8_t* nextAlloc;
std::vector<boost::shared_array<uint8_t> > mem;
unsigned long capacityRemaining;
uint64_t elementCount;
unsigned long elementSize;
uint64_t currentlyStored;
bool tmpSpace;
uint8_t* nextAlloc;
};
}

425
utils/common/hasher.h
View File

@@ -20,7 +20,7 @@
*
*****************************************************************************/
/** @file
/** @file
* class Hasher interface
*/
@@ -38,241 +38,296 @@ namespace utils
inline uint32_t rotl32(uint32_t x, int8_t r)
{
return (x << r) | (x >> (32 - r));
return (x << r) | (x >> (32 - r));
}
inline uint32_t fmix(uint32_t h)
{
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
return h;
}
inline uint64_t fmix(uint64_t k)
{
k ^= k >> 33;
k *= 0xff51afd7ed558ccdULL;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53ULL;
k ^= k >> 33;
k ^= k >> 33;
k *= 0xff51afd7ed558ccdULL;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53ULL;
k ^= k >> 33;
return k;
return k;
}
inline uint64_t rotl64(uint64_t x, int8_t r)
{
return (x << r) | (x >> (64 - r));
return (x << r) | (x >> (64 - r));
}
class Hasher {
class Hasher
{
public:
inline uint32_t operator()(const std::string &s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const std::string& s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const char *data, uint64_t len) const
{
const int nblocks = len / 4;
inline uint32_t operator()(const char* data, uint64_t len) const
{
const int nblocks = len / 4;
uint32_t h1 = 0;
uint32_t h1 = 0;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
//----------
// body
const uint32_t * blocks = (const uint32_t *) (data + nblocks * 4);
const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4);
for (int i = -nblocks; i; i++) {
uint32_t k1 = blocks[i];
for (int i = -nblocks; i; i++)
{
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
//----------
// tail
//----------
// tail
const uint8_t * tail = (const uint8_t*) (data + nblocks * 4);
const uint8_t* tail = (const uint8_t*) (data + nblocks * 4);
uint32_t k1 = 0;
uint32_t k1 = 0;
switch (len & 3) {
case 3: k1 ^= tail[2] << 16;
case 2: k1 ^= tail[1] << 8;
case 1: k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
switch (len & 3)
{
case 3:
k1 ^= tail[2] << 16;
//----------
// finalization
case 2:
k1 ^= tail[1] << 8;
h1 ^= len;
h1 = fmix(h1);
return h1;
}
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h1 = fmix(h1);
return h1;
}
};
class Hasher_r {
class Hasher_r
{
public:
inline uint32_t operator()(const char *data, uint64_t len, uint32_t seed) const
{
const int nblocks = len / 4;
inline uint32_t operator()(const char* data, uint64_t len, uint32_t seed) const
{
const int nblocks = len / 4;
uint32_t h1 = seed;
uint32_t h1 = seed;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
//----------
// body
const uint32_t * blocks = (const uint32_t *) (data + nblocks * 4);
const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4);
for (int i = -nblocks; i; i++) {
uint32_t k1 = blocks[i];
for (int i = -nblocks; i; i++)
{
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
//----------
// tail
//----------
// tail
const uint8_t * tail = (const uint8_t*) (data + nblocks * 4);
const uint8_t* tail = (const uint8_t*) (data + nblocks * 4);
uint32_t k1 = 0;
uint32_t k1 = 0;
switch (len & 3) {
case 3: k1 ^= tail[2] << 16;
case 2: k1 ^= tail[1] << 8;
case 1: k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
switch (len & 3)
{
case 3:
k1 ^= tail[2] << 16;
return h1;
}
inline uint32_t finalize(uint32_t seed, uint32_t len) const {
seed ^= len;
seed = fmix(seed);
return seed;
}
case 2:
k1 ^= tail[1] << 8;
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
return h1;
}
inline uint32_t finalize(uint32_t seed, uint32_t len) const
{
seed ^= len;
seed = fmix(seed);
return seed;
}
};
class Hasher128 {
class Hasher128
{
public:
inline uint64_t operator()(const char *data, uint64_t len) const
{
const int nblocks = len / 16;
inline uint64_t operator()(const char* data, uint64_t len) const
{
const int nblocks = len / 16;
uint64_t h1 = 0;
uint64_t h2 = 0;
uint64_t h1 = 0;
uint64_t h2 = 0;
const uint64_t c1 = 0x87c37b91114253d5ULL;
const uint64_t c2 = 0x4cf5ad432745937fULL;
const uint64_t c1 = 0x87c37b91114253d5ULL;
const uint64_t c2 = 0x4cf5ad432745937fULL;
//----------
// body
//----------
// body
const uint64_t * blocks = (const uint64_t *) (data);
const uint64_t* blocks = (const uint64_t*) (data);
for (int i = 0; i < nblocks; i++) {
uint64_t k1 = blocks[i * 2 + 0];
uint64_t k2 = blocks[i * 2 + 1];
for (int i = 0; i < nblocks; i++)
{
uint64_t k1 = blocks[i * 2 + 0];
uint64_t k2 = blocks[i * 2 + 1];
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
h1 = rotl64(h1, 27);
h1 += h2;
h1 = h1 * 5 + 0x52dce729;
h1 = rotl64(h1, 27);
h1 += h2;
h1 = h1 * 5 + 0x52dce729;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
h2 = rotl64(h2, 31);
h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}
h2 = rotl64(h2, 31);
h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}
//----------
// tail
//----------
// tail
const uint8_t * tail = (const uint8_t*) (data + nblocks * 16);
const uint8_t* tail = (const uint8_t*) (data + nblocks * 16);
uint64_t k1 = 0;
uint64_t k2 = 0;
uint64_t k1 = 0;
uint64_t k2 = 0;
switch (len & 15) {
case 15: k2 ^= uint64_t(tail[14]) << 48;
case 14: k2 ^= uint64_t(tail[13]) << 40;
case 13: k2 ^= uint64_t(tail[12]) << 32;
case 12: k2 ^= uint64_t(tail[11]) << 24;
case 11: k2 ^= uint64_t(tail[10]) << 16;
case 10: k2 ^= uint64_t(tail[9]) << 8;
case 9: k2 ^= uint64_t(tail[8]) << 0;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
case 8: k1 ^= uint64_t(tail[7]) << 56;
case 7: k1 ^= uint64_t(tail[6]) << 48;
case 6: k1 ^= uint64_t(tail[5]) << 40;
case 5: k1 ^= uint64_t(tail[4]) << 32;
case 4: k1 ^= uint64_t(tail[3]) << 24;
case 3: k1 ^= uint64_t(tail[2]) << 16;
case 2: k1 ^= uint64_t(tail[1]) << 8;
case 1: k1 ^= uint64_t(tail[0]) << 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
};
switch (len & 15)
{
case 15:
k2 ^= uint64_t(tail[14]) << 48;
//----------
// finalization
case 14:
k2 ^= uint64_t(tail[13]) << 40;
h1 ^= len;
h2 ^= len;
case 13:
k2 ^= uint64_t(tail[12]) << 32;
h1 += h2;
h2 += h1;
case 12:
k2 ^= uint64_t(tail[11]) << 24;
h1 = fmix(h1);
h2 = fmix(h2);
case 11:
k2 ^= uint64_t(tail[10]) << 16;
h1 += h2;
h2 += h1;
case 10:
k2 ^= uint64_t(tail[9]) << 8;
return h1;
}
case 9:
k2 ^= uint64_t(tail[8]) << 0;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
case 8:
k1 ^= uint64_t(tail[7]) << 56;
case 7:
k1 ^= uint64_t(tail[6]) << 48;
case 6:
k1 ^= uint64_t(tail[5]) << 40;
case 5:
k1 ^= uint64_t(tail[4]) << 32;
case 4:
k1 ^= uint64_t(tail[3]) << 24;
case 3:
k1 ^= uint64_t(tail[2]) << 16;
case 2:
k1 ^= uint64_t(tail[1]) << 8;
case 1:
k1 ^= uint64_t(tail[0]) << 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h2 ^= len;
h1 += h2;
h2 += h1;
h1 = fmix(h1);
h2 = fmix(h2);
h1 += h2;
h2 += h1;
return h1;
}
};
//------------------------------------------------------------------------------
@@ -282,15 +337,15 @@ public:
//------------------------------------------------------------------------------
class TupleHasher
{
public:
TupleHasher(uint32_t len) : fHashLen(len) {}
inline uint64_t operator()(const uint8_t* hashKey) const
{
return fHasher(reinterpret_cast<const char*>(hashKey), fHashLen);
}
private:
Hasher fHasher;
uint32_t fHashLen;
public:
TupleHasher(uint32_t len) : fHashLen(len) {}
inline uint64_t operator()(const uint8_t* hashKey) const
{
return fHasher(reinterpret_cast<const char*>(hashKey), fHashLen);
}
private:
Hasher fHasher;
uint32_t fHashLen;
};
@@ -301,14 +356,14 @@ class TupleHasher
//------------------------------------------------------------------------------
class TupleComparator
{
public:
TupleComparator(uint32_t len) : fCmpLen(len) {}
inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const
{
return (memcmp(hashKey1, hashKey2, fCmpLen) == 0);
}
private:
uint32_t fCmpLen;
public:
TupleComparator(uint32_t len) : fCmpLen(len) {}
inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const
{
return (memcmp(hashKey1, hashKey2, fCmpLen) == 0);
}
private:
uint32_t fCmpLen;
};

290
utils/common/nullvaluemanip.cpp
View File

@@ -22,146 +22,220 @@
using namespace std;
using namespace execplan;
namespace utils {
namespace utils
{
uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth)
{
switch (t) {
case CalpontSystemCatalog::TINYINT:
return joblist::TINYINTNULL;
case CalpontSystemCatalog::SMALLINT:
return joblist::SMALLINTNULL;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return joblist::INTNULL;
case CalpontSystemCatalog::FLOAT:
switch (t)
{
case CalpontSystemCatalog::TINYINT:
return joblist::TINYINTNULL;
case CalpontSystemCatalog::SMALLINT:
return joblist::SMALLINTNULL;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return joblist::INTNULL;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return joblist::FLOATNULL;
case CalpontSystemCatalog::DATE:
return joblist::DATENULL;
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
return joblist::FLOATNULL;
case CalpontSystemCatalog::DATE:
return joblist::DATENULL;
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT: {
switch (colWidth) {
case 1: return joblist::CHAR1NULL;
case 2: return joblist::CHAR2NULL;
case 3:
case 4: return joblist::CHAR4NULL;
case 5:
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
default:
throw logic_error("getNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
switch (colWidth)
{
case 1:
return joblist::CHAR1NULL;
case 2:
return joblist::CHAR2NULL;
case 3:
case 4:
return joblist::CHAR4NULL;
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("getNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth) {
case 1 : return joblist::TINYINTNULL;
case 2 : return joblist::SMALLINTNULL;
case 4 : return joblist::INTNULL;
default: return joblist::BIGINTNULL;
}
break;
}
switch (colWidth)
{
case 1 :
return joblist::TINYINTNULL;
case 2 :
return joblist::SMALLINTNULL;
case 4 :
return joblist::INTNULL;
default:
return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return joblist::UTINYINTNULL;
case CalpontSystemCatalog::USMALLINT:
return joblist::USMALLINTNULL;
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return joblist::UINTNULL;
case CalpontSystemCatalog::UBIGINT:
return joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE:
return -1; // no NULL value for long double yet, this is a nan.
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
}
case CalpontSystemCatalog::LONGDOUBLE:
return -1; // no NULL value for long double yet, this is a nan.
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
}
}
int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth)
{
switch (t) {
case CalpontSystemCatalog::TINYINT:
return (int64_t) ((int8_t) joblist::TINYINTNULL);
case CalpontSystemCatalog::SMALLINT:
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return (int64_t) ((int32_t) joblist::INTNULL);
case CalpontSystemCatalog::FLOAT:
switch (t)
{
case CalpontSystemCatalog::TINYINT:
return (int64_t) ((int8_t) joblist::TINYINTNULL);
case CalpontSystemCatalog::SMALLINT:
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return (int64_t) ((int32_t) joblist::INTNULL);
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return (int64_t) ((int32_t) joblist::FLOATNULL);
case CalpontSystemCatalog::DATE:
return (int64_t) ((int32_t) joblist::DATENULL);
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
return (int64_t) ((int32_t) joblist::FLOATNULL);
case CalpontSystemCatalog::DATE:
return (int64_t) ((int32_t) joblist::DATENULL);
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT: {
switch (colWidth) {
case 1: return (int64_t) ((int8_t) joblist::CHAR1NULL);
case 2: return (int64_t) ((int16_t) joblist::CHAR2NULL);
case 3:
case 4: return (int64_t) ((int32_t) joblist::CHAR4NULL);
case 5:
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
default:
throw logic_error("getSignedNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL: {
switch (colWidth) {
case 1 : return (int64_t) ((int8_t) joblist::TINYINTNULL);
case 2 : return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case 4 : return (int64_t) ((int32_t) joblist::INTNULL);
default: return joblist::BIGINTNULL;
}
break;
}
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
switch (colWidth)
{
case 1:
return (int64_t) ((int8_t) joblist::CHAR1NULL);
case 2:
return (int64_t) ((int16_t) joblist::CHAR2NULL);
case 3:
case 4:
return (int64_t) ((int32_t) joblist::CHAR4NULL);
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("getSignedNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1 :
return (int64_t) ((int8_t) joblist::TINYINTNULL);
case 2 :
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case 4 :
return (int64_t) ((int32_t) joblist::INTNULL);
default:
return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return (int64_t) ((int8_t) joblist::UTINYINTNULL);
case CalpontSystemCatalog::USMALLINT:
return (int64_t) ((int16_t) joblist::USMALLINTNULL);
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return (int64_t) ((int32_t) joblist::UINTNULL);
case CalpontSystemCatalog::UBIGINT:
return (int64_t)joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE:
return -1; // no NULL value for long double yet, this is a nan.
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getSignedNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
}
case CalpontSystemCatalog::LONGDOUBLE:
return -1; // no NULL value for long double yet, this is a nan.
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getSignedNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
}
}

3
utils/common/nullvaluemanip.h
View File

@@ -21,7 +21,8 @@
#include "../../dbcon/execplan/calpontsystemcatalog.h"
namespace utils {
namespace utils
{
// returns the NULL value for our 'numeric' types including short strings.
// The width is only relevant for long string columns.

95
utils/common/poolallocator.cpp
View File

@@ -32,69 +32,74 @@ using namespace boost;
namespace utils
{
PoolAllocator & PoolAllocator::operator=(const PoolAllocator &v)
PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v)
{
allocSize = v.allocSize;
tmpSpace = v.tmpSpace;
deallocateAll();
return *this;
allocSize = v.allocSize;
tmpSpace = v.tmpSpace;
deallocateAll();
return *this;
}
void PoolAllocator::deallocateAll()
{
capacityRemaining = 0;
nextAlloc = NULL;
memUsage = 0;
mem.clear();
oob.clear();
capacityRemaining = 0;
nextAlloc = NULL;
memUsage = 0;
mem.clear();
oob.clear();
}
void PoolAllocator::newBlock()
{
shared_array<uint8_t> next;
shared_array<uint8_t> next;
capacityRemaining = allocSize;
if (!tmpSpace || mem.size() == 0) {
next.reset(new uint8_t[allocSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
nextAlloc = mem.front().get();
capacityRemaining = allocSize;
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[allocSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
nextAlloc = mem.front().get();
}
void * PoolAllocator::allocate(uint64_t size)
void* PoolAllocator::allocate(uint64_t size)
{
void *ret;
void* ret;
if (size > allocSize) {
OOBMemInfo memInfo;
memUsage += size;
memInfo.mem.reset(new uint8_t[size]);
memInfo.size = size;
ret = (void *) memInfo.mem.get();
oob[ret] = memInfo;
return ret;
}
if (size > allocSize)
{
OOBMemInfo memInfo;
if (size > capacityRemaining)
newBlock();
ret = (void *) nextAlloc;
nextAlloc += size;
capacityRemaining -= size;
memUsage += size;
return ret;
memUsage += size;
memInfo.mem.reset(new uint8_t[size]);
memInfo.size = size;
ret = (void*) memInfo.mem.get();
oob[ret] = memInfo;
return ret;
}
if (size > capacityRemaining)
newBlock();
ret = (void*) nextAlloc;
nextAlloc += size;
capacityRemaining -= size;
memUsage += size;
return ret;
}
void PoolAllocator::deallocate(void *p)
void PoolAllocator::deallocate(void* p)
{
OutOfBandMap::iterator it = oob.find(p);
if (it == oob.end())
return;
memUsage -= it->second.size;
oob.erase(it);
OutOfBandMap::iterator it = oob.find(p);
if (it == oob.end())
return;
memUsage -= it->second.size;
oob.erase(it);
}
}

78
utils/common/poolallocator.h
View File

@@ -32,52 +32,60 @@
#include <map>
#include <boost/shared_array.hpp>
namespace utils {
namespace utils
{
class PoolAllocator
{
public:
static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages
static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages
explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false) :
allocSize(windowSize),
tmpSpace(isTmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0) { }
PoolAllocator(const PoolAllocator &p) :
allocSize(p.allocSize),
tmpSpace(p.tmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0) { }
virtual ~PoolAllocator() {}
explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false) :
allocSize(windowSize),
tmpSpace(isTmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0) { }
PoolAllocator(const PoolAllocator& p) :
allocSize(p.allocSize),
tmpSpace(p.tmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0) { }
virtual ~PoolAllocator() {}
PoolAllocator & operator=(const PoolAllocator &);
PoolAllocator& operator=(const PoolAllocator&);
void *allocate(uint64_t size);
void deallocate(void *p);
void deallocateAll();
void* allocate(uint64_t size);
void deallocate(void* p);
void deallocateAll();
inline uint64_t getMemUsage() const { return memUsage; }
unsigned getWindowSize() const { return allocSize; }
inline uint64_t getMemUsage() const
{
return memUsage;
}
unsigned getWindowSize() const
{
return allocSize;
}
private:
void newBlock();
void newBlock();
unsigned allocSize;
std::vector<boost::shared_array<uint8_t> > mem;
bool tmpSpace;
unsigned capacityRemaining;
uint64_t memUsage;
uint8_t *nextAlloc;
struct OOBMemInfo {
boost::shared_array<uint8_t> mem;
uint64_t size;
};
typedef std::map<void *, OOBMemInfo> OutOfBandMap;
OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd
unsigned allocSize;
std::vector<boost::shared_array<uint8_t> > mem;
bool tmpSpace;
unsigned capacityRemaining;
uint64_t memUsage;
uint8_t* nextAlloc;
struct OOBMemInfo
{
boost::shared_array<uint8_t> mem;
uint64_t size;
};
typedef std::map<void*, OOBMemInfo> OutOfBandMap;
OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd
};
}

246
utils/common/simpleallocator.h
View File

@@ -51,154 +51,212 @@ template<typename T> class SimpleAllocator;
#define OPT_NODE_UNITS 10
class SimplePool
{
public:
SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0) {}
~SimplePool() { reset(); }
public:
SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0) {}
~SimplePool()
{
reset();
}
inline void* allocate(size_t n, const void* = 0);
inline void deallocate(void* p, size_t n);
inline size_t max_size() const throw();
inline uint64_t getMemUsage() const;
inline void* allocate(size_t n, const void* = 0);
inline void deallocate(void* p, size_t n);
inline size_t max_size() const throw();
inline uint64_t getMemUsage() const;
private:
static const size_t fUnitPerChunk = OPT_NODE_UNITS*10240;
static const size_t fUnitPerChunk = OPT_NODE_UNITS * 10240;
inline void reset();
inline void allocateNewChunk();
inline void reset();
inline void allocateNewChunk();
// MemUnit stores a pointer to next unit before allocated, and T after allocated.
union MemUnit
{
MemUnit* fNext;
uint64_t fData;
} *fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit
// MemUnit stores a pointer to next unit before allocated, and T after allocated.
union MemUnit
{
MemUnit* fNext;
uint64_t fData;
}* fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit
std::list<MemUnit*> fBlockList;
uint64_t fTableMemSize;
std::list<MemUnit*> fBlockList;
uint64_t fTableMemSize;
static const size_t fUnitSize = sizeof(MemUnit);
static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS;
static const size_t fChunkSize = fUnitSize * fUnitPerChunk;
static const size_t fUnitSize = sizeof(MemUnit);
static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS;
static const size_t fChunkSize = fUnitSize * fUnitPerChunk;
};
template<typename T>
class SimpleAllocator
{
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T *pointer;
typedef const T *const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<typename U> struct rebind { typedef SimpleAllocator<U> other; };
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<typename U> struct rebind
{
typedef SimpleAllocator<U> other;
};
SimpleAllocator() throw() {}
SimpleAllocator(boost::shared_ptr<SimplePool> pool) throw() { fPool = pool; }
SimpleAllocator(const SimpleAllocator& alloc) { fPool = alloc.fPool; }
template<class U> SimpleAllocator(const SimpleAllocator<U>& alloc) { fPool = alloc.fPool; }
SimpleAllocator() throw() {}
SimpleAllocator(boost::shared_ptr<SimplePool> pool) throw()
{
fPool = pool;
}
SimpleAllocator(const SimpleAllocator& alloc)
{
fPool = alloc.fPool;
}
template<class U> SimpleAllocator(const SimpleAllocator<U>& alloc)
{
fPool = alloc.fPool;
}
~SimpleAllocator() throw() { }
~SimpleAllocator() throw() { }
pointer address(reference x) const { return &x; }
const_pointer address(const_reference x) const { return &x; }
pointer address(reference x) const
{
return &x;
}
const_pointer address(const_reference x) const
{
return &x;
}
pointer allocate(size_type n, const void* = 0)
{ return static_cast<pointer>(fPool->allocate(n*sizeof(T))); }
void deallocate(pointer p, size_type n)
{ fPool->deallocate(p, n*sizeof(T)); }
pointer allocate(size_type n, const void* = 0)
{
return static_cast<pointer>(fPool->allocate(n * sizeof(T)));
}
void deallocate(pointer p, size_type n)
{
fPool->deallocate(p, n * sizeof(T));
}
#ifdef _MSC_VER
//The MSVC STL library really needs this to return a big number...
size_type max_size() const throw() { return std::numeric_limits<size_type>::max(); }
//The MSVC STL library really needs this to return a big number...
size_type max_size() const throw()
{
return std::numeric_limits<size_type>::max();
}
#else
size_type max_size() const throw() { return fPool->max_size()/sizeof(T); }
size_type max_size() const throw()
{
return fPool->max_size() / sizeof(T);
}
#endif
void construct(pointer ptr, const T& val) { new ((void *)ptr) T(val); }
void destroy(pointer ptr) { ptr->T::~T(); }
void construct(pointer ptr, const T& val)
{
new ((void*)ptr) T(val);
}
void destroy(pointer ptr)
{
ptr->T::~T();
}
SimplePool* getPool() { return fPool; }
void setPool(SimplePool* pool) { fPool = pool; }
SimplePool* getPool()
{
return fPool;
}
void setPool(SimplePool* pool)
{
fPool = pool;
}
boost::shared_ptr<SimplePool> fPool;
boost::shared_ptr<SimplePool> fPool;
};
// inlines
inline void * SimplePool::allocate(size_t n, const void *dur)
inline void* SimplePool::allocate(size_t n, const void* dur)
{
// make sure the block allocated is on unit boundary
size_t unitCount = n / fUnitSize;
if ((n % fUnitSize) != 0)
unitCount += 1;
// make sure the block allocated is on unit boundary
size_t unitCount = n / fUnitSize;
// if for control table, let new allocator handle it.
if (unitCount > OPT_NODE_UNITS) {
fTableMemSize += n;
return new uint8_t[n];
}
// allocate node
MemUnit *curr = fNext;
do {
if (curr == NULL) {
allocateNewChunk();
curr = fNext;
}
fNext = curr + unitCount;
if (fNext > fEnd)
curr = NULL;
} while (!curr);
if ((n % fUnitSize) != 0)
unitCount += 1;
return curr;
// if for control table, let new allocator handle it.
if (unitCount > OPT_NODE_UNITS)
{
fTableMemSize += n;
return new uint8_t[n];
}
// allocate node
MemUnit* curr = fNext;
do
{
if (curr == NULL)
{
allocateNewChunk();
curr = fNext;
}
fNext = curr + unitCount;
if (fNext > fEnd)
curr = NULL;
}
while (!curr);
return curr;
}
inline void SimplePool::deallocate(void* p, size_t n)
{
// only delete the old control table, which is allocated by new allocator.
if (n > fMaxNodeSize)
{
fTableMemSize -= n;
delete [] (static_cast<uint8_t*>(p));
}
// only delete the old control table, which is allocated by new allocator.
if (n > fMaxNodeSize)
{
fTableMemSize -= n;
delete [] (static_cast<uint8_t*>(p));
}
}
inline size_t SimplePool::max_size() const throw()
inline size_t SimplePool::max_size() const throw()
{
return fUnitSize * fUnitPerChunk;
return fUnitSize * fUnitPerChunk;
}
inline uint64_t SimplePool::getMemUsage() const
{
return fTableMemSize + fBlockList.size() * fChunkSize +
// add list overhead, element type is a pointer, and
// lists store a next pointer.
fBlockList.size() * 2 * sizeof(void *);
return fTableMemSize + fBlockList.size() * fChunkSize +
// add list overhead, element type is a pointer, and
// lists store a next pointer.
fBlockList.size() * 2 * sizeof(void*);
}
inline void SimplePool::reset()
{
for (std::list<MemUnit*>::iterator i = fBlockList.begin(); i != fBlockList.end(); i++)
delete [] (*i);
fNext = NULL;
fEnd = NULL;
for (std::list<MemUnit*>::iterator i = fBlockList.begin(); i != fBlockList.end(); i++)
delete [] (*i);
fNext = NULL;
fEnd = NULL;
}
inline void SimplePool::allocateNewChunk()
{
MemUnit* chunk = new MemUnit[fUnitPerChunk];
fBlockList.push_back(chunk);
fNext = chunk;
fEnd = chunk + fUnitPerChunk;
MemUnit* chunk = new MemUnit[fUnitPerChunk];
fBlockList.push_back(chunk);
fNext = chunk;
fEnd = chunk + fUnitPerChunk;
}
template <typename T1, typename T2>
inline bool operator==(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&) {return true;}
inline bool operator==(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&)
{
return true;
}
template <typename T1, typename T2>
inline bool operator!=(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&) {return false;}
inline bool operator!=(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&)
{
return false;
}
}
#endif // UTILS_SIMPLEALLOCATOR_H

105
utils/common/stlpoolallocator.h
View File

@@ -32,13 +32,14 @@
#ifndef STLPOOLALLOCATOR_H_
#define STLPOOLALLOCATOR_H_
namespace utils {
namespace utils
{
/* If using the pool allocator with a boost smart ptr, use an instance of this
as the deleter. */
struct BoostPoolDeallocator
{
inline void operator()(void *ptr) { };
inline void operator()(void* ptr) { };
};
/* This is an STL-compliant wrapper for PoolAllocator + an optimization for containers
@@ -47,63 +48,69 @@ struct BoostPoolDeallocator
template<class T>
class STLPoolAllocator
{
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T *pointer;
typedef const T *const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<class U> struct rebind { typedef STLPoolAllocator<U> other; };
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<class U> struct rebind
{
typedef STLPoolAllocator<U> other;
};
STLPoolAllocator() throw();
STLPoolAllocator(const STLPoolAllocator &) throw();
STLPoolAllocator(uint32_t capacity) throw();
template<class U> STLPoolAllocator(const STLPoolAllocator<U> &) throw();
~STLPoolAllocator();
STLPoolAllocator() throw();
STLPoolAllocator(const STLPoolAllocator&) throw();
STLPoolAllocator(uint32_t capacity) throw();
template<class U> STLPoolAllocator(const STLPoolAllocator<U>&) throw();
~STLPoolAllocator();
STLPoolAllocator<T>& operator=(const STLPoolAllocator<T> &);
STLPoolAllocator<T>& operator=(const STLPoolAllocator<T>&);
void usePoolAllocator(boost::shared_ptr<PoolAllocator> b);
boost::shared_ptr<utils::PoolAllocator> getPoolAllocator();
void usePoolAllocator(boost::shared_ptr<PoolAllocator> b);
boost::shared_ptr<utils::PoolAllocator> getPoolAllocator();
pointer allocate(size_type, const void *hint = 0);
void deallocate(pointer p, size_type n);
size_type max_size() const throw();
inline uint64_t getMemUsage() const { return pa->getMemUsage(); }
pointer allocate(size_type, const void* hint = 0);
void deallocate(pointer p, size_type n);
size_type max_size() const throw();
inline uint64_t getMemUsage() const
{
return pa->getMemUsage();
}
void construct(pointer p, const T& val);
void destroy(pointer p);
void construct(pointer p, const T& val);
void destroy(pointer p);
static const uint32_t DEFAULT_SIZE = 4096*sizeof(T);
static const uint32_t DEFAULT_SIZE = 4096 * sizeof(T);
boost::shared_ptr<utils::PoolAllocator> pa;
boost::shared_ptr<utils::PoolAllocator> pa;
};
template<class T>
STLPoolAllocator<T>::STLPoolAllocator() throw()
{
pa.reset(new PoolAllocator(DEFAULT_SIZE));
pa.reset(new PoolAllocator(DEFAULT_SIZE));
}
template<class T>
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<T> &s) throw()
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<T>& s) throw()
{
pa = s.pa;
pa = s.pa;
}
template<class T>
STLPoolAllocator<T>::STLPoolAllocator(uint32_t capacity) throw()
{
pa.reset(new PoolAllocator(capacity));
pa.reset(new PoolAllocator(capacity));
}
template<class T>
template<class U>
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<U> &s) throw()
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<U>& s) throw()
{
pa = s.pa;
pa = s.pa;
}
template<class T>
@@ -114,64 +121,64 @@ STLPoolAllocator<T>::~STLPoolAllocator()
template<class T>
void STLPoolAllocator<T>::usePoolAllocator(boost::shared_ptr<PoolAllocator> p)
{
pa = p;
pa = p;
}
template<class T>
boost::shared_ptr<utils::PoolAllocator> STLPoolAllocator<T>::getPoolAllocator()
{
return pa;
return pa;
}
template<class T>
typename STLPoolAllocator<T>::pointer
STLPoolAllocator<T>::allocate(typename STLPoolAllocator<T>::size_type s,
typename std::allocator<void>::const_pointer hint)
typename std::allocator<void>::const_pointer hint)
{
return (pointer) pa->allocate(s*sizeof(T));
return (pointer) pa->allocate(s * sizeof(T));
}
template<class T>
void STLPoolAllocator<T>::deallocate(typename STLPoolAllocator<T>::pointer p,
typename STLPoolAllocator<T>::size_type n)
typename STLPoolAllocator<T>::size_type n)
{
pa->deallocate((void *) p);
pa->deallocate((void*) p);
}
template<class T>
typename STLPoolAllocator<T>::size_type STLPoolAllocator<T>::max_size() const throw()
{
return std::numeric_limits<uint64_t>::max()/sizeof(T);
return std::numeric_limits<uint64_t>::max() / sizeof(T);
}
template<class T>
void STLPoolAllocator<T>::construct(typename STLPoolAllocator<T>::pointer p, const T& val)
{
new((void *)p) T(val);
new ((void*)p) T(val);
}
template<class T>
void STLPoolAllocator<T>::destroy(typename STLPoolAllocator<T>::pointer p)
{
p->T::~T();
p->T::~T();
}
template<class T>
STLPoolAllocator<T>& STLPoolAllocator<T>::operator=(const STLPoolAllocator<T> &c)
STLPoolAllocator<T>& STLPoolAllocator<T>::operator=(const STLPoolAllocator<T>& c)
{
pa = c.pa;
return *this;
pa = c.pa;
return *this;
}
template<typename T>
bool operator==(const STLPoolAllocator<T> &, const STLPoolAllocator<T> &)
bool operator==(const STLPoolAllocator<T>&, const STLPoolAllocator<T>&)
{
return true;
return true;
}
template<typename T>
bool operator!=(const STLPoolAllocator<T> &, const STLPoolAllocator<T> &)
bool operator!=(const STLPoolAllocator<T>&, const STLPoolAllocator<T>&)
{
return false;
return false;
}
}

97
utils/common/syncstream.h
View File

@@ -71,70 +71,79 @@ namespace syncstream
class syncbuf : public std::streambuf
{
public:
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f) {}
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f) {}
protected:
/** Write character in the case of overflow */
virtual int overflow(int c = EOF) {
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow() {
int c = getc(fptr);
if (c != EOF)
ungetc(c, fptr);
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow() {
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF) {
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync() {
return fflush(fptr);
}
/** Write character in the case of overflow */
virtual int overflow(int c = EOF)
{
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow()
{
int c = getc(fptr);
if (c != EOF)
ungetc(c, fptr);
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow()
{
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF)
{
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync()
{
return fflush(fptr);
}
private:
FILE* fptr;
FILE* fptr;
};
/** An istream adaptor for input FILE* streams */
class isyncstream : public std::istream
{
public:
/** ctor */
isyncstream() : istream(&buf), buf(0) {}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const {
return &buf;
}
/** ctor */
isyncstream() : istream(&buf), buf(0) {}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
syncbuf buf;
};
/** An ostream adaptor for output FILE* streams */
class osyncstream : public std::ostream
{
public:
/** ctor */
osyncstream() : ostream(&buf), buf(0) {}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const {
return &buf;
}
/** ctor */
osyncstream() : ostream(&buf), buf(0) {}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
syncbuf buf;
};
}