database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

clang format apply

Browse Source
This commit is contained in:
Leonid Fedorov
2022-01-21 16:43:49 +00:00
parent 6b6411229f
commit 04752ec546
1376 changed files with 393460 additions and 412662 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
utils/common/MonitorProcMem.cpp
View File

@ -45,10 +45,9 @@ using namespace logging;
namespace utils
{
uint64_t MonitorProcMem::fMemTotal = 1;
uint64_t MonitorProcMem::fMemFree = 0;
int MonitorProcMem::fMemPctCheck = 0;
int MonitorProcMem::fMemPctCheck = 0;
//------------------------------------------------------------------------------
// Thread entry point function, that drives a thread to periodically (every
@ -57,30 +56,30 @@ int MonitorProcMem::fMemPctCheck = 0;
//------------------------------------------------------------------------------
void MonitorProcMem::operator()() const
{
while (1)
while (1)
{
if (fMaxPct > 0)
{
if (fMaxPct > 0)
{
size_t pct = rss() * 100 / fMemTotal;
size_t pct = rss() * 100 / fMemTotal;
if (pct > fMaxPct)
{
cerr << "PrimProc: Too much memory allocated!" << endl;
if (pct > fMaxPct)
{
cerr << "PrimProc: Too much memory allocated!" << endl;
LoggingID logid(fSubsystemID);
logging::Message msg(logging::M0045);
logging::Message::Args args;
msg.format(args);
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
exit(1);
}
}
fMemFree = cg.getFreeMemory();
//calculateFreeMem();
pause_();
LoggingID logid(fSubsystemID);
logging::Message msg(logging::M0045);
logging::Message::Args args;
msg.format(args);
logging::Logger logger(logid.fSubsysID);
logger.logMessage(LOG_TYPE_CRITICAL, msg, logid);
exit(1);
}
}
fMemFree = cg.getFreeMemory();
// calculateFreeMem();
pause_();
}
}
//------------------------------------------------------------------------------
@ -89,7 +88,7 @@ void MonitorProcMem::operator()() const
//------------------------------------------------------------------------------
size_t MonitorProcMem::memTotal() const
{
return cg.getTotalMemory();
return cg.getTotalMemory();
}
//------------------------------------------------------------------------------
@ -98,83 +97,81 @@ size_t MonitorProcMem::memTotal() const
//------------------------------------------------------------------------------
size_t MonitorProcMem::rss() const
{
uint64_t rss;
uint64_t rss;
#if defined(_MSC_VER)
HANDLE hProcess;
PROCESS_MEMORY_COUNTERS pmc;
hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |
PROCESS_VM_READ,
FALSE, fPid );
HANDLE hProcess;
PROCESS_MEMORY_COUNTERS pmc;
hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, fPid);
if (NULL == hProcess)
return 0;
if (NULL == hProcess)
return 0;
if ( GetProcessMemoryInfo( hProcess, &pmc, sizeof(pmc)) )
rss = pmc.WorkingSetSize;
else
rss = 0;
if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc)))
rss = pmc.WorkingSetSize;
else
rss = 0;
CloseHandle( hProcess );
CloseHandle(hProcess);
#elif defined(__FreeBSD__)
ostringstream cmd;
cmd << "ps -a -o rss -p " << getpid() << " | tail +2";
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.str().c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
rss = atoi(input) * 1024LL;
ostringstream cmd;
cmd << "ps -a -o rss -p " << getpid() << " | tail +2";
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.str().c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
rss = atoi(input) * 1024LL;
#else
ostringstream pstat;
pstat << "/proc/" << fPid << "/statm";
ifstream in(pstat.str().c_str());
size_t x;
ostringstream pstat;
pstat << "/proc/" << fPid << "/statm";
ifstream in(pstat.str().c_str());
size_t x;
in >> x;
in >> rss;
in >> x;
in >> rss;
//rss is now in pages, convert to bytes
rss *= fPageSize;
// rss is now in pages, convert to bytes
rss *= fPageSize;
#endif
return static_cast<size_t>(rss);
return static_cast<size_t>(rss);
}
//------------------------------------------------------------------------------
// Stays in "sleep" state till fSleepSec seconds have elapsed.
//------------------------------------------------------------------------------
void MonitorProcMem::pause_( ) const
void MonitorProcMem::pause_() const
{
struct timespec req;
struct timespec rem;
struct timespec req;
struct timespec rem;
req.tv_sec = fSleepSec;
req.tv_nsec = 0;
req.tv_sec = fSleepSec;
req.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
while (1)
{
while (1)
{
#ifdef _MSC_VER
Sleep(req.tv_sec * 1000);
Sleep(req.tv_sec * 1000);
#else
if (nanosleep(&req, &rem) != 0)
{
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
continue;
}
}
if (nanosleep(&req, &rem) != 0)
{
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
continue;
}
}
#endif
break;
}
break;
}
}
//------------------------------------------------------------------------------
@ -182,7 +179,7 @@ void MonitorProcMem::pause_( ) const
//------------------------------------------------------------------------------
unsigned MonitorProcMem::memUsedPct()
{
return ((100 * (fMemTotal - fMemFree)) / fMemTotal);
return ((100 * (fMemTotal - fMemFree)) / fMemTotal);
}
//------------------------------------------------------------------------------
@ -190,8 +187,8 @@ unsigned MonitorProcMem::memUsedPct()
//------------------------------------------------------------------------------
bool MonitorProcMem::isMemAvailable(size_t memRequest)
{
int memAvailPct = ((100 * (fMemFree - memRequest)) / fMemTotal);
return (memAvailPct < fMemPctCheck);
int memAvailPct = ((100 * (fMemFree - memRequest)) / fMemTotal);
return (memAvailPct < fMemPctCheck);
}
} // end of namespace
} // namespace utils

133
utils/common/MonitorProcMem.h
View File

@ -35,7 +35,6 @@
namespace utils
{
/** @brief A class to monitor a process's memory usage.
*
* This class will monitor and terminate the process if
@ -43,86 +42,78 @@ namespace utils
*/
class MonitorProcMem
{
public:
/** @brief MonitorProcMem constructor
*
* @param maxPct (in) maximum allowable memory usage
* @param memChk (in) monitor total system physical memory usage
* @param msgLog (in) message logger to log msg to
* @param sec (in) number of seconds between memory checks
*/
explicit MonitorProcMem(size_t maxPct,
size_t memChk,
uint32_t subsystemID,
unsigned sec = 1) :
fPid ( getpid() ),
fMaxPct ( maxPct ),
fSleepSec ( sec ),
fSubsystemID ( subsystemID ),
fPageSize ( getpagesize() )
{
fMemPctCheck = memChk;
fMemTotal = memTotal();
}
public:
/** @brief MonitorProcMem constructor
*
* @param maxPct (in) maximum allowable memory usage
* @param memChk (in) monitor total system physical memory usage
* @param msgLog (in) message logger to log msg to
* @param sec (in) number of seconds between memory checks
*/
explicit MonitorProcMem(size_t maxPct, size_t memChk, uint32_t subsystemID, unsigned sec = 1)
: fPid(getpid()), fMaxPct(maxPct), fSleepSec(sec), fSubsystemID(subsystemID), fPageSize(getpagesize())
{
fMemPctCheck = memChk;
fMemTotal = memTotal();
}
virtual ~MonitorProcMem() {};
virtual ~MonitorProcMem(){};
/** @brief Thread entry point
*
* Entry point for this thread that monitors memory usage.
*/
virtual void operator()() const;
/** @brief Thread entry point
*
* Entry point for this thread that monitors memory usage.
*/
virtual void operator()() const;
/* Accessor to return the free memory
*
*/
size_t getFreeMem() const
{
return fMemFree;
}
/* Accessor to return the free memory
*
*/
size_t getFreeMem() const
{
return fMemFree;
}
/* return the % of total memory used
*
*/
static unsigned memUsedPct();
/* return the % of total memory used
*
*/
static unsigned memUsedPct();
/* return true if memory usage is below configured limit
*
*/
static bool isMemAvailable(size_t memRequest = 0);
/* return true if memory usage is below configured limit
*
*/
static bool isMemAvailable(size_t memRequest = 0);
protected:
//Defaults are okay
//MonitorProcMem (const MonitorProcMem& rhs);
//MonitorProcMem& operator=(const MonitorProcMem& rhs);
protected:
// Defaults are okay
// MonitorProcMem (const MonitorProcMem& rhs);
// MonitorProcMem& operator=(const MonitorProcMem& rhs);
/** return the current process RSS size in MB
*
*/
size_t rss() const;
/** return the current process RSS size in MB
*
*/
size_t rss() const;
/* return the system RAM size in MB
*
*/
size_t memTotal() const;
/* return the system RAM size in MB
*
*/
size_t memTotal() const;
/* pause for fSleepSec seconds between memory usage checks
*
*/
void pause_() const;
/* pause for fSleepSec seconds between memory usage checks
*
*/
void pause_() const;
pid_t fPid; // process pid
size_t fMaxPct; // max allowable % memory use
unsigned fSleepSec; // sleep interval in seconds
uint32_t fSubsystemID; // Subsystem ID for Logger
int fPageSize; // page size for this host (in bytes)
pid_t fPid; // process pid
size_t fMaxPct; // max allowable % memory use
unsigned fSleepSec; // sleep interval in seconds
uint32_t fSubsystemID; // Subsystem ID for Logger
int fPageSize; // page size for this host (in bytes)
// @bug4507, monitor % of total used system memory
static uint64_t fMemTotal;
static uint64_t fMemFree;
static int fMemPctCheck;
mutable CGroupConfigurator cg;
// @bug4507, monitor % of total used system memory
static uint64_t fMemTotal;
static uint64_t fMemFree;
static int fMemPctCheck;
mutable CGroupConfigurator cg;
};
}
} // namespace utils

518
utils/common/any.hpp Executable file → Normal file
View File

@ -18,290 +18,294 @@ namespace static_any
{
namespace anyimpl
{
struct empty_any
{
};
struct empty_any
{
};
struct base_any_policy
{
virtual void static_delete(void** x) = 0;
virtual void copy_from_value(void const* src, void** dest) = 0;
virtual void clone(void* const* src, void** dest) = 0;
virtual void move(void* const* src, void** dest) = 0;
virtual void* get_value(void** src) = 0;
virtual size_t get_size() = 0;
protected:
~base_any_policy() = default;
};
struct base_any_policy
{
virtual void static_delete(void** x) = 0;
virtual void copy_from_value(void const* src, void** dest) = 0;
virtual void clone(void* const* src, void** dest) = 0;
virtual void move(void* const* src, void** dest) = 0;
virtual void* get_value(void** src) = 0;
virtual size_t get_size() = 0;
//inline base_any_policy::~base_any_policy() throw () {}
protected:
~base_any_policy() = default;
};
template<typename T>
struct typed_base_any_policy : base_any_policy
{
virtual size_t get_size()
{
return sizeof(T);
}
protected:
~typed_base_any_policy() = default;
};
// inline base_any_policy::~base_any_policy() throw () {}
template<typename T>
struct small_any_policy : typed_base_any_policy<T>
{
virtual ~small_any_policy() = default;
virtual void static_delete(void** x)
{
*x = 0;
}
virtual void copy_from_value(void const* src, void** dest)
{
new(dest) T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = *src;
}
virtual void move(void* const* src, void** dest)
{
*dest = *src;
}
virtual void* get_value(void** src)
{
return reinterpret_cast<void*>(src);
}
};
template <typename T>
struct typed_base_any_policy : base_any_policy
{
virtual size_t get_size()
{
return sizeof(T);
}
template<typename T>
struct big_any_policy : typed_base_any_policy<T>
{
virtual ~big_any_policy() = default;
virtual void static_delete(void** x)
{
if (*x)
delete(*reinterpret_cast<T**>(x));
*x = NULL;
}
virtual void copy_from_value(void const* src, void** dest)
{
*dest = new T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = new T(**reinterpret_cast<T* const*>(src));
}
virtual void move(void* const* src, void** dest)
{
(*reinterpret_cast<T**>(dest))->~T();
**reinterpret_cast<T**>(dest) = **reinterpret_cast<T* const*>(src);
}
virtual void* get_value(void** src)
{
return *src;
}
};
protected:
~typed_base_any_policy() = default;
};
template<typename T>
struct choose_policy
{
typedef big_any_policy<T> type;
};
template <typename T>
struct small_any_policy : typed_base_any_policy<T>
{
virtual ~small_any_policy() = default;
virtual void static_delete(void** x)
{
*x = 0;
}
virtual void copy_from_value(void const* src, void** dest)
{
new (dest) T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = *src;
}
virtual void move(void* const* src, void** dest)
{
*dest = *src;
}
virtual void* get_value(void** src)
{
return reinterpret_cast<void*>(src);
}
};
template<typename T>
struct choose_policy<T*>
{
typedef small_any_policy<T*> type;
};
template <typename T>
struct big_any_policy : typed_base_any_policy<T>
{
virtual ~big_any_policy() = default;
virtual void static_delete(void** x)
{
if (*x)
delete (*reinterpret_cast<T**>(x));
*x = NULL;
}
virtual void copy_from_value(void const* src, void** dest)
{
*dest = new T(*reinterpret_cast<T const*>(src));
}
virtual void clone(void* const* src, void** dest)
{
*dest = new T(**reinterpret_cast<T* const*>(src));
}
virtual void move(void* const* src, void** dest)
{
(*reinterpret_cast<T**>(dest))->~T();
**reinterpret_cast<T**>(dest) = **reinterpret_cast<T* const*>(src);
}
virtual void* get_value(void** src)
{
return *src;
}
};
struct any;
template <typename T>
struct choose_policy
{
typedef big_any_policy<T> type;
};
/// Choosing the policy for an any type is illegal, but should never happen.
/// This is designed to throw a compiler error.
template<>
struct choose_policy<any>
{
typedef void type;
};
template <typename T>
struct choose_policy<T*>
{
typedef small_any_policy<T*> type;
};
/// Specializations for big types.
#define BIG_POLICY(TYPE) template<> struct \
choose_policy<TYPE> { typedef big_any_policy<TYPE> type; };
struct any;
BIG_POLICY(int128_t);
/// Specializations for small types.
#define SMALL_POLICY(TYPE) template<> struct \
choose_policy<TYPE> { typedef small_any_policy<TYPE> type; };
/// Choosing the policy for an any type is illegal, but should never happen.
/// This is designed to throw a compiler error.
template <>
struct choose_policy<any>
{
typedef void type;
};
SMALL_POLICY(char);
SMALL_POLICY(signed char);
SMALL_POLICY(unsigned char);
SMALL_POLICY(signed short);
SMALL_POLICY(unsigned short);
SMALL_POLICY(signed int);
SMALL_POLICY(unsigned int);
SMALL_POLICY(signed long);
SMALL_POLICY(unsigned long);
SMALL_POLICY(signed long long);
SMALL_POLICY(unsigned long long);
SMALL_POLICY(float);
SMALL_POLICY(double);
SMALL_POLICY(bool);
/// Specializations for big types.
#define BIG_POLICY(TYPE) \
template <> \
struct choose_policy<TYPE> \
{ \
typedef big_any_policy<TYPE> type; \
};
BIG_POLICY(int128_t);
/// Specializations for small types.
#define SMALL_POLICY(TYPE) \
template <> \
struct choose_policy<TYPE> \
{ \
typedef small_any_policy<TYPE> type; \
};
SMALL_POLICY(char);
SMALL_POLICY(signed char);
SMALL_POLICY(unsigned char);
SMALL_POLICY(signed short);
SMALL_POLICY(unsigned short);
SMALL_POLICY(signed int);
SMALL_POLICY(unsigned int);
SMALL_POLICY(signed long);
SMALL_POLICY(unsigned long);
SMALL_POLICY(signed long long);
SMALL_POLICY(unsigned long long);
SMALL_POLICY(float);
SMALL_POLICY(double);
SMALL_POLICY(bool);
#undef SMALL_POLICY
/// This function will return a different policy for each type.
template<typename T>
base_any_policy* get_policy()
{
static typename choose_policy<T>::type policy;
return &policy;
};
}
/// This function will return a different policy for each type.
template <typename T>
base_any_policy* get_policy()
{
static typename choose_policy<T>::type policy;
return &policy;
};
} // namespace anyimpl
class any
{
private:
// fields
anyimpl::base_any_policy* policy;
void* object;
private:
// fields
anyimpl::base_any_policy* policy;
void* object;
public:
/// Initializing constructor.
template <typename T>
any(const T& x)
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
public:
/// Initializing constructor.
template <typename T>
any(const T& x) : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Empty constructor.
any() : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
}
/// Special initializing constructor for string literals.
any(const char* x) : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Copy constructor.
any(const any& x) : policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Destructor.
~any()
{
policy->static_delete(&object);
}
/// Assignment function from another any.
any& assign(const any& x)
{
reset();
policy = x.policy;
policy->clone(&x.object, &object);
return *this;
}
/// Assignment function.
template <typename T>
any& assign(const T& x)
{
reset();
policy = anyimpl::get_policy<T>();
policy->copy_from_value(&x, &object);
return *this;
}
/// Assignment operator.
template <typename T>
any& operator=(const T& x)
{
return assign(x);
}
/// Assignment operator, specialed for literal strings.
/// They have types like const char [6] which don't work as expected.
any& operator=(const char* x)
{
return assign(x);
}
/// Less than operator for sorting
bool operator<(const any& x) const
{
if (policy == x.policy)
{
assign(x);
}
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1), x.policy->get_value(&p2), policy->get_size()) < 0 ? 1 : 0;
}
return 0;
}
/// Empty constructor.
any()
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
/// equal operator
bool operator==(const any& x) const
{
if (policy == x.policy)
{
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1), x.policy->get_value(&p2), policy->get_size()) == 0 ? 1 : 0;
}
return 0;
}
/// Special initializing constructor for string literals.
any(const char* x)
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Utility functions
uint8_t getHash() const
{
void* p1 = const_cast<void*>(object);
return *(uint64_t*)policy->get_value(&p1) % 4048;
}
any& swap(any& x)
{
std::swap(policy, x.policy);
std::swap(object, x.object);
return *this;
}
/// Copy constructor.
any(const any& x)
: policy(anyimpl::get_policy<anyimpl::empty_any>()), object(NULL)
{
assign(x);
}
/// Cast operator. You can only cast to the original type.
template <typename T>
T& cast()
{
if (policy != anyimpl::get_policy<T>())
throw std::runtime_error("static_any: type mismatch in cast");
T* r = reinterpret_cast<T*>(policy->get_value(&object));
return *r;
}
/// Destructor.
~any()
{
policy->static_delete(&object);
}
/// Returns true if the any contains no value.
bool empty() const
{
return policy == anyimpl::get_policy<anyimpl::empty_any>();
}
/// Assignment function from another any.
any& assign(const any& x)
{
reset();
policy = x.policy;
policy->clone(&x.object, &object);
return *this;
}
/// Frees any allocated memory, and sets the value to NULL.
void reset()
{
policy->static_delete(&object);
policy = anyimpl::get_policy<anyimpl::empty_any>();
}
/// Assignment function.
template <typename T>
any& assign(const T& x)
{
reset();
policy = anyimpl::get_policy<T>();
policy->copy_from_value(&x, &object);
return *this;
}
/// Assignment operator.
template<typename T>
any& operator=(const T& x) {
return assign(x);
}
/// Assignment operator, specialed for literal strings.
/// They have types like const char [6] which don't work as expected.
any& operator=(const char* x) {
return assign(x);
}
/// Less than operator for sorting
bool operator<(const any& x) const
{
if (policy == x.policy)
{
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1),
x.policy->get_value(&p2),
policy->get_size()) < 0 ? 1 : 0;
}
return 0;
}
/// equal operator
bool operator==(const any& x) const
{
if (policy == x.policy)
{
void* p1 = const_cast<void*>(object);
void* p2 = const_cast<void*>(x.object);
return memcmp(policy->get_value(&p1),
x.policy->get_value(&p2),
policy->get_size()) == 0 ? 1 : 0;
}
return 0;
}
/// Utility functions
uint8_t getHash() const
{
void* p1 = const_cast<void*>(object);
return *(uint64_t*)policy->get_value(&p1) % 4048;
}
any& swap(any& x)
{
std::swap(policy, x.policy);
std::swap(object, x.object);
return *this;
}
/// Cast operator. You can only cast to the original type.
template<typename T>
T& cast()
{
if (policy != anyimpl::get_policy<T>())
throw std::runtime_error("static_any: type mismatch in cast");
T* r = reinterpret_cast<T*>(policy->get_value(&object));
return *r;
}
/// Returns true if the any contains no value.
bool empty() const
{
return policy == anyimpl::get_policy<anyimpl::empty_any>();
}
/// Frees any allocated memory, and sets the value to NULL.
void reset()
{
policy->static_delete(&object);
policy = anyimpl::get_policy<anyimpl::empty_any>();
}
/// Returns true if the two types are the same.
bool compatible(const any& x) const
{
return policy == x.policy;
}
/// Returns true if the two types are the same.
bool compatible(const any& x) const
{
return policy == x.policy;
}
};
}
} // namespace static_any

123
utils/common/atomicops.h
View File

@ -34,108 +34,97 @@ Boost has something in interprocess::ipcdetail, but it doesn't have 64-bit API's
namespace atomicops
{
//Returns the resulting, incremented value
// Returns the resulting, incremented value
template <typename T>
inline T atomicInc(volatile T* mem)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
return InterlockedIncrement(reinterpret_cast<volatile LONG*>(mem));
switch (sizeof(T))
{
case 4:
default: return InterlockedIncrement(reinterpret_cast<volatile LONG*>(mem));
case 8:
return InterlockedIncrement64(reinterpret_cast<volatile LONGLONG*>(mem));
}
case 8: return InterlockedIncrement64(reinterpret_cast<volatile LONGLONG*>(mem));
}
#else
return __sync_add_and_fetch(mem, 1);
return __sync_add_and_fetch(mem, 1);
#endif
}
//decrements, but returns the pre-decrement value
// decrements, but returns the pre-decrement value
template <typename T>
inline T atomicDec(volatile T* mem)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
return InterlockedDecrement(reinterpret_cast<volatile LONG*>(mem)) + 1;
switch (sizeof(T))
{
case 4:
default: return InterlockedDecrement(reinterpret_cast<volatile LONG*>(mem)) + 1;
case 8:
return InterlockedDecrement64(reinterpret_cast<volatile LONGLONG*>(mem)) + 1;
}
case 8: return InterlockedDecrement64(reinterpret_cast<volatile LONGLONG*>(mem)) + 1;
}
#else
return __sync_fetch_and_add(mem, -1);
return __sync_fetch_and_add(mem, -1);
#endif
}
//Returns the resulting value (but doesn't need to yet)
// Returns the resulting value (but doesn't need to yet)
template <typename T>
inline T atomicAdd(volatile T* mem, T val)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), val);
break;
switch (sizeof(T))
{
case 4:
default: InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), val); break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), val);
break;
}
case 8: InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), val); break;
}
return *mem;
return *mem;
#else
return __sync_add_and_fetch(mem, val);
return __sync_add_and_fetch(mem, val);
#endif
}
//Returns the resulting value
// Returns the resulting value
template <typename T>
inline T atomicSub(volatile T* mem, T val)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), -(static_cast<LONG>(val)));
break;
switch (sizeof(T))
{
case 4:
default: InterlockedExchangeAdd(reinterpret_cast<volatile LONG*>(mem), -(static_cast<LONG>(val))); break;
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), -(static_cast<LONGLONG>(val)));
break;
}
case 8:
InterlockedExchangeAdd64(reinterpret_cast<volatile LONGLONG*>(mem), -(static_cast<LONGLONG>(val)));
break;
}
return *mem;
return *mem;
#else
return __sync_sub_and_fetch(mem, val);
return __sync_sub_and_fetch(mem, val);
#endif
}
//Implements a memory barrier
// Implements a memory barrier
inline void atomicMb()
{
#ifdef _MSC_VER
MemoryBarrier();
MemoryBarrier();
#else
__sync_synchronize();
__sync_synchronize();
#endif
}
//Returns true iff the CAS took place, that is
// Returns true iff the CAS took place, that is
// if (*mem == comp) {
// *mem = swap;
// return true;
@ -147,32 +136,32 @@ inline bool atomicCAS(volatile T* mem, T comp, T swap)
{
#ifdef _MSC_VER
switch (sizeof(T))
{
case 4:
default:
//The function returns the initial value of the mem parameter
return (InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(mem), swap, comp) == comp);
switch (sizeof(T))
{
case 4:
default:
// The function returns the initial value of the mem parameter
return (InterlockedCompareExchange(reinterpret_cast<volatile LONG*>(mem), swap, comp) == comp);
case 8:
return (InterlockedCompareExchange64(reinterpret_cast<volatile LONGLONG*>(mem), swap, comp) == comp);
}
case 8:
return (InterlockedCompareExchange64(reinterpret_cast<volatile LONGLONG*>(mem), swap, comp) == comp);
}
#else
//If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is successful and swap was written.
return __sync_bool_compare_and_swap(mem, comp, swap);
// If the current value of *mem is comp, then write swap into *comp. Return true if the comparison is
// successful and swap was written.
return __sync_bool_compare_and_swap(mem, comp, swap);
#endif
}
//Implements a scheduler yield
// Implements a scheduler yield
inline void atomicYield()
{
#ifdef _MSC_VER
SwitchToThread();
SwitchToThread();
#else
sched_yield();
sched_yield();
#endif
}
}
} // namespace atomicops

6
utils/common/branchpred.h
View File

@ -26,8 +26,6 @@
#endif
#ifndef LIKELY
#define LIKELY(x) __builtin_expect((x),1)
#define UNLIKELY(x) __builtin_expect((x),0)
#define LIKELY(x) __builtin_expect((x), 1)
#define UNLIKELY(x) __builtin_expect((x), 0)
#endif

681
utils/common/cgroupconfigurator.cpp
View File

@ -30,58 +30,61 @@
using namespace boost;
using namespace std;
// minor space-savers
#define RETURN_NO_GROUP(err) do { \
if (!printedWarning) { \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! The group " << cGroupName << " does not exist."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
#define RETURN_NO_GROUP(err) \
do \
{ \
if (!printedWarning) \
{ \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! The group " << cGroupName << " does not exist."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
#define RETURN_READ_ERROR(err) do { \
if (!printedWarning) { \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! Could not read the file " << filename << "."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
#define RETURN_READ_ERROR(err) \
do \
{ \
if (!printedWarning) \
{ \
printedWarning = true; \
ostringstream os; \
os << "CGroup warning! Could not read the file " << filename << "."; \
cerr << os.str() << endl; \
log(logging::LOG_TYPE_WARNING, os.str()); \
} \
return err; \
} while (0)
namespace
{
void log(logging::LOG_TYPE whichLogFile, const string& msg)
{
logging::Logger logger(12); //12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
logging::Logger logger(12); // 12 = configcpp
logger.logMessage(whichLogFile, msg, logging::LoggingID(12));
}
}
} // namespace
namespace utils
{
CGroupConfigurator::CGroupConfigurator()
{
config = config::Config::makeConfig();
config = config::Config::makeConfig();
cGroupName = config->getConfig("SystemConfig", "CGroup");
cGroupName = config->getConfig("SystemConfig", "CGroup");
if (cGroupName.empty())
cGroupDefined = false;
else
cGroupDefined = true;
if (cGroupName.empty())
cGroupDefined = false;
else
cGroupDefined = true;
totalMemory = 0;
totalSwap = 0;
printedWarning = false;
totalMemory = 0;
totalSwap = 0;
printedWarning = false;
}
CGroupConfigurator::~CGroupConfigurator()
@ -90,445 +93,441 @@ CGroupConfigurator::~CGroupConfigurator()
uint32_t CGroupConfigurator::getNumCoresFromCGroup()
{
ostringstream filename_os;
filename_os << "/sys/fs/cgroup/cpuset/" << cGroupName << "/cpus";
string filename = filename_os.str();
ostringstream filename_os;
filename_os << "/sys/fs/cgroup/cpuset/" << cGroupName << "/cpus";
string filename = filename_os.str();
ifstream in(filename.c_str());
string cpusString;
uint32_t cpus = 0;
ifstream in(filename.c_str());
string cpusString;
uint32_t cpus = 0;
if (!in)
RETURN_NO_GROUP(0);
if (!in)
RETURN_NO_GROUP(0);
try
try
{
// Need to parse & count how many CPUs we have access to
in >> cpusString;
}
catch (...)
{
RETURN_READ_ERROR(0);
}
// the file has comma-deliminted CPU ranges like "0-7,9,11-12".
size_t first = 0, last;
bool lastRange = false;
while (!lastRange)
{
size_t dash;
string oneRange;
last = cpusString.find(',', first);
if (last == string::npos)
{
// Need to parse & count how many CPUs we have access to
in >> cpusString;
lastRange = true;
oneRange = cpusString.substr(first);
}
catch (...)
else
oneRange = cpusString.substr(first, last - first - 1);
if ((dash = oneRange.find('-')) == string::npos) // single-cpu range
cpus++;
else
{
RETURN_READ_ERROR(0);
const char* data = oneRange.c_str();
uint32_t firstCPU = strtol(data, NULL, 10);
uint32_t lastCPU = strtol(&data[dash + 1], NULL, 10);
cpus += lastCPU - firstCPU + 1;
}
// the file has comma-deliminted CPU ranges like "0-7,9,11-12".
size_t first = 0, last;
bool lastRange = false;
first = last + 1;
}
while (!lastRange)
{
size_t dash;
string oneRange;
last = cpusString.find(',', first);
if (last == string::npos)
{
lastRange = true;
oneRange = cpusString.substr(first);
}
else
oneRange = cpusString.substr(first, last - first - 1);
if ((dash = oneRange.find('-')) == string::npos) // single-cpu range
cpus++;
else
{
const char* data = oneRange.c_str();
uint32_t firstCPU = strtol(data, NULL, 10);
uint32_t lastCPU = strtol(&data[dash + 1], NULL, 10);
cpus += lastCPU - firstCPU + 1;
}
first = last + 1;
}
//cout << "found " << cpus << " CPUS in the string " << cpusString << endl;
return cpus;
// cout << "found " << cpus << " CPUS in the string " << cpusString << endl;
return cpus;
}
uint32_t CGroupConfigurator::getNumCoresFromProc()
{
#ifdef _MSC_VER
SYSTEM_INFO siSysInfo;
GetSystemInfo(&siSysInfo);
return siSysInfo.dwNumberOfProcessors;
SYSTEM_INFO siSysInfo;
GetSystemInfo(&siSysInfo);
return siSysInfo.dwNumberOfProcessors;
#else
uint32_t nc = sysconf(_SC_NPROCESSORS_ONLN);
uint32_t nc = sysconf(_SC_NPROCESSORS_ONLN);
return nc;
return nc;
#endif
}
uint32_t CGroupConfigurator::getNumCores()
{
/*
Detect if InfiniDB is in a C-Group
- get the group ID
If not, get the number of cores from /proc
*/
uint32_t ret;
/*
Detect if InfiniDB is in a C-Group
- get the group ID
If not, get the number of cores from /proc
*/
uint32_t ret;
if (!cGroupDefined)
ret = getNumCoresFromProc();
else
{
ret = getNumCoresFromCGroup();
if (!cGroupDefined)
ret = getNumCoresFromProc();
else
{
ret = getNumCoresFromCGroup();
if (ret == 0)
ret = getNumCoresFromProc();
}
if (ret == 0)
ret = getNumCoresFromProc();
}
//cout << "There are " << ret << " cores available" << endl;
return ret;
// cout << "There are " << ret << " cores available" << endl;
return ret;
}
uint64_t CGroupConfigurator::getTotalMemory()
{
uint64_t ret;
uint64_t ret;
if (totalMemory != 0)
return totalMemory;
if (!cGroupDefined)
ret = getTotalMemoryFromProc();
else
{
ret = getTotalMemoryFromCGroup();
if (ret == 0)
ret = getTotalMemoryFromProc();
}
//cout << "Total mem available is " << ret << endl;
totalMemory = ret;
if (totalMemory != 0)
return totalMemory;
if (!cGroupDefined)
ret = getTotalMemoryFromProc();
else
{
ret = getTotalMemoryFromCGroup();
if (ret == 0)
ret = getTotalMemoryFromProc();
}
// cout << "Total mem available is " << ret << endl;
totalMemory = ret;
return totalMemory;
}
uint64_t CGroupConfigurator::getTotalMemoryFromProc()
{
size_t memTot;
size_t memTot;
#if defined(_MSC_VER)
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat) == 0)
//FIXME: Assume 2GB?
memTot = 2 * 1024 * 1024;
else
{
if (GlobalMemoryStatusEx(&memStat) == 0)
// FIXME: Assume 2GB?
memTot = 2 * 1024 * 1024;
else
{
#ifndef _WIN64
memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys);
memStat.ullTotalPhys = std::min(memStat.ullTotalVirtual, memStat.ullTotalPhys);
#endif
//We now have the total phys mem in bytes
memTot = memStat.ullTotalPhys / 1024;
}
// We now have the total phys mem in bytes
memTot = memStat.ullTotalPhys / 1024;
}
#elif defined(__FreeBSD__)
string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'");
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
memTot = atoi(input);
string cmd("sysctl -a | awk '/realmem/ {print int(($2+1023)/1024);}'");
FILE* cmdPipe;
char input[80];
cmdPipe = popen(cmd.c_str(), "r");
input[0] = '\0';
fgets(input, 80, cmdPipe);
input[79] = '\0';
pclose(cmdPipe);
memTot = atoi(input);
#else
ifstream in("/proc/meminfo");
string x;
ifstream in("/proc/meminfo");
string x;
in >> x;
in >> memTot;
in >> x;
in >> memTot;
#endif
//memTot is now in KB, convert to bytes
memTot *= 1024;
// memTot is now in KB, convert to bytes
memTot *= 1024;
return memTot;
return memTot;
}
uint64_t CGroupConfigurator::getTotalMemoryFromCGroup()
{
ifstream in;
uint64_t ret;
ostringstream os;
string filename;
ifstream in;
uint64_t ret;
ostringstream os;
string filename;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.limit_in_bytes";
filename = os.str();
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.limit_in_bytes";
filename = os.str();
in.open(filename.c_str());
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(0);
if (!in)
RETURN_NO_GROUP(0);
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(0);
}
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(0);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getFreeMemory()
{
uint64_t ret;
uint64_t ret;
if (!cGroupDefined)
ret = getFreeMemoryFromProc();
if (!cGroupDefined)
ret = getFreeMemoryFromProc();
else
{
uint64_t usage = getMemUsageFromCGroup();
if (usage == 0)
ret = getFreeMemoryFromProc();
else
{
uint64_t usage = getMemUsageFromCGroup();
ret = getTotalMemory() - usage;
}
if (usage == 0)
ret = getFreeMemoryFromProc();
else
ret = getTotalMemory() - usage;
}
//cout << "free memory = " << ret << endl;
return ret;
// cout << "free memory = " << ret << endl;
return ret;
}
uint64_t CGroupConfigurator::getMemUsageFromCGroup()
{
uint64_t ret = 0;
bool found = false;
char oneline[80];
uint64_t ret = 0;
bool found = false;
char oneline[80];
if (memUsageFilename.empty())
if (memUsageFilename.empty())
{
ostringstream filename;
filename << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
memUsageFilename = filename.str();
}
ifstream in(memUsageFilename.c_str());
string& filename = memUsageFilename;
if (!in)
RETURN_NO_GROUP(0);
try
{
while (in && !found)
{
ostringstream filename;
filename << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
memUsageFilename = filename.str();
in.getline(oneline, 80);
if (strncmp(oneline, "rss", 2) == 0)
{
ret = atoll(&oneline[3]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(0);
}
ifstream in(memUsageFilename.c_str());
string& filename = memUsageFilename;
if (!in)
RETURN_NO_GROUP(0);
try
{
while (in && !found)
{
in.getline(oneline, 80);
if (strncmp(oneline, "rss", 2) == 0)
{
ret = atoll(&oneline[3]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(0);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getFreeMemoryFromProc()
{
uint64_t memFree = 0;
uint64_t buffers = 0;
uint64_t cached = 0;
uint64_t memTotal = 0;
uint64_t memAvailable = 0;
uint64_t memFree = 0;
uint64_t buffers = 0;
uint64_t cached = 0;
uint64_t memTotal = 0;
uint64_t memAvailable = 0;
#if defined(_MSC_VER)
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
MEMORYSTATUSEX memStat;
memStat.dwLength = sizeof(memStat);
if (GlobalMemoryStatusEx(&memStat))
{
memAvailable = memStat.ullAvailPhys;
if (GlobalMemoryStatusEx(&memStat))
{
memAvailable = memStat.ullAvailPhys;
#ifndef _WIN64
uint64_t tmp = getTotalMemoryFromProc();
uint64_t tmp = getTotalMemoryFromProc();
if (memFree > tmp)
memAvailable = tmp;
if (memFree > tmp)
memAvailable = tmp;
#endif
}
}
#elif defined(__FreeBSD__)
// FreeBSD is not supported, no optimization.
memAvailable = 0;
// FreeBSD is not supported, no optimization.
memAvailable = 0;
#else
ifstream in("/proc/meminfo");
string x;
ifstream in("/proc/meminfo");
string x;
in >> x; // MemTotal:
in >> memTotal;
in >> x; // kB
in >> x; // MemTotal:
in >> memTotal;
in >> x; // kB
in >> x; // MemFree:
in >> memFree;
in >> x; // kB
in >> x; // MemFree:
in >> memFree;
in >> x; // kB
//check if available or buffers is passed
in >> x;
// check if available or buffers is passed
in >> x;
if ( x == "MemAvailable:")
{
in >> memAvailable; // MemAvailable
}
else
{
// centos 6 and older OSs
in >> buffers;
in >> x; // kB
if (x == "MemAvailable:")
{
in >> memAvailable; // MemAvailable
}
else
{
// centos 6 and older OSs
in >> buffers;
in >> x; // kB
in >> x; // Cached:
in >> cached;
in >> x; // Cached:
in >> cached;
memAvailable = memFree + buffers + cached;
}
memAvailable = memFree + buffers + cached;
}
#endif
// amount available for application
memAvailable *= 1024;
return memAvailable;
// amount available for application
memAvailable *= 1024;
return memAvailable;
}
uint64_t CGroupConfigurator::getTotalSwapSpace()
{
int64_t ret;
int64_t ret;
if (totalSwap != 0)
return totalSwap;
if (totalSwap != 0)
return totalSwap;
if (!cGroupDefined)
ret = getTotalSwapFromSysinfo();
if (!cGroupDefined)
ret = getTotalSwapFromSysinfo();
else
{
ret = getTotalMemAndSwapFromCGroup();
// if no limit is set in the cgroup, the file contains maxint64. Use sysinfo in that case.
if (ret == -1 || ret == numeric_limits<int64_t>::max())
ret = getTotalSwapFromSysinfo();
else
{
ret = getTotalMemAndSwapFromCGroup();
ret -= getTotalMemory();
}
// if no limit is set in the cgroup, the file contains maxint64. Use sysinfo in that case.
if (ret == -1 || ret == numeric_limits<int64_t>::max())
ret = getTotalSwapFromSysinfo();
else
ret -= getTotalMemory();
}
//cout << "total swap=" << ret << endl;
totalSwap = ret;
return ret;
// cout << "total swap=" << ret << endl;
totalSwap = ret;
return ret;
}
uint64_t CGroupConfigurator::getTotalSwapFromSysinfo()
{
struct sysinfo si;
struct sysinfo si;
sysinfo(&si);
return si.totalswap;
sysinfo(&si);
return si.totalswap;
}
int64_t CGroupConfigurator::getTotalMemAndSwapFromCGroup()
{
int64_t ret;
ifstream in;
string filename;
ostringstream os;
int64_t ret;
ifstream in;
string filename;
ostringstream os;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.memsw.limit_in_bytes";
filename = os.str();
in.open(filename.c_str());
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.memsw.limit_in_bytes";
filename = os.str();
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
if (!in)
RETURN_NO_GROUP(-1);
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
try
{
in >> ret;
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getSwapInUse()
{
int64_t ret;
int64_t ret;
if (!cGroupDefined)
ret = getSwapInUseFromSysinfo();
else
{
ret = getSwapInUseFromCGroup();
if (!cGroupDefined)
ret = getSwapInUseFromSysinfo();
else
{
ret = getSwapInUseFromCGroup();
if (ret == -1)
ret = getSwapInUseFromSysinfo();
}
if (ret == -1)
ret = getSwapInUseFromSysinfo();
}
//cout << "current swap in use=" << ret << endl;
return ret;
// cout << "current swap in use=" << ret << endl;
return ret;
}
int64_t CGroupConfigurator::getSwapInUseFromCGroup()
{
int64_t ret = -1;
ifstream in;
bool found = false;
char oneline[80];
int64_t ret = -1;
ifstream in;
bool found = false;
char oneline[80];
if (usedSwapFilename.empty())
if (usedSwapFilename.empty())
{
ostringstream os;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
usedSwapFilename = os.str();
}
string& filename = usedSwapFilename;
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
try
{
while (in && !found)
{
ostringstream os;
os << "/sys/fs/cgroup/memory/" << cGroupName << "/memory.stat";
usedSwapFilename = os.str();
in.getline(oneline, 80);
if (strncmp(oneline, "swap", 4) == 0)
{
ret = atoll(&oneline[5]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
string& filename = usedSwapFilename;
in.open(filename.c_str());
if (!in)
RETURN_NO_GROUP(-1);
try
{
while (in && !found)
{
in.getline(oneline, 80);
if (strncmp(oneline, "swap", 4) == 0)
{
ret = atoll(&oneline[5]);
found = true;
}
}
}
catch (...)
{
RETURN_READ_ERROR(-1);
}
return ret;
return ret;
}
uint64_t CGroupConfigurator::getSwapInUseFromSysinfo()
{
struct sysinfo si;
struct sysinfo si;
sysinfo(&si);
return si.totalswap - si.freeswap;
sysinfo(&si);
return si.totalswap - si.freeswap;
}
}
} // namespace utils

67
utils/common/cgroupconfigurator.h
View File

@ -25,50 +25,47 @@
namespace utils
{
/* This class wraps a few methods for getting configuration variables that potentially
come from a cgroup. Might move it to utils/configcpp, and/or change the name. */
class CGroupConfigurator
{
public:
CGroupConfigurator();
virtual ~CGroupConfigurator();
public:
CGroupConfigurator();
virtual ~CGroupConfigurator();
uint32_t getNumCores();
uint64_t getTotalMemory();
uint64_t getFreeMemory();
uint64_t getTotalSwapSpace();
uint64_t getSwapInUse();
uint32_t getNumCores();
uint64_t getTotalMemory();
uint64_t getFreeMemory();
uint64_t getTotalSwapSpace();
uint64_t getSwapInUse();
bool usingCGroup()
{
return cGroupDefined;
}
bool usingCGroup()
{
return cGroupDefined;
}
private:
uint32_t getNumCoresFromProc();
uint32_t getNumCoresFromCGroup();
uint64_t getTotalMemoryFromProc();
uint64_t getTotalMemoryFromCGroup();
uint64_t getFreeMemoryFromProc();
uint64_t getMemUsageFromCGroup();
uint64_t getTotalSwapFromSysinfo();
int64_t getTotalMemAndSwapFromCGroup();
uint64_t getSwapInUseFromSysinfo();
int64_t getSwapInUseFromCGroup();
private:
uint32_t getNumCoresFromProc();
uint32_t getNumCoresFromCGroup();
uint64_t getTotalMemoryFromProc();
uint64_t getTotalMemoryFromCGroup();
uint64_t getFreeMemoryFromProc();
uint64_t getMemUsageFromCGroup();
uint64_t getTotalSwapFromSysinfo();
int64_t getTotalMemAndSwapFromCGroup();
uint64_t getSwapInUseFromSysinfo();
int64_t getSwapInUseFromCGroup();
std::string memUsageFilename;
std::string usedSwapFilename;
std::string cGroupName;
bool cGroupDefined;
config::Config* config;
uint64_t totalMemory;
uint64_t totalSwap;
bool printedWarning;
std::string memUsageFilename;
std::string usedSwapFilename;
std::string cGroupName;
bool cGroupDefined;
config::Config* config;
uint64_t totalMemory;
uint64_t totalSwap;
bool printedWarning;
};
}
} // namespace utils

45
utils/common/checks.h
View File

@ -19,31 +19,34 @@
#include <type_traits>
#include "mcs_int128.h"
namespace utils {
namespace utils
{
template <typename T>
typename std::enable_if<std::is_unsigned<T>::value || datatypes::is_uint128_t<T>::value, bool>::type
is_nonnegative(T)
{
return true;
};
template <typename T>
typename std::enable_if<std::is_unsigned<T>::value ||
datatypes::is_uint128_t<T>::value,
bool>::type
is_nonnegative(T) { return true; };
typename std::enable_if<std::is_signed<T>::value || datatypes::is_int128_t<T>::value, bool>::type
is_nonnegative(T v)
{
return v >= 0;
};
template <typename T>
typename std::enable_if<std::is_signed<T>::value ||
datatypes::is_int128_t<T>::value,
bool>::type
is_nonnegative(T v) { return v >= 0; };
typename std::enable_if<std::is_unsigned<T>::value || datatypes::is_uint128_t<T>::value, bool>::type
is_negative(T)
{
return false;
};
template <typename T>
typename std::enable_if<std::is_unsigned<T>::value ||
datatypes::is_uint128_t<T>::value,
bool>::type
is_negative(T) { return false; };
template <typename T>
typename std::enable_if<std::is_signed<T>::value ||
datatypes::is_int128_t<T>::value,
bool>::type
is_negative(T v) { return v < 0; };
} // namespace utils
typename std::enable_if<std::is_signed<T>::value || datatypes::is_int128_t<T>::value, bool>::type is_negative(
T v)
{
return v < 0;
};
} // namespace utils

235
utils/common/collation.h
View File

@ -19,18 +19,18 @@
#if defined(PREFER_MY_CONFIG_H)
#if !defined(MY_CONFIG_H)
#error my_config.h was not included (but PREFER_MY_CONFIG_H was set)
#endif
#if !defined(MY_CONFIG_H)
#error my_config.h was not included (but PREFER_MY_CONFIG_H was set)
#endif
#include "mcsconfig_conflicting_defs_remember.h"
#include "mcsconfig_conflicting_defs_undef.h"
#include "mcsconfig_conflicting_defs_remember.h"
#include "mcsconfig_conflicting_defs_undef.h"
#else
#if defined(MY_CONFIG_H)
#error my_config.h was included before mcsconfig.h (and PREFER_MY_CONFIG_H was not set)
#endif
#endif //PREFER_MY_CONFIG_H
#if defined(MY_CONFIG_H)
#error my_config.h was included before mcsconfig.h (and PREFER_MY_CONFIG_H was not set)
#endif
#endif // PREFER_MY_CONFIG_H
#include "mcsconfig.h"
@ -48,11 +48,11 @@
#endif
#ifndef TRUE
#define TRUE (1)
#define TRUE (1)
#endif
#ifndef DBUG_ASSERT
#define DBUG_ASSERT(x) idbassert(x)
#define DBUG_ASSERT(x) idbassert(x)
#define DBUG_ASSERT_TEMPORARILY_DEFINED
#endif
@ -72,21 +72,21 @@ typedef char my_bool;
typedef unsigned char uchar;
#if defined(__GNUC__) && !defined(_lint)
typedef char pchar; /* Mixed prototypes can take char */
typedef char puchar; /* Mixed prototypes can take char */
typedef char pbool; /* Mixed prototypes can take char */
typedef short pshort; /* Mixed prototypes can take short int */
typedef float pfloat; /* Mixed prototypes can take float */
typedef char pchar; /* Mixed prototypes can take char */
typedef char puchar; /* Mixed prototypes can take char */
typedef char pbool; /* Mixed prototypes can take char */
typedef short pshort; /* Mixed prototypes can take short int */
typedef float pfloat; /* Mixed prototypes can take float */
#else
typedef int pchar; /* Mixed prototypes can't take char */
typedef uint puchar; /* Mixed prototypes can't take char */
typedef int pbool; /* Mixed prototypes can't take char */
typedef int pshort; /* Mixed prototypes can't take short int */
typedef double pfloat; /* Mixed prototypes can't take float */
typedef int pchar; /* Mixed prototypes can't take char */
typedef uint puchar; /* Mixed prototypes can't take char */
typedef int pbool; /* Mixed prototypes can't take char */
typedef int pshort; /* Mixed prototypes can't take short int */
typedef double pfloat; /* Mixed prototypes can't take float */
#endif
typedef const struct charset_info_st CHARSET_INFO;
extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO *default_charset_info;
extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO* default_charset_info;
#include "m_ctype.h"
@ -97,127 +97,114 @@ extern "C" MYSQL_PLUGIN_IMPORT CHARSET_INFO *default_charset_info;
#undef DBUG_ASSERT
#endif
#if defined(PREFER_MY_CONFIG_H)
#include "mcsconfig_conflicting_defs_restore.h"
#include "mcsconfig_conflicting_defs_restore.h"
#endif
namespace datatypes
{
class MariaDBHasher
{
ulong mPart1;
ulong mPart2;
public:
MariaDBHasher()
:mPart1(1), mPart2(4)
{ }
MariaDBHasher & add(CHARSET_INFO * cs, const char *str, size_t length)
{
cs->hash_sort((const uchar *) str, length, &mPart1, &mPart2);
return *this;
}
MariaDBHasher & add(CHARSET_INFO *cs, const utils::ConstString &str)
{
return add(cs, str.str(), str.length());
}
uint32_t finalize() const
{
return (uint32_t)mPart1;
}
};
ulong mPart1;
ulong mPart2;
public:
MariaDBHasher() : mPart1(1), mPart2(4)
{
}
MariaDBHasher& add(CHARSET_INFO* cs, const char* str, size_t length)
{
cs->hash_sort((const uchar*)str, length, &mPart1, &mPart2);
return *this;
}
MariaDBHasher& add(CHARSET_INFO* cs, const utils::ConstString& str)
{
return add(cs, str.str(), str.length());
}
uint32_t finalize() const
{
return (uint32_t)mPart1;
}
};
// A reference to MariaDB CHARSET_INFO.
class Charset
{
protected:
const struct charset_info_st * mCharset;
public:
Charset(CHARSET_INFO & cs) :mCharset(&cs) { }
Charset(CHARSET_INFO *cs)
:mCharset(cs ? cs : &my_charset_bin)
{ }
Charset(uint32_t charsetNumber);
CHARSET_INFO & getCharset() const { return *mCharset; }
uint32_t hash(const char *data, uint64_t len) const
{
return MariaDBHasher().add(mCharset, data, len).finalize();
}
bool eq(const std::string & str1, const std::string & str2) const
{
return mCharset->strnncollsp(str1.data(), str1.length(),
str2.data(), str2.length()) == 0;
}
int strnncollsp(const utils::ConstString &str1,
const utils::ConstString &str2) const
{
return mCharset->strnncollsp(str1.str(), str1.length(),
str2.str(), str2.length());
}
int strnncollsp(const char *str1, size_t length1,
const char *str2, size_t length2) const
{
return mCharset->strnncollsp(str1, length1, str2, length2);
}
int strnncollsp(const unsigned char *str1, size_t length1,
const unsigned char *str2, size_t length2) const
{
return mCharset->strnncollsp((const char *) str1, length1,
(const char *) str2, length2);
}
bool test_if_important_data(const char *str, const char *end) const
{
if (mCharset->state & MY_CS_NOPAD)
return str < end;
return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end;
protected:
const struct charset_info_st* mCharset;
}
bool like(bool neg,
const utils::ConstString &subject,
const utils::ConstString &pattern) const
{
bool res= !mCharset->wildcmp(subject.str(), subject.end(),
pattern.str(), pattern.end(),
'\\','_','%');
return neg ? !res : res;
}
public:
Charset(CHARSET_INFO& cs) : mCharset(&cs)
{
}
Charset(CHARSET_INFO* cs) : mCharset(cs ? cs : &my_charset_bin)
{
}
Charset(uint32_t charsetNumber);
CHARSET_INFO& getCharset() const
{
return *mCharset;
}
uint32_t hash(const char* data, uint64_t len) const
{
return MariaDBHasher().add(mCharset, data, len).finalize();
}
bool eq(const std::string& str1, const std::string& str2) const
{
return mCharset->strnncollsp(str1.data(), str1.length(), str2.data(), str2.length()) == 0;
}
int strnncollsp(const utils::ConstString& str1, const utils::ConstString& str2) const
{
return mCharset->strnncollsp(str1.str(), str1.length(), str2.str(), str2.length());
}
int strnncollsp(const char* str1, size_t length1, const char* str2, size_t length2) const
{
return mCharset->strnncollsp(str1, length1, str2, length2);
}
int strnncollsp(const unsigned char* str1, size_t length1, const unsigned char* str2, size_t length2) const
{
return mCharset->strnncollsp((const char*)str1, length1, (const char*)str2, length2);
}
bool test_if_important_data(const char* str, const char* end) const
{
if (mCharset->state & MY_CS_NOPAD)
return str < end;
return str + mCharset->scan(str, end, MY_SEQ_SPACES) < end;
}
bool like(bool neg, const utils::ConstString& subject, const utils::ConstString& pattern) const
{
bool res = !mCharset->wildcmp(subject.str(), subject.end(), pattern.str(), pattern.end(), '\\', '_', '%');
return neg ? !res : res;
}
};
class CollationAwareHasher: public Charset
class CollationAwareHasher : public Charset
{
public:
CollationAwareHasher(const Charset &cs)
:Charset(cs)
{ }
inline uint32_t operator()(const std::string& s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const char* data, uint64_t len) const
{
return Charset::hash(data, len);
}
public:
CollationAwareHasher(const Charset& cs) : Charset(cs)
{
}
inline uint32_t operator()(const std::string& s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const char* data, uint64_t len) const
{
return Charset::hash(data, len);
}
};
class CollationAwareComparator: public Charset
class CollationAwareComparator : public Charset
{
public:
CollationAwareComparator(const Charset &cs)
:Charset(cs)
{ }
bool operator()(const std::string & str1, const std::string & str2) const
{
return Charset::eq(str1, str2);
}
public:
CollationAwareComparator(const Charset& cs) : Charset(cs)
{
}
bool operator()(const std::string& str1, const std::string& str2) const
{
return Charset::eq(str1, str2);
}
};
} // end of namespace datatypes
} // end of namespace datatypes

121
utils/common/columnwidth.h
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2020 MariaDB Corporation
/* Copyright (C) 2020 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -21,70 +21,61 @@
namespace utils
{
const uint8_t MAXLEGACYWIDTH = 8ULL;
const uint8_t MAXCOLUMNWIDTH = 16ULL;
inline bool isWide(uint8_t width)
{
return width > MAXLEGACYWIDTH;
}
inline bool isNarrow(uint8_t width)
{
return width <= MAXLEGACYWIDTH;
}
/** @brief Map a DECIMAL precision to data width in bytes */
inline uint8_t widthByPrecision(unsigned p)
{
if (LIKELY(p > 18 && p < 39))
return 16;
switch (p)
{
case 1:
case 2:
return 1;
case 3:
case 4:
return 2;
case 5:
case 6:
case 7:
case 8:
case 9:
return 4;
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
case 16:
case 17:
case 18:
return 8;
default:
return 16;
}
}
inline uint8_t precisionByWidth(unsigned w)
{
switch(w)
{
case 16:
return 38;
// In case we will support decimals that spans 32 bytes.
default:
return 65;
}
}
const uint8_t MAXLEGACYWIDTH = 8ULL;
const uint8_t MAXCOLUMNWIDTH = 16ULL;
inline bool isWide(uint8_t width)
{
return width > MAXLEGACYWIDTH;
}
inline bool isNarrow(uint8_t width)
{
return width <= MAXLEGACYWIDTH;
}
/** @brief Map a DECIMAL precision to data width in bytes */
inline uint8_t widthByPrecision(unsigned p)
{
if (LIKELY(p > 18 && p < 39))
return 16;
switch (p)
{
case 1:
case 2: return 1;
case 3:
case 4: return 2;
case 5:
case 6:
case 7:
case 8:
case 9: return 4;
case 10:
case 11:
case 12:
case 13:
case 14:
case 15:
case 16:
case 17:
case 18: return 8;
default: return 16;
}
}
inline uint8_t precisionByWidth(unsigned w)
{
switch (w)
{
case 16: return 38;
// In case we will support decimals that spans 32 bytes.
default: return 65;
}
}
} // namespace utils

50
utils/common/conststring.h
View File

@ -15,31 +15,38 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <string>
#include <string.h>
namespace utils
{
class ConstString
{
protected:
const char *mStr;
protected:
const char* mStr;
size_t mLength;
public:
ConstString(const char *str, size_t length)
:mStr(str), mLength(length)
{ }
explicit ConstString(const std::string &str)
:mStr(str.data()), mLength(str.length())
{ }
const char *str() const { return mStr; }
const char *end() const { return mStr + mLength; }
size_t length() const { return mLength; }
public:
ConstString(const char* str, size_t length) : mStr(str), mLength(length)
{
}
explicit ConstString(const std::string& str) : mStr(str.data()), mLength(str.length())
{
}
const char* str() const
{
return mStr;
}
const char* end() const
{
return mStr + mLength;
}
size_t length() const
{
return mLength;
}
std::string toString() const
{
return std::string(mStr, mLength);
@ -48,18 +55,17 @@ public:
{
return mLength == 1 && mStr[0] == ch;
}
bool eq(const ConstString &rhs) const
bool eq(const ConstString& rhs) const
{
return mLength == rhs.mLength && !memcmp(mStr, rhs.mStr, mLength);
}
ConstString & rtrimZero()
ConstString& rtrimZero()
{
for ( ; mLength && mStr[mLength - 1] == '\0'; mLength--)
{ }
for (; mLength && mStr[mLength - 1] == '\0'; mLength--)
{
}
return *this;
}
};
} // namespace utils
} // namespace utils

44
utils/common/crashtrace.cpp
View File

@ -27,26 +27,26 @@
void fatalHandler(int sig)
{
char filename[128];
void* addrs[128];
snprintf(filename, 128, "%s/trace/%s.%d.log", MCSLOGDIR, program_invocation_short_name, getpid());
FILE* logfile = fopen(filename, "w");
char s[30];
struct tm tim;
time_t now;
now = time(NULL);
tim = *(localtime(&now));
strftime(s, 30, "%F %T", &tim);
fprintf(logfile, "Date/time: %s\n", s);
fprintf(logfile, "Signal: %d\n\n", sig);
fflush(logfile);
int fd = fileno(logfile);
int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0]));
backtrace_symbols_fd(addrs, count, fd);
fclose(logfile);
struct sigaction sigact;
memset(&sigact, 0, sizeof(sigact));
sigact.sa_handler = SIG_DFL;
sigaction(sig, &sigact, NULL);
raise(sig);
char filename[128];
void* addrs[128];
snprintf(filename, 128, "%s/trace/%s.%d.log", MCSLOGDIR, program_invocation_short_name, getpid());
FILE* logfile = fopen(filename, "w");
char s[30];
struct tm tim;
time_t now;
now = time(NULL);
tim = *(localtime(&now));
strftime(s, 30, "%F %T", &tim);
fprintf(logfile, "Date/time: %s\n", s);
fprintf(logfile, "Signal: %d\n\n", sig);
fflush(logfile);
int fd = fileno(logfile);
int count = backtrace(addrs, sizeof(addrs) / sizeof(addrs[0]));
backtrace_symbols_fd(addrs, count, fd);
fclose(logfile);
struct sigaction sigact;
memset(&sigact, 0, sizeof(sigact));
sigact.sa_handler = SIG_DFL;
sigaction(sig, &sigact, NULL);
raise(sig);
}

92
utils/common/fixedallocator.cpp
View File

@ -16,11 +16,11 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
//This is one of the first files we compile, check the compiler...
// This is one of the first files we compile, check the compiler...
#if defined(__GNUC__)
#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
#error "This is a very old GCC, and it's probably not going to work."
@ -42,80 +42,78 @@ using namespace boost;
namespace utils
{
FixedAllocator::FixedAllocator(const FixedAllocator& f)
{
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
capacityRemaining = 0;
currentlyStored = 0;
useLock = f.useLock;
lock = false;
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
capacityRemaining = 0;
currentlyStored = 0;
useLock = f.useLock;
lock = false;
}
FixedAllocator& FixedAllocator::operator=(const FixedAllocator& f)
{
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
useLock = f.useLock;
lock = false;
deallocateAll();
return *this;
elementCount = f.elementCount;
elementSize = f.elementSize;
tmpSpace = f.tmpSpace;
useLock = f.useLock;
lock = false;
deallocateAll();
return *this;
}
void FixedAllocator::setUseLock(bool useIt)
{
useLock = useIt;
useLock = useIt;
}
void FixedAllocator::setAllocSize(uint allocSize)
{
elementSize = allocSize;
elementSize = allocSize;
}
void FixedAllocator::newBlock()
{
shared_array<uint8_t> next;
shared_array<uint8_t> next;
capacityRemaining = elementCount * elementSize;
capacityRemaining = elementCount * elementSize;
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[elementCount * elementSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
{
currentlyStored = 0;
nextAlloc = mem.front().get();
}
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[elementCount * elementSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
{
currentlyStored = 0;
nextAlloc = mem.front().get();
}
}
void FixedAllocator::truncateBy(uint32_t amt)
{
if (useLock)
getSpinlock(lock);
nextAlloc -= amt;
capacityRemaining += amt;
currentlyStored -= amt;
if (useLock)
releaseSpinlock(lock);
if (useLock)
getSpinlock(lock);
nextAlloc -= amt;
capacityRemaining += amt;
currentlyStored -= amt;
if (useLock)
releaseSpinlock(lock);
}
void FixedAllocator::deallocateAll()
{
mem.clear();
currentlyStored = 0;
capacityRemaining = 0;
mem.clear();
currentlyStored = 0;
capacityRemaining = 0;
}
uint64_t FixedAllocator::getMemUsage() const
{
return (mem.size() * elementCount * elementSize);
return (mem.size() * elementCount * elementSize);
}
}
} // namespace utils

152
utils/common/fixedallocator.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/* This allocator is for frequent small allocations that all get deallocated at once.
It allocates large blocks of memory from the system and distributes 'allocsize'
@ -48,93 +48,101 @@
namespace utils
{
class FixedAllocator
{
public:
EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS = (4096 * 4); // should be a multiple of pagesize
public:
EXPORT static const unsigned long DEFAULT_NUM_ELEMENTS = (4096 * 4); // should be a multiple of pagesize
EXPORT FixedAllocator() :
capacityRemaining(0),
elementCount(DEFAULT_NUM_ELEMENTS),
elementSize(0),
currentlyStored(0),
tmpSpace(false),
nextAlloc(0),
useLock(false),
lock(false) {}
EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false,
unsigned long numElements = DEFAULT_NUM_ELEMENTS) :
capacityRemaining(0),
elementCount(numElements),
elementSize(allocSize),
currentlyStored(0),
tmpSpace(isTmpSpace),
nextAlloc(0),
useLock(false),
lock(false) {}
EXPORT FixedAllocator(const FixedAllocator&);
EXPORT FixedAllocator& operator=(const FixedAllocator&);
virtual ~FixedAllocator() {}
EXPORT FixedAllocator()
: capacityRemaining(0)
, elementCount(DEFAULT_NUM_ELEMENTS)
, elementSize(0)
, currentlyStored(0)
, tmpSpace(false)
, nextAlloc(0)
, useLock(false)
, lock(false)
{
}
EXPORT explicit FixedAllocator(unsigned long allocSize, bool isTmpSpace = false,
unsigned long numElements = DEFAULT_NUM_ELEMENTS)
: capacityRemaining(0)
, elementCount(numElements)
, elementSize(allocSize)
, currentlyStored(0)
, tmpSpace(isTmpSpace)
, nextAlloc(0)
, useLock(false)
, lock(false)
{
}
EXPORT FixedAllocator(const FixedAllocator&);
EXPORT FixedAllocator& operator=(const FixedAllocator&);
virtual ~FixedAllocator()
{
}
EXPORT void* allocate();
EXPORT void* allocate(uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead)
EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution
void deallocate() { } // does nothing
EXPORT void deallocateAll(); // drops all memory in use
EXPORT uint64_t getMemUsage() const;
void setUseLock(bool);
void setAllocSize(uint);
EXPORT void* allocate();
EXPORT void* allocate(
uint32_t len); // a hack to make it work more like a pool allocator (use PoolAllocator instead)
EXPORT void truncateBy(uint32_t amt); // returns a portion of mem just allocated; use with caution
void deallocate()
{
} // does nothing
EXPORT void deallocateAll(); // drops all memory in use
EXPORT uint64_t getMemUsage() const;
void setUseLock(bool);
void setAllocSize(uint);
private:
void newBlock();
private:
void newBlock();
std::vector<boost::shared_array<uint8_t> > mem;
unsigned long capacityRemaining;
uint64_t elementCount;
unsigned long elementSize;
uint64_t currentlyStored;
bool tmpSpace;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
std::vector<boost::shared_array<uint8_t> > mem;
unsigned long capacityRemaining;
uint64_t elementCount;
unsigned long elementSize;
uint64_t currentlyStored;
bool tmpSpace;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
};
inline void* FixedAllocator::allocate()
{
void* ret;
void* ret;
if (useLock)
getSpinlock(lock);
if (capacityRemaining < elementSize)
newBlock();
if (useLock)
getSpinlock(lock);
if (capacityRemaining < elementSize)
newBlock();
ret = nextAlloc;
nextAlloc += elementSize;
capacityRemaining -= elementSize;
currentlyStored += elementSize;
if (useLock)
releaseSpinlock(lock);
return ret;
ret = nextAlloc;
nextAlloc += elementSize;
capacityRemaining -= elementSize;
currentlyStored += elementSize;
if (useLock)
releaseSpinlock(lock);
return ret;
}
inline void* FixedAllocator::allocate(uint32_t len)
{
void* ret;
void* ret;
if (useLock)
getSpinlock(lock);
if (capacityRemaining < len)
newBlock();
if (useLock)
getSpinlock(lock);
if (capacityRemaining < len)
newBlock();
ret = nextAlloc;
nextAlloc += len;
capacityRemaining -= len;
currentlyStored += len;
if (useLock)
releaseSpinlock(lock);
return ret;
ret = nextAlloc;
nextAlloc += len;
capacityRemaining -= len;
currentlyStored += len;
if (useLock)
releaseSpinlock(lock);
return ret;
}
#undef EXPORT
} // namespace
} // namespace utils

243
utils/common/genericparser.h
View File

@ -15,32 +15,29 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "conststring.h"
namespace genericparser
{
using utils::ConstString;
class Tokenizer
{
protected:
const char *mStr;
const char *mEnd;
public:
explicit Tokenizer(const char *str, size_t length)
:mStr(str), mEnd(str + length)
{ }
protected:
const char* mStr;
const char* mEnd;
public:
explicit Tokenizer(const char* str, size_t length) : mStr(str), mEnd(str + length)
{
}
size_t length() const
{
return mEnd - mStr;
}
const char *ptr() const
const char* ptr() const
{
return mStr;
}
@ -65,18 +62,20 @@ public:
{
if (!isSpace())
return ConstString(nullptr, 0);
const char *start = mStr;
for ( ; isSpace() ; mStr++)
{ }
const char* start = mStr;
for (; isSpace(); mStr++)
{
}
return ConstString(start, mStr - start);
}
ConstString tokenDigits()
{
if (!isDigit())
return ConstString(nullptr, 0);
const char *start = mStr;
for ( ; isDigit() ; mStr++)
{ }
const char* start = mStr;
for (; isDigit(); mStr++)
{
}
return ConstString(start, mStr - start);
}
ConstString tokenChar(char chr)
@ -93,19 +92,19 @@ public:
}
};
class Parser: public Tokenizer
class Parser : public Tokenizer
{
protected:
protected:
bool mSyntaxError;
public:
explicit Parser(const char *str, size_t length)
:Tokenizer(str, length), mSyntaxError(false)
{ }
explicit Parser(const std::string & str)
:Parser(str.data(), str.length())
{ }
Parser & skipLeadingSpaces()
public:
explicit Parser(const char* str, size_t length) : Tokenizer(str, length), mSyntaxError(false)
{
}
explicit Parser(const std::string& str) : Parser(str.data(), str.length())
{
}
Parser& skipLeadingSpaces()
{
tokenSpaces();
return *this;
@ -119,7 +118,7 @@ public:
mSyntaxError = true;
return false;
}
const char *tokStart() const
const char* tokStart() const
{
return mStr;
}
@ -128,16 +127,14 @@ public:
return ConstString(mStr, 0);
}
// A helper class template to set the parser syntax error
// if A returned isNull() after parsing.
template<class A>
class SetSyntaxErrorOnNull :public A
template <class A>
class SetSyntaxErrorOnNull : public A
{
public:
SetSyntaxErrorOnNull(Parser *p)
:A(p)
public:
SetSyntaxErrorOnNull(Parser* p) : A(p)
{
if (A::isNull())
p->setSyntaxError();
@ -146,15 +143,14 @@ public:
// A helper class template for a rule in the form: <res> := [ <a> ]
template<class A>
class Opt: public A
template <class A>
class Opt : public A
{
public:
explicit Opt(const A &rhs)
:A(rhs)
{ }
explicit Opt(Parser *p)
:A(p)
public:
explicit Opt(const A& rhs) : A(rhs)
{
}
explicit Opt(Parser* p) : A(p)
{
if (A::isNull() && !p->syntaxError())
A::operator=(A::empty(p));
@ -169,7 +165,6 @@ public:
// M - mandatory - this part is required during parse time
// O - optional - this part is optional during parse time
// A helper class template for a rule in the form: <res> := <a> <b>
// i.e. both parts are mandatory at parse time
// The value of <a> is not important, and is created
@ -178,53 +173,52 @@ public:
// Example:
// <period> <unsigned integer>
template<class A, class B>
class UD2MM: public B
template <class A, class B>
class UD2MM : public B
{
public:
explicit UD2MM(Parser *p)
:B(A(p).isNull() ? B() :SetSyntaxErrorOnNull<B>(p))
{ }
explicit UD2MM(const B & b)
:B(b)
{ }
explicit UD2MM()
:B()
{ }
bool isNull() const { return B::isNull(); }
public:
explicit UD2MM(Parser* p) : B(A(p).isNull() ? B() : SetSyntaxErrorOnNull<B>(p))
{
}
explicit UD2MM(const B& b) : B(b)
{
}
explicit UD2MM() : B()
{
}
bool isNull() const
{
return B::isNull();
}
};
// A helper class template for a rule in the form: <res> := <a> <b>
// i.e. both parts are mandatory at parse time.
template<class A, class B>
class DD2MM: public A,
public B
template <class A, class B>
class DD2MM : public A, public B
{
public:
public:
// Sets syntax error if <a> was not followed by <b>
explicit DD2MM(Parser *p)
:A(p),
B(A::isNull() ? B() : SetSyntaxErrorOnNull<B>(p))
{ }
explicit DD2MM(const A & a, const B &b)
:A(b), B(b)
{ }
explicit DD2MM(Parser* p) : A(p), B(A::isNull() ? B() : SetSyntaxErrorOnNull<B>(p))
{
}
explicit DD2MM(const A& a, const B& b) : A(b), B(b)
{
}
};
// A helper class template for a rule in the form: <res> := <a> [ <b> ]
// i.e. <a> is mandatory, <b> is optional at parse time.
template<class A, class B>
class DD2MO: public A,
public B
template <class A, class B>
class DD2MO : public A, public B
{
public:
explicit DD2MO(Parser *p)
:A(p),
B(A::isNull() ? B() : B(p))
{ }
explicit DD2MO(const A &a, const B &b)
:A(a), B(b)
{ }
public:
explicit DD2MO(Parser* p) : A(p), B(A::isNull() ? B() : B(p))
{
}
explicit DD2MO(const A& a, const B& b) : A(a), B(b)
{
}
};
// A helper class template for a rule in the form: <res> := <a> [ <b> ]
@ -232,73 +226,68 @@ public:
// The value of <a> is not important and is not included
// into the target class, e.g.
// <period> [ <unsigned integer> ]
template<class A, class B>
class UD2MO: public B
template <class A, class B>
class UD2MO : public B
{
public:
explicit UD2MO(Parser *p)
:B(A(p).isNull() ? B() : B(p))
{ }
explicit UD2MO(const B &b)
:B(b)
{ }
explicit UD2MO()
:B()
{ }
public:
explicit UD2MO(Parser* p) : B(A(p).isNull() ? B() : B(p))
{
}
explicit UD2MO(const B& b) : B(b)
{
}
explicit UD2MO() : B()
{
}
};
// A helper class template for a rule in the form: <res> := <a> [ <b> ]
// i.e. <a> is mandatory, <b> is optional at parse time.
// The result class derives from "A".
// The result class puts "B" as a member.
template<class A, class B>
class DM2MO: public A
template <class A, class B>
class DM2MO : public A
{
protected:
protected:
B mB;
public:
explicit DM2MO(Parser *p)
:A(p),
mB(A::isNull() ? B() : B(p))
{ }
};
public:
explicit DM2MO(Parser* p) : A(p), mB(A::isNull() ? B() : B(p))
{
}
};
// A helper class template for a rule in the form: <res> := [ <a> ] <b>
// i.e. <a> is optional, <b> is mandatory at parse time.
template<class A,class B>
class DD2OM: public Opt<A>,
public B
template <class A, class B>
class DD2OM : public Opt<A>, public B
{
public:
explicit DD2OM(Parser *p)
:Opt<A>(p), B(p)
public:
explicit DD2OM(Parser* p) : Opt<A>(p), B(p)
{
if (B::isNull())
p->setSyntaxError();
}
explicit DD2OM()
:Opt<A>(A())
{ }
explicit DD2OM(const A & a, const B & b)
:Opt<A>(a), B(b)
{ }
explicit DD2OM() : Opt<A>(A())
{
}
explicit DD2OM(const A& a, const B& b) : Opt<A>(a), B(b)
{
}
};
// A helper class template for a rule in the form: <res> := a | b
template<class Container, class A, class B>
class Choice2: public Container
template <class Container, class A, class B>
class Choice2 : public Container
{
public:
explicit Choice2(const A & a)
:Container(a)
{ }
explicit Choice2(const B & b)
:Container(b)
{ }
explicit Choice2(Parser *p)
:Container(A(p))
public:
explicit Choice2(const A& a) : Container(a)
{
}
explicit Choice2(const B& b) : Container(b)
{
}
explicit Choice2(Parser* p) : Container(A(p))
{
if (Container::isNull() && !p->syntaxError())
*this = Choice2(B(p));
@ -306,6 +295,4 @@ public:
};
};
} // namespace genericparser
} // namespace genericparser

585
utils/common/hasher.h
View File

@ -38,331 +38,331 @@ namespace utils
inline uint32_t rotl32(uint32_t x, int8_t r)
{
return (x << r) | (x >> (32 - r));
return (x << r) | (x >> (32 - r));
}
inline uint32_t fmix(uint32_t h)
{
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
return h;
}
inline uint64_t fmix(uint64_t k)
{
k ^= k >> 33;
k *= 0xff51afd7ed558ccdULL;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53ULL;
k ^= k >> 33;
k ^= k >> 33;
k *= 0xff51afd7ed558ccdULL;
k ^= k >> 33;
k *= 0xc4ceb9fe1a85ec53ULL;
k ^= k >> 33;
return k;
return k;
}
inline uint64_t rotl64(uint64_t x, int8_t r)
{
return (x << r) | (x >> (64 - r));
return (x << r) | (x >> (64 - r));
}
class Hasher
{
public:
inline uint32_t operator()(const std::string& s) const
public:
inline uint32_t operator()(const std::string& s) const
{
return operator()(s.data(), s.length());
}
inline uint32_t operator()(const char* data, uint64_t len) const
{
const int nblocks = len / 4;
uint32_t h1 = 0;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
return operator()(s.data(), s.length());
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
inline uint32_t operator()(const char* data, uint64_t len) const
//----------
// tail
const uint8_t* tail = (const uint8_t*)(data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
const int nblocks = len / 4;
case 3:
k1 ^= tail[2] << 16;
/* fall through */
case 2:
k1 ^= tail[1] << 8;
/* fall through */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
uint32_t h1 = 0;
//----------
// finalization
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
h1 ^= len;
//----------
// body
h1 = fmix(h1);
const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*) (data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
case 3:
k1 ^= tail[2] << 16;
/* fall through */
case 2:
k1 ^= tail[1] << 8;
/* fall through */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h1 = fmix(h1);
return h1;
}
return h1;
}
};
class Hasher_r
{
public:
inline uint32_t operator()(const char* data, uint64_t len, uint32_t seed) const
public:
inline uint32_t operator()(const char* data, uint64_t len, uint32_t seed) const
{
const int nblocks = len / 4;
uint32_t h1 = seed;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
const int nblocks = len / 4;
uint32_t k1 = blocks[i];
uint32_t h1 = seed;
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
const uint32_t c1 = 0xcc9e2d51;
const uint32_t c2 = 0x1b873593;
//----------
// body
const uint32_t* blocks = (const uint32_t*) (data + nblocks * 4);
for (int i = -nblocks; i; i++)
{
uint32_t k1 = blocks[i];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*) (data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
case 3:
k1 ^= tail[2] << 16;
/* FALLTHRU */
case 2:
k1 ^= tail[1] << 8;
/* FALLTHRU */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
return h1;
h1 ^= k1;
h1 = rotl32(h1, 13);
h1 = h1 * 5 + 0xe6546b64;
}
inline uint32_t finalize(uint32_t seed, uint32_t len) const
//----------
// tail
const uint8_t* tail = (const uint8_t*)(data + nblocks * 4);
uint32_t k1 = 0;
switch (len & 3)
{
seed ^= len;
seed = fmix(seed);
return seed;
}
case 3:
k1 ^= tail[2] << 16;
/* FALLTHRU */
case 2:
k1 ^= tail[1] << 8;
/* FALLTHRU */
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = rotl32(k1, 15);
k1 *= c2;
h1 ^= k1;
};
return h1;
}
inline uint32_t finalize(uint32_t seed, uint32_t len) const
{
seed ^= len;
seed = fmix(seed);
return seed;
}
};
class Hasher128
{
public:
inline uint64_t operator()(const char* data, uint64_t len) const
public:
inline uint64_t operator()(const char* data, uint64_t len) const
{
const int nblocks = len / 16;
uint64_t h1 = 0;
uint64_t h2 = 0;
const uint64_t c1 = 0x87c37b91114253d5ULL;
const uint64_t c2 = 0x4cf5ad432745937fULL;
//----------
// body
const uint64_t* blocks = (const uint64_t*)(data);
for (int i = 0; i < nblocks; i++)
{
const int nblocks = len / 16;
uint64_t k1 = blocks[i * 2 + 0];
uint64_t k2 = blocks[i * 2 + 1];
uint64_t h1 = 0;
uint64_t h2 = 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
const uint64_t c1 = 0x87c37b91114253d5ULL;
const uint64_t c2 = 0x4cf5ad432745937fULL;
h1 = rotl64(h1, 27);
h1 += h2;
h1 = h1 * 5 + 0x52dce729;
//----------
// body
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
const uint64_t* blocks = (const uint64_t*) (data);
for (int i = 0; i < nblocks; i++)
{
uint64_t k1 = blocks[i * 2 + 0];
uint64_t k2 = blocks[i * 2 + 1];
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
h1 = rotl64(h1, 27);
h1 += h2;
h1 = h1 * 5 + 0x52dce729;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
h2 = rotl64(h2, 31);
h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*) (data + nblocks * 16);
uint64_t k1 = 0;
uint64_t k2 = 0;
switch (len & 15)
{
case 15:
k2 ^= uint64_t(tail[14]) << 48;
/* FALLTHRU */
case 14:
k2 ^= uint64_t(tail[13]) << 40;
/* FALLTHRU */
case 13:
k2 ^= uint64_t(tail[12]) << 32;
/* FALLTHRU */
case 12:
k2 ^= uint64_t(tail[11]) << 24;
/* FALLTHRU */
case 11:
k2 ^= uint64_t(tail[10]) << 16;
/* FALLTHRU */
case 10:
k2 ^= uint64_t(tail[9]) << 8;
/* FALLTHRU */
case 9:
k2 ^= uint64_t(tail[8]) << 0;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
/* FALLTHRU */
case 8:
k1 ^= uint64_t(tail[7]) << 56;
/* FALLTHRU */
case 7:
k1 ^= uint64_t(tail[6]) << 48;
/* FALLTHRU */
case 6:
k1 ^= uint64_t(tail[5]) << 40;
/* FALLTHRU */
case 5:
k1 ^= uint64_t(tail[4]) << 32;
/* FALLTHRU */
case 4:
k1 ^= uint64_t(tail[3]) << 24;
/* FALLTHRU */
case 3:
k1 ^= uint64_t(tail[2]) << 16;
/* FALLTHRU */
case 2:
k1 ^= uint64_t(tail[1]) << 8;
/* FALLTHRU */
case 1:
k1 ^= uint64_t(tail[0]) << 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h2 ^= len;
h1 += h2;
h2 += h1;
h1 = fmix(h1);
h2 = fmix(h2);
h1 += h2;
h2 += h1;
return h1;
h2 = rotl64(h2, 31);
h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
}
//----------
// tail
const uint8_t* tail = (const uint8_t*)(data + nblocks * 16);
uint64_t k1 = 0;
uint64_t k2 = 0;
switch (len & 15)
{
case 15:
k2 ^= uint64_t(tail[14]) << 48;
/* FALLTHRU */
case 14:
k2 ^= uint64_t(tail[13]) << 40;
/* FALLTHRU */
case 13:
k2 ^= uint64_t(tail[12]) << 32;
/* FALLTHRU */
case 12:
k2 ^= uint64_t(tail[11]) << 24;
/* FALLTHRU */
case 11:
k2 ^= uint64_t(tail[10]) << 16;
/* FALLTHRU */
case 10:
k2 ^= uint64_t(tail[9]) << 8;
/* FALLTHRU */
case 9:
k2 ^= uint64_t(tail[8]) << 0;
k2 *= c2;
k2 = rotl64(k2, 33);
k2 *= c1;
h2 ^= k2;
/* FALLTHRU */
case 8:
k1 ^= uint64_t(tail[7]) << 56;
/* FALLTHRU */
case 7:
k1 ^= uint64_t(tail[6]) << 48;
/* FALLTHRU */
case 6:
k1 ^= uint64_t(tail[5]) << 40;
/* FALLTHRU */
case 5:
k1 ^= uint64_t(tail[4]) << 32;
/* FALLTHRU */
case 4:
k1 ^= uint64_t(tail[3]) << 24;
/* FALLTHRU */
case 3:
k1 ^= uint64_t(tail[2]) << 16;
/* FALLTHRU */
case 2:
k1 ^= uint64_t(tail[1]) << 8;
/* FALLTHRU */
case 1:
k1 ^= uint64_t(tail[0]) << 0;
k1 *= c1;
k1 = rotl64(k1, 31);
k1 *= c2;
h1 ^= k1;
};
//----------
// finalization
h1 ^= len;
h2 ^= len;
h1 += h2;
h2 += h1;
h1 = fmix(h1);
h2 = fmix(h2);
h1 += h2;
h2 += h1;
return h1;
}
};
// TODO a copy of these classes also exists in primitiveprocessor.h; consolidate
class Hash128
{
public:
inline size_t operator()(const int128_t i) const
{
return (uint64_t)((int64_t) i);
}
public:
inline size_t operator()(const int128_t i) const
{
return (uint64_t)((int64_t)i);
}
};
class Equal128
{
public:
inline bool operator()(const int128_t f1, const int128_t f2) const
{
return f1 == f2;
}
public:
inline bool operator()(const int128_t f1, const int128_t f2) const
{
return f1 == f2;
}
};
//------------------------------------------------------------------------------
@ -372,17 +372,19 @@ public:
//------------------------------------------------------------------------------
class TupleHasher
{
public:
TupleHasher(uint32_t len) : fHashLen(len) {}
inline uint64_t operator()(const uint8_t* hashKey) const
{
return fHasher(reinterpret_cast<const char*>(hashKey), fHashLen);
}
private:
Hasher fHasher;
uint32_t fHashLen;
};
public:
TupleHasher(uint32_t len) : fHashLen(len)
{
}
inline uint64_t operator()(const uint8_t* hashKey) const
{
return fHasher(reinterpret_cast<const char*>(hashKey), fHashLen);
}
private:
Hasher fHasher;
uint32_t fHashLen;
};
//------------------------------------------------------------------------------
/** @brief class TupleComparator
@ -391,16 +393,17 @@ private:
//------------------------------------------------------------------------------
class TupleComparator
{
public:
TupleComparator(uint32_t len) : fCmpLen(len) {}
inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const
{
return (memcmp(hashKey1, hashKey2, fCmpLen) == 0);
}
private:
uint32_t fCmpLen;
public:
TupleComparator(uint32_t len) : fCmpLen(len)
{
}
inline bool operator()(const uint8_t* hashKey1, const uint8_t* hashKey2) const
{
return (memcmp(hashKey1, hashKey2, fCmpLen) == 0);
}
private:
uint32_t fCmpLen;
};
}
} // namespace utils

46
utils/common/hashfamily.h
View File

@ -22,34 +22,32 @@
namespace utils
{
class HashFamily
{
public:
HashFamily(const utils::Hasher_r& h,
const uint64_t intermediateHash,
const uint64_t len,
const datatypes::MariaDBHasher& hM) : mHasher(h),
mMariaDBHasher(hM),
mHasher_rHash(intermediateHash),
mHasher_rLen(len)
{ }
public:
HashFamily(const utils::Hasher_r& h, const uint64_t intermediateHash, const uint64_t len,
const datatypes::MariaDBHasher& hM)
: mHasher(h), mMariaDBHasher(hM), mHasher_rHash(intermediateHash), mHasher_rLen(len)
{
}
// Algorithm, seed and factor are taken from this discussion
// https://stackoverflow.com/questions/1646807/quick-and-simple-hash-code-combinations
inline uint64_t finalize() const
{
return (seed * factor + mHasher.finalize(mHasher_rHash, mHasher_rLen)) * factor + mMariaDBHasher.finalize();
}
private:
constexpr static uint64_t seed = 1009ULL;
constexpr static uint64_t factor = 9176ULL;
// Algorithm, seed and factor are taken from this discussion
// https://stackoverflow.com/questions/1646807/quick-and-simple-hash-code-combinations
inline uint64_t finalize() const
{
return (seed * factor + mHasher.finalize(mHasher_rHash, mHasher_rLen)) * factor +
mMariaDBHasher.finalize();
}
const utils::Hasher_r& mHasher;
const datatypes::MariaDBHasher& mMariaDBHasher;
const uint64_t mHasher_rHash;
const uint32_t mHasher_rLen;
private:
constexpr static uint64_t seed = 1009ULL;
constexpr static uint64_t factor = 9176ULL;
const utils::Hasher_r& mHasher;
const datatypes::MariaDBHasher& mMariaDBHasher;
const uint64_t mHasher_rHash;
const uint32_t mHasher_rLen;
};
}
} // namespace utils
// vim:ts=2 sw=2:

1
utils/common/mariadb_my_sys.h
View File

@ -34,4 +34,3 @@
#include <my_sys.h>
#include "mcsconfig_conflicting_defs_restore.h"

6
utils/common/mcs_basic_types.h
View File

@ -14,13 +14,11 @@
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA.
*/
MA 02110-1301, USA.
*/
#pragma once
using int128_t = __int128;
using uint128_t = unsigned __int128;
// vim:ts=2 sw=2:

363
utils/common/nullvaluemanip.cpp
View File

@ -16,7 +16,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "nullvaluemanip.h"
#include <sstream>
@ -25,234 +24,180 @@ using namespace execplan;
namespace utils
{
uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth)
{
switch (t)
switch (t)
{
case CalpontSystemCatalog::TINYINT: return joblist::TINYINTNULL;
case CalpontSystemCatalog::SMALLINT: return joblist::SMALLINTNULL;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT: return joblist::INTNULL;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT: return joblist::FLOATNULL;
case CalpontSystemCatalog::DATE: return joblist::DATENULL;
case CalpontSystemCatalog::BIGINT: return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP: return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME: return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
case CalpontSystemCatalog::TINYINT:
return joblist::TINYINTNULL;
switch (colWidth)
{
case 1: return joblist::CHAR1NULL;
case CalpontSystemCatalog::SMALLINT:
return joblist::SMALLINTNULL;
case 2: return joblist::CHAR2NULL;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return joblist::INTNULL;
case 3:
case 4: return joblist::CHAR4NULL;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return joblist::FLOATNULL;
case 5:
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
case CalpontSystemCatalog::DATE:
return joblist::DATENULL;
default: throw logic_error("getNullValue() Can't return the NULL string");
}
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP:
return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME:
return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
switch (colWidth)
{
case 1:
return joblist::CHAR1NULL;
case 2:
return joblist::CHAR2NULL;
case 3:
case 4:
return joblist::CHAR4NULL;
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("getNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1 :
return joblist::TINYINTNULL;
case 2 :
return joblist::SMALLINTNULL;
case 4 :
return joblist::INTNULL;
default:
return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return joblist::UTINYINTNULL;
case CalpontSystemCatalog::USMALLINT:
return joblist::USMALLINTNULL;
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return joblist::UINTNULL;
case CalpontSystemCatalog::UBIGINT:
return joblist::UBIGINTNULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1: return joblist::TINYINTNULL;
case 2: return joblist::SMALLINTNULL;
case 4: return joblist::INTNULL;
default: return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT: return joblist::UTINYINTNULL;
case CalpontSystemCatalog::USMALLINT: return joblist::USMALLINTNULL;
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT: return joblist::UINTNULL;
case CalpontSystemCatalog::UBIGINT: return joblist::UBIGINTNULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl;
throw logic_error(os.str());
}
}
int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth)
{
switch (t)
switch (t)
{
case CalpontSystemCatalog::TINYINT: return (int64_t)((int8_t)joblist::TINYINTNULL);
case CalpontSystemCatalog::SMALLINT: return (int64_t)((int16_t)joblist::SMALLINTNULL);
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT: return (int64_t)((int32_t)joblist::INTNULL);
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT: return (int64_t)((int32_t)joblist::FLOATNULL);
case CalpontSystemCatalog::DATE: return (int64_t)((int32_t)joblist::DATENULL);
case CalpontSystemCatalog::BIGINT: return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE: return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME: return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP: return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME: return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
case CalpontSystemCatalog::TINYINT:
return (int64_t) ((int8_t) joblist::TINYINTNULL);
switch (colWidth)
{
case 1: return (int64_t)((int8_t)joblist::CHAR1NULL);
case CalpontSystemCatalog::SMALLINT:
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case 2: return (int64_t)((int16_t)joblist::CHAR2NULL);
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return (int64_t) ((int32_t) joblist::INTNULL);
case 3:
case 4: return (int64_t)((int32_t)joblist::CHAR4NULL);
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return (int64_t) ((int32_t) joblist::FLOATNULL);
case 5:
case 6:
case 7:
case 8: return joblist::CHAR8NULL;
case CalpontSystemCatalog::DATE:
return (int64_t) ((int32_t) joblist::DATENULL);
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::TIMESTAMP:
return joblist::TIMESTAMPNULL;
case CalpontSystemCatalog::TIME:
return joblist::TIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
switch (colWidth)
{
case 1:
return (int64_t) ((int8_t) joblist::CHAR1NULL);
case 2:
return (int64_t) ((int16_t) joblist::CHAR2NULL);
case 3:
case 4:
return (int64_t) ((int32_t) joblist::CHAR4NULL);
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("getSignedNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1 :
return (int64_t) ((int8_t) joblist::TINYINTNULL);
case 2 :
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case 4 :
return (int64_t) ((int32_t) joblist::INTNULL);
case 8:
return joblist::BIGINTNULL;
default:
ostringstream os;
os << "getSignedNullValue(): got bad column width (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return (int64_t) ((int8_t) joblist::UTINYINTNULL);
case CalpontSystemCatalog::USMALLINT:
return (int64_t) ((int16_t) joblist::USMALLINTNULL);
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return (int64_t) ((int32_t) joblist::UINTNULL);
case CalpontSystemCatalog::UBIGINT:
return (int64_t)joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE:
return (int64_t)joblist::LONGDOUBLENULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getSignedNullValue(): got bad column type (" << t <<
"). Width=" << colWidth << endl;
throw logic_error(os.str());
default: throw logic_error("getSignedNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
switch (colWidth)
{
case 1: return (int64_t)((int8_t)joblist::TINYINTNULL);
case 2: return (int64_t)((int16_t)joblist::SMALLINTNULL);
case 4: return (int64_t)((int32_t)joblist::INTNULL);
case 8: return joblist::BIGINTNULL;
default:
ostringstream os;
os << "getSignedNullValue(): got bad column width (" << t << "). Width=" << colWidth << endl;
throw logic_error(os.str());
}
break;
}
case CalpontSystemCatalog::UTINYINT: return (int64_t)((int8_t)joblist::UTINYINTNULL);
case CalpontSystemCatalog::USMALLINT: return (int64_t)((int16_t)joblist::USMALLINTNULL);
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT: return (int64_t)((int32_t)joblist::UINTNULL);
case CalpontSystemCatalog::UBIGINT: return (int64_t)joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE: return (int64_t)joblist::LONGDOUBLENULL;
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "getSignedNullValue(): got bad column type (" << t << "). Width=" << colWidth << endl;
throw logic_error(os.str());
}
}
}
} // namespace utils

6
utils/common/nullvaluemanip.h
View File

@ -15,19 +15,15 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "../../dbcon/execplan/calpontsystemcatalog.h"
namespace utils
{
// returns the NULL value for our 'numeric' types including short strings.
// The width is only relevant for long string columns.
uint64_t getNullValue(execplan::CalpontSystemCatalog::ColDataType, uint32_t colWidth = 0);
int64_t getSignedNullValue(execplan::CalpontSystemCatalog::ColDataType, uint32_t colWidth = 0);
}
} // namespace utils

49
utils/common/pipe.h
View File

@ -23,11 +23,12 @@
class Pipe
{
int fd[2];
public:
public:
Pipe()
{
fd[0]= 0;
fd[1]= 0;
fd[0] = 0;
fd[1] = 0;
}
~Pipe()
{
@ -45,35 +46,35 @@ public:
{
return ::pipe(fd) == -1;
}
bool init() // TODO: remove this
bool init() // TODO: remove this
{
return open();
}
ssize_t read(char *str, size_t nbytes)
ssize_t read(char* str, size_t nbytes)
{
return ::read(fd[0], str, nbytes);
}
ssize_t readtm(const struct timeval &tv, void *buf, size_t nbytes)
ssize_t readtm(const struct timeval& tv, void* buf, size_t nbytes)
{
fd_set rfds;
FD_ZERO(&rfds);
FD_SET(fd[0], &rfds);
struct timeval tmptv = tv;
int retval = select(fd[0] + 1, &rfds, NULL, NULL, &tmptv);
if (retval == -1)
return -1;
if (!retval)
{
errno= ETIME;
return -1;
}
return ::read(fd[0], buf, nbytes);
fd_set rfds;
FD_ZERO(&rfds);
FD_SET(fd[0], &rfds);
struct timeval tmptv = tv;
int retval = select(fd[0] + 1, &rfds, NULL, NULL, &tmptv);
if (retval == -1)
return -1;
if (!retval)
{
errno = ETIME;
return -1;
}
return ::read(fd[0], buf, nbytes);
}
ssize_t write(const char *str, size_t nbytes)
ssize_t write(const char* str, size_t nbytes)
{
return ::write(fd[1], str, nbytes);
}
ssize_t write(const std::string &str)
ssize_t write(const std::string& str)
{
return write(str.data(), str.length());
}
@ -82,14 +83,12 @@ public:
if (fd[0])
{
::close(fd[0]);
fd[0]= 0;
fd[0] = 0;
}
if (fd[1])
{
::close(fd[1]);
fd[1]= 0;
fd[1] = 0;
}
}
};

95
utils/common/poolallocator.cpp
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
#include <iostream>
//#define NDEBUG
@ -31,72 +31,71 @@ using namespace boost;
namespace utils
{
PoolAllocator& PoolAllocator::operator=(const PoolAllocator& v)
{
allocSize = v.allocSize;
tmpSpace = v.tmpSpace;
useLock = v.useLock;
deallocateAll();
return *this;
allocSize = v.allocSize;
tmpSpace = v.tmpSpace;
useLock = v.useLock;
deallocateAll();
return *this;
}
void PoolAllocator::deallocateAll()
{
capacityRemaining = 0;
nextAlloc = NULL;
memUsage = 0;
mem.clear();
oob.clear();
capacityRemaining = 0;
nextAlloc = NULL;
memUsage = 0;
mem.clear();
oob.clear();
}
void PoolAllocator::newBlock()
{
shared_array<uint8_t> next;
shared_array<uint8_t> next;
capacityRemaining = allocSize;
capacityRemaining = allocSize;
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[allocSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
nextAlloc = mem.front().get();
if (!tmpSpace || mem.size() == 0)
{
next.reset(new uint8_t[allocSize]);
mem.push_back(next);
nextAlloc = next.get();
}
else
nextAlloc = mem.front().get();
}
void * PoolAllocator::allocOOB(uint64_t size)
void* PoolAllocator::allocOOB(uint64_t size)
{
OOBMemInfo memInfo;
OOBMemInfo memInfo;
memUsage += size;
memInfo.mem.reset(new uint8_t[size]);
memInfo.size = size;
void *ret = (void*) memInfo.mem.get();
oob[ret] = memInfo;
return ret;
memUsage += size;
memInfo.mem.reset(new uint8_t[size]);
memInfo.size = size;
void* ret = (void*)memInfo.mem.get();
oob[ret] = memInfo;
return ret;
}
void PoolAllocator::deallocate(void* p)
{
bool _false = false;
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
OutOfBandMap::iterator it = oob.find(p);
bool _false = false;
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
OutOfBandMap::iterator it = oob.find(p);
if (it == oob.end())
{
if (useLock)
lock.store(false, std::memory_order_release);
return;
}
memUsage -= it->second.size;
oob.erase(it);
if (it == oob.end())
{
if (useLock)
lock.store(false, std::memory_order_release);
lock.store(false, std::memory_order_release);
return;
}
memUsage -= it->second.size;
oob.erase(it);
if (useLock)
lock.store(false, std::memory_order_release);
}
}
} // namespace utils

167
utils/common/poolallocator.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/* This allocator is an attempt to consolidate small allocations and
deallocations to boost performance and reduce mem fragmentation. */
@ -34,100 +34,105 @@
namespace utils
{
class PoolAllocator
{
public:
static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages
public:
static const unsigned DEFAULT_WINDOW_SIZE = 4096 * 40; // should be an integral # of pages
explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false, bool _useLock = false) :
allocSize(windowSize),
tmpSpace(isTmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0),
useLock(_useLock),
lock(false) { }
PoolAllocator(const PoolAllocator& p) :
allocSize(p.allocSize),
tmpSpace(p.tmpSpace),
capacityRemaining(0),
memUsage(0),
nextAlloc(0),
useLock(p.useLock),
lock(false) { }
virtual ~PoolAllocator() {}
explicit PoolAllocator(unsigned windowSize = DEFAULT_WINDOW_SIZE, bool isTmpSpace = false,
bool _useLock = false)
: allocSize(windowSize)
, tmpSpace(isTmpSpace)
, capacityRemaining(0)
, memUsage(0)
, nextAlloc(0)
, useLock(_useLock)
, lock(false)
{
}
PoolAllocator(const PoolAllocator& p)
: allocSize(p.allocSize)
, tmpSpace(p.tmpSpace)
, capacityRemaining(0)
, memUsage(0)
, nextAlloc(0)
, useLock(p.useLock)
, lock(false)
{
}
virtual ~PoolAllocator()
{
}
PoolAllocator& operator=(const PoolAllocator&);
PoolAllocator& operator=(const PoolAllocator&);
void* allocate(uint64_t size);
void deallocate(void* p);
void deallocateAll();
void* allocate(uint64_t size);
void deallocate(void* p);
void deallocateAll();
inline uint64_t getMemUsage() const
{
return memUsage;
}
unsigned getWindowSize() const
{
return allocSize;
}
inline uint64_t getMemUsage() const
{
return memUsage;
}
unsigned getWindowSize() const
{
return allocSize;
}
void setUseLock(bool ul)
{
useLock = ul;
}
void setUseLock(bool ul)
{
useLock = ul;
}
private:
void newBlock();
void *allocOOB(uint64_t size);
private:
void newBlock();
void* allocOOB(uint64_t size);
unsigned allocSize;
std::vector<boost::shared_array<uint8_t> > mem;
bool tmpSpace;
unsigned capacityRemaining;
uint64_t memUsage;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
unsigned allocSize;
std::vector<boost::shared_array<uint8_t> > mem;
bool tmpSpace;
unsigned capacityRemaining;
uint64_t memUsage;
uint8_t* nextAlloc;
bool useLock;
std::atomic<bool> lock;
struct OOBMemInfo
{
boost::shared_array<uint8_t> mem;
uint64_t size;
};
typedef std::map<void*, OOBMemInfo> OutOfBandMap;
OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd
struct OOBMemInfo
{
boost::shared_array<uint8_t> mem;
uint64_t size;
};
typedef std::map<void*, OOBMemInfo> OutOfBandMap;
OutOfBandMap oob; // for mem chunks bigger than the window size; these can be dealloc'd
};
inline void* PoolAllocator::allocate(uint64_t size)
{
void *ret;
bool _false = false;
void* ret;
bool _false = false;
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
if (size > allocSize)
{
ret = allocOOB(size);
if (useLock)
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
if (size > allocSize)
{
ret = allocOOB(size);
if (useLock)
lock.store(false, std::memory_order_release);
return ret;
}
if (size > capacityRemaining)
newBlock();
ret = (void*) nextAlloc;
nextAlloc += size;
capacityRemaining -= size;
memUsage += size;
if (useLock)
lock.store(false, std::memory_order_release);
lock.store(false, std::memory_order_release);
return ret;
}
if (size > capacityRemaining)
newBlock();
ret = (void*)nextAlloc;
nextAlloc += size;
capacityRemaining -= size;
memUsage += size;
if (useLock)
lock.store(false, std::memory_order_release);
return ret;
}
}
} // namespace utils

4143
utils/common/robin_hood.h
View File

File diff suppressed because it is too large Load Diff

141
utils/common/service.h
View File

@ -20,95 +20,94 @@
#include <signal.h>
#include "pipe.h"
class Service
{
protected:
// The service name, for logging
const std::string m_name;
// The pipe to send messages from the child to the parent
Pipe m_pipe;
protected:
// The service name, for logging
const std::string m_name;
// The pipe to send messages from the child to the parent
Pipe m_pipe;
static void common_signal_handler_CHLD(int sig)
{
}
static void common_signal_handler_CHLD(int sig)
{ }
void InitCommonSignalHandlers()
{
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = common_signal_handler_CHLD;
sigaction(SIGCHLD, &sa, NULL);
}
void InitCommonSignalHandlers()
int RunForking()
{
int err;
InitCommonSignalHandlers();
if (m_pipe.open() || (err = fork()) < 0)
{
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = common_signal_handler_CHLD;
sigaction(SIGCHLD, &sa, NULL);
// Pipe or fork failed
LogErrno();
return 1;
}
int RunForking()
if (err > 0) // Parent
return Parent();
return Child();
}
virtual int Parent()
{
char str[100];
// Read the message from the child
ssize_t nbytes = m_pipe.readtm({120, 0}, str, sizeof(str));
if (nbytes >= 0)
{
int err;
InitCommonSignalHandlers();
if (m_pipe.open() || (err = fork()) < 0)
{
// Pipe or fork failed
LogErrno();
return 1;
}
if (err > 0) // Parent
return Parent();
return Child();
ParentLogChildMessage(std::string(str, nbytes));
}
virtual int Parent()
else
{
char str[100];
// Read the message from the child
ssize_t nbytes= m_pipe.readtm({120,0}, str, sizeof(str));
if (nbytes >= 0)
{
ParentLogChildMessage(std::string(str, nbytes));
}
else
{
// read() failed
LogErrno();
return 1;
}
return 0;
// read() failed
LogErrno();
return 1;
}
return 0;
}
public:
Service(const std::string &name)
:m_name(name)
{ }
virtual ~Service() { }
public:
Service(const std::string& name) : m_name(name)
{
}
virtual ~Service()
{
}
void NotifyServiceStarted()
void NotifyServiceStarted()
{
if (m_pipe.is_open_for_write())
{
if (m_pipe.is_open_for_write())
{
std::ostringstream str;
str << m_name << " main process has started";
m_pipe.write(str.str());
}
std::ostringstream str;
str << m_name << " main process has started";
m_pipe.write(str.str());
}
}
void NotifyServiceInitializationFailed()
void NotifyServiceInitializationFailed()
{
if (m_pipe.is_open_for_write())
{
if (m_pipe.is_open_for_write())
{
std::ostringstream str;
str << m_name << " main process initialization failed";
m_pipe.write(str.str());
}
std::ostringstream str;
str << m_name << " main process initialization failed";
m_pipe.write(str.str());
}
}
// Used by both Parent and Child to log errors
virtual void LogErrno()= 0;
// Used by Parent to log an initialization notification message from child
virtual void ParentLogChildMessage(const std::string &str)= 0;
// The main service process job
virtual int Child()= 0;
// Used by both Parent and Child to log errors
virtual void LogErrno() = 0;
// Used by Parent to log an initialization notification message from child
virtual void ParentLogChildMessage(const std::string& str) = 0;
// The main service process job
virtual int Child() = 0;
};

829
utils/common/simd_sse.h
View File

@ -17,455 +17,456 @@
#pragma once
#if defined(__x86_64__ )
#if defined(__x86_64__)
#include <cstdint>
#include <type_traits>
#ifdef __OPTIMIZE__
#include <smmintrin.h>
#include <emmintrin.h>
#define MCS_FORCE_INLINE __attribute__((__always_inline__))
#include <smmintrin.h>
#include <emmintrin.h>
#define MCS_FORCE_INLINE __attribute__((__always_inline__))
#else
#define __OPTIMIZE__
#include <smmintrin.h>
#include <emmintrin.h>
#undef __OPTIMIZE__
#define MCS_FORCE_INLINE inline
#define __OPTIMIZE__
#include <smmintrin.h>
#include <emmintrin.h>
#undef __OPTIMIZE__
#define MCS_FORCE_INLINE inline
#endif
#include <mcs_datatype.h>
namespace simd
{
using vi128_t = __m128i;
using msk128_t = uint16_t;
using int128_t = __int128;
using MT = uint16_t;
// This ugly wrapper used to allow to use __m128i as a template class parameter argument
struct vi128_wr
using vi128_t = __m128i;
using msk128_t = uint16_t;
using int128_t = __int128;
using MT = uint16_t;
// This ugly wrapper used to allow to use __m128i as a template class parameter argument
struct vi128_wr
{
__m128i v;
};
template <typename VT, int WIDTH>
class SimdFilterProcessor
{
};
template <>
class SimdFilterProcessor<vi128_wr, 16>
{
// This is a dummy class that is not currently used.
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = int128_t;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
__m128i v;
};
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(&fill));
}
template<typename VT, int WIDTH>
class SimdFilterProcessor
{ };
template<>
class SimdFilterProcessor<vi128_wr, 16>
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
// This is a dummy class that is not currently used.
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = int128_t;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(&fill));
}
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
MCS_FORCE_INLINE MT cmpDummy(vi128_t& x, vi128_t& y)
{
return 0xFFFF;
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE uint16_t convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 8>
MCS_FORCE_INLINE MT cmpDummy(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<8>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set_epi64x(fill, fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_or_si128(_mm_cmpgt_epi64(x, y),_mm_cmpeq_epi64(x, y)));
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi64(x, y));
}
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpNe(x, y) ^ cmpGt(x, y);
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 4>
return 0xFFFF;
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<4>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi32(fill);
}
return cmpDummy(x, y);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 2>
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<2>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi16(fill);
}
return cmpDummy(x, y);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template<>
class SimdFilterProcessor<vi128_wr, 1>
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<1>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi8(fill);
}
return cmpDummy(x, y);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return cmpDummy(x, y);
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
// misc
MCS_FORCE_INLINE uint16_t convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi8(x, y));
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF;
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
// permute
/* TODO Available in AVX-512
MCS_FORCE_INLINE vi128_t perm8Bits(vi128_t& x, vi128_t& idx)
{
return _mm_permutexvar_epi8(x, idx);
}
*/
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
template <>
class SimdFilterProcessor<vi128_wr, 8>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<8>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set_epi64x(fill, fill);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
// Compare
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_or_si128(_mm_cmpgt_epi64(x, y), _mm_cmpeq_epi64(x, y)));
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi64(x, y));
}
} // end of simd
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y));
}
#endif // if defined(__x86_64__ )
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return cmpNe(x, y) ^ cmpGt(x, y);
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi64(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template <>
class SimdFilterProcessor<vi128_wr, 4>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<4>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi32(fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi32(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi32(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template <>
class SimdFilterProcessor<vi128_wr, 2>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<2>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi16(fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi16(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi16(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
template <>
class SimdFilterProcessor<vi128_wr, 1>
{
public:
constexpr static const uint16_t vecByteSize = 16U;
constexpr static const uint16_t vecBitSize = 128U;
using T = datatypes::WidthToSIntegralType<1>::type;
using SIMD_WRAPPER_TYPE = simd::vi128_wr;
using SIMD_TYPE = simd::vi128_t;
// Load value
MCS_FORCE_INLINE vi128_t loadValue(const T fill)
{
return _mm_set1_epi8(fill);
}
// Load from
MCS_FORCE_INLINE vi128_t loadFrom(const char* from)
{
return _mm_loadu_si128(reinterpret_cast<const vi128_t*>(from));
}
// Compare
MCS_FORCE_INLINE MT cmpEq(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpGe(vi128_t& x, vi128_t& y)
{
return cmpLt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpGt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpgt_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpLe(vi128_t& x, vi128_t& y)
{
return cmpGt(x, y) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpLt(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmplt_epi8(x, y));
}
MCS_FORCE_INLINE MT cmpNe(vi128_t& x, vi128_t& y)
{
return _mm_movemask_epi8(_mm_cmpeq_epi8(x, y)) ^ 0xFFFF;
}
MCS_FORCE_INLINE MT cmpAlwaysFalse(vi128_t& x, vi128_t& y)
{
return 0;
}
// permute
/* TODO Available in AVX-512
MCS_FORCE_INLINE vi128_t perm8Bits(vi128_t& x, vi128_t& idx)
{
return _mm_permutexvar_epi8(x, idx);
}
*/
// misc
MCS_FORCE_INLINE MT convertVectorToBitMask(vi128_t& vmask)
{
return _mm_movemask_epi8(vmask);
}
MCS_FORCE_INLINE vi128_t setToZero()
{
return _mm_setzero_si128();
}
// store
MCS_FORCE_INLINE void storeWMask(vi128_t& x, vi128_t& vmask, char* dst)
{
_mm_maskmoveu_si128(x, vmask, dst);
}
MCS_FORCE_INLINE void store(char* dst, vi128_t& x)
{
_mm_storeu_si128(reinterpret_cast<vi128_t*>(dst), x);
}
};
} // namespace simd
#endif // if defined(__x86_64__ )
// vim:ts=2 sw=2:

289
utils/common/simpleallocator.h
View File

@ -37,11 +37,11 @@
namespace utils
{
// A specialized allocator for std::tr1::unordered_multimap<uint64_t, uint64_t> based joiner
// or std::tr1::unordered_map<uint8_t*, uint8_t*> based aggregation.
// User shall initialize a pool and pass it to allocator, release the pool when map is done.
template<typename T> class SimpleAllocator;
template <typename T>
class SimpleAllocator;
// this pool is best for node size of 3*sizeof(int64).
// map nodes are taken from fixed size blocks, and control hash tables are from ::new.
@ -51,208 +51,211 @@ template<typename T> class SimpleAllocator;
#define OPT_NODE_UNITS 10
class SimplePool
{
public:
SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0) {}
~SimplePool()
{
reset();
}
public:
SimplePool() : fNext(NULL), fEnd(NULL), fTableMemSize(0)
{
}
~SimplePool()
{
reset();
}
inline void* allocate(size_t n, const void* = 0);
inline void deallocate(void* p, size_t n);
inline size_t max_size() const throw();
inline uint64_t getMemUsage() const;
inline void* allocate(size_t n, const void* = 0);
inline void deallocate(void* p, size_t n);
inline size_t max_size() const throw();
inline uint64_t getMemUsage() const;
private:
static const size_t fUnitPerChunk = OPT_NODE_UNITS * 10240;
private:
static const size_t fUnitPerChunk = OPT_NODE_UNITS * 10240;
inline void reset();
inline void allocateNewChunk();
inline void reset();
inline void allocateNewChunk();
// MemUnit stores a pointer to next unit before allocated, and T after allocated.
union MemUnit
{
MemUnit* fNext;
uint64_t fData;
}* fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit
// MemUnit stores a pointer to next unit before allocated, and T after allocated.
union MemUnit
{
MemUnit* fNext;
uint64_t fData;
} * fNext, *fEnd; // fNext: next available unit, fEnd: one off the last unit
std::list<MemUnit*> fBlockList;
uint64_t fTableMemSize;
std::list<MemUnit*> fBlockList;
uint64_t fTableMemSize;
static const size_t fUnitSize = sizeof(MemUnit);
static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS;
static const size_t fChunkSize = fUnitSize * fUnitPerChunk;
static const size_t fUnitSize = sizeof(MemUnit);
static const size_t fMaxNodeSize = fUnitSize * OPT_NODE_UNITS;
static const size_t fChunkSize = fUnitSize * fUnitPerChunk;
};
template<typename T>
template <typename T>
class SimpleAllocator
{
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<typename U> struct rebind
{
typedef SimpleAllocator<U> other;
};
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template <typename U>
struct rebind
{
typedef SimpleAllocator<U> other;
};
SimpleAllocator() throw() {}
SimpleAllocator(std::shared_ptr<SimplePool> pool) throw()
{
fPool = pool;
}
SimpleAllocator(const SimpleAllocator& alloc)
{
fPool = alloc.fPool;
}
template<class U> SimpleAllocator(const SimpleAllocator<U>& alloc)
{
fPool = alloc.fPool;
}
SimpleAllocator() throw()
{
}
SimpleAllocator(std::shared_ptr<SimplePool> pool) throw()
{
fPool = pool;
}
SimpleAllocator(const SimpleAllocator& alloc)
{
fPool = alloc.fPool;
}
template <class U>
SimpleAllocator(const SimpleAllocator<U>& alloc)
{
fPool = alloc.fPool;
}
~SimpleAllocator() throw() { }
~SimpleAllocator() throw()
{
}
inline pointer address(reference x) const
{
return &x;
}
inline const_pointer address(const_reference x) const
{
return &x;
}
inline pointer address(reference x) const
{
return &x;
}
inline const_pointer address(const_reference x) const
{
return &x;
}
inline pointer allocate(size_type n, const void* = 0)
{
return static_cast<pointer>(fPool->allocate(n * sizeof(T)));
}
inline void deallocate(pointer p, size_type n)
{
fPool->deallocate(p, n * sizeof(T));
}
inline pointer allocate(size_type n, const void* = 0)
{
return static_cast<pointer>(fPool->allocate(n * sizeof(T)));
}
inline void deallocate(pointer p, size_type n)
{
fPool->deallocate(p, n * sizeof(T));
}
#ifdef _MSC_VER
//The MSVC STL library really needs this to return a big number...
size_type max_size() const throw()
{
return std::numeric_limits<size_type>::max();
}
// The MSVC STL library really needs this to return a big number...
size_type max_size() const throw()
{
return std::numeric_limits<size_type>::max();
}
#else
inline size_type max_size() const throw()
{
return fPool->max_size() / sizeof(T);
}
inline size_type max_size() const throw()
{
return fPool->max_size() / sizeof(T);
}
#endif
inline void construct(pointer ptr, const T& val)
{
new ((void*)ptr) T(val);
}
inline void destroy(pointer ptr)
{
ptr->T::~T();
}
inline void construct(pointer ptr, const T& val)
{
new ((void*)ptr) T(val);
}
inline void destroy(pointer ptr)
{
ptr->T::~T();
}
inline void setPool(SimplePool* pool)
{
fPool.reset(pool);
}
inline void setPool(SimplePool* pool)
{
fPool.reset(pool);
}
std::shared_ptr<SimplePool> fPool;
std::shared_ptr<SimplePool> fPool;
};
// inlines
inline void* SimplePool::allocate(size_t n, const void* dur)
{
// make sure the block allocated is on unit boundary
size_t unitCount = n / fUnitSize;
// make sure the block allocated is on unit boundary
size_t unitCount = n / fUnitSize;
if ((n % fUnitSize) != 0)
unitCount += 1;
if ((n % fUnitSize) != 0)
unitCount += 1;
// if for control table, let new allocator handle it.
if (unitCount > OPT_NODE_UNITS)
// if for control table, let new allocator handle it.
if (unitCount > OPT_NODE_UNITS)
{
fTableMemSize += n;
return new uint8_t[n];
}
// allocate node
MemUnit* curr = fNext;
do
{
if (curr == NULL)
{
fTableMemSize += n;
return new uint8_t[n];
allocateNewChunk();
curr = fNext;
}
// allocate node
MemUnit* curr = fNext;
fNext = curr + unitCount;
do
{
if (curr == NULL)
{
allocateNewChunk();
curr = fNext;
}
if (fNext > fEnd)
curr = NULL;
} while (!curr);
fNext = curr + unitCount;
if (fNext > fEnd)
curr = NULL;
}
while (!curr);
return curr;
return curr;
}
inline void SimplePool::deallocate(void* p, size_t n)
{
// only delete the old control table, which is allocated by new allocator.
if (n > fMaxNodeSize)
{
fTableMemSize -= n;
delete [] (static_cast<uint8_t*>(p));
}
// only delete the old control table, which is allocated by new allocator.
if (n > fMaxNodeSize)
{
fTableMemSize -= n;
delete[](static_cast<uint8_t*>(p));
}
}
inline size_t SimplePool::max_size() const throw()
{
return fUnitSize * fUnitPerChunk;
return fUnitSize * fUnitPerChunk;
}
inline uint64_t SimplePool::getMemUsage() const
{
return fTableMemSize + fBlockList.size() * fChunkSize +
// add list overhead, element type is a pointer, and
// lists store a next pointer.
fBlockList.size() * 2 * sizeof(void*);
return fTableMemSize + fBlockList.size() * fChunkSize +
// add list overhead, element type is a pointer, and
// lists store a next pointer.
fBlockList.size() * 2 * sizeof(void*);
}
inline void SimplePool::reset()
{
for (std::list<MemUnit*>::iterator i = fBlockList.begin(); i != fBlockList.end(); i++)
delete [] (*i);
for (std::list<MemUnit*>::iterator i = fBlockList.begin(); i != fBlockList.end(); i++)
delete[](*i);
fNext = NULL;
fEnd = NULL;
fNext = NULL;
fEnd = NULL;
}
inline void SimplePool::allocateNewChunk()
{
MemUnit* chunk = new MemUnit[fUnitPerChunk];
fBlockList.push_back(chunk);
fNext = chunk;
fEnd = chunk + fUnitPerChunk;
MemUnit* chunk = new MemUnit[fUnitPerChunk];
fBlockList.push_back(chunk);
fNext = chunk;
fEnd = chunk + fUnitPerChunk;
}
template <typename T1, typename T2>
inline bool operator==(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&)
{
return true;
return true;
}
template <typename T1, typename T2>
inline bool operator!=(const SimpleAllocator<T1>&, const SimpleAllocator<T2>&)
{
return false;
return false;
}
}
} // namespace utils

23
utils/common/spinlock.h
View File

@ -4,24 +4,23 @@
namespace utils
{
inline void getSpinlock(std::atomic<bool> &lock)
inline void getSpinlock(std::atomic<bool>& lock)
{
bool _false = false;
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
bool _false = false;
while (!lock.compare_exchange_weak(_false, true, std::memory_order_acquire))
_false = false;
}
inline bool trySpinlock(std::atomic<bool> &lock)
inline bool trySpinlock(std::atomic<bool>& lock)
{
bool _false = false;
bool ret = lock.compare_exchange_weak(_false, true, std::memory_order_acquire);
return ret;
bool _false = false;
bool ret = lock.compare_exchange_weak(_false, true, std::memory_order_acquire);
return ret;
}
inline void releaseSpinlock(std::atomic<bool> &lock)
inline void releaseSpinlock(std::atomic<bool>& lock)
{
lock.store(false, std::memory_order_release);
lock.store(false, std::memory_order_release);
}
}
} // namespace utils

606
utils/common/statistics.cpp
View File

@ -35,405 +35,407 @@ using ColumnsCache = std::vector<std::unordered_set<uint32_t>>;
StatisticsManager* StatisticsManager::instance()
{
static StatisticsManager* sm = new StatisticsManager();
return sm;
static StatisticsManager* sm = new StatisticsManager();
return sm;
}
void StatisticsManager::analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace)
{
std::lock_guard<std::mutex> lock(mut);
auto rowCount = rowGroup.getRowCount();
const auto columnCount = rowGroup.getColumnCount();
if (!rowCount || !columnCount)
return;
std::lock_guard<std::mutex> lock(mut);
auto rowCount = rowGroup.getRowCount();
const auto columnCount = rowGroup.getColumnCount();
if (!rowCount || !columnCount)
return;
auto& oids = rowGroup.getOIDs();
auto& oids = rowGroup.getOIDs();
rowgroup::Row r;
rowGroup.initRow(&r);
rowGroup.getRow(0, &r);
rowgroup::Row r;
rowGroup.initRow(&r);
rowGroup.getRow(0, &r);
ColumnsCache columns(columnCount, std::unordered_set<uint32_t>());
// Init key types.
for (uint32_t index = 0; index < columnCount; ++index)
keyTypes[oids[index]] = KeyType::PK;
ColumnsCache columns(columnCount, std::unordered_set<uint32_t>());
// Init key types.
for (uint32_t index = 0; index < columnCount; ++index)
keyTypes[oids[index]] = KeyType::PK;
const uint32_t maxRowCount = 4096;
// TODO: We should read just couple of blocks from columns, not all data, but this requires
// more deep refactoring of column commands.
rowCount = std::min(rowCount, maxRowCount);
// This is strange, it's a CS but I'm processing data as row by row, how to fix it?
for (uint32_t i = 0; i < rowCount; ++i)
const uint32_t maxRowCount = 4096;
// TODO: We should read just couple of blocks from columns, not all data, but this requires
// more deep refactoring of column commands.
rowCount = std::min(rowCount, maxRowCount);
// This is strange, it's a CS but I'm processing data as row by row, how to fix it?
for (uint32_t i = 0; i < rowCount; ++i)
{
for (uint32_t j = 0; j < columnCount; ++j)
{
for (uint32_t j = 0; j < columnCount; ++j)
{
if (r.isNullValue(j) || columns[j].count(r.getIntField(j)))
keyTypes[oids[j]] = KeyType::FK;
else
columns[j].insert(r.getIntField(j));
}
r.nextRow();
if (r.isNullValue(j) || columns[j].count(r.getIntField(j)))
keyTypes[oids[j]] = KeyType::FK;
else
columns[j].insert(r.getIntField(j));
}
r.nextRow();
}
if (trace)
output(StatisticsType::PK_FK);
if (trace)
output(StatisticsType::PK_FK);
}
void StatisticsManager::output(StatisticsType statisticsType)
{
if (statisticsType == StatisticsType::PK_FK)
{
std::cout << "Columns count: " << keyTypes.size() << std::endl;
for (const auto& p : keyTypes)
std::cout << p.first << " " << (int) p.second << std::endl;
}
if (statisticsType == StatisticsType::PK_FK)
{
std::cout << "Columns count: " << keyTypes.size() << std::endl;
for (const auto& p : keyTypes)
std::cout << p.first << " " << (int)p.second << std::endl;
}
}
// Someday it will be a virtual method, based on statistics type we processing.
std::unique_ptr<char[]> StatisticsManager::convertStatsToDataStream(uint64_t& dataStreamSize)
{
// Number of pairs.
uint64_t count = keyTypes.size();
// count, [[uid, keyType], ... ]
dataStreamSize = sizeof(uint64_t) + count * (sizeof(uint32_t) + sizeof(KeyType));
// Number of pairs.
uint64_t count = keyTypes.size();
// count, [[uid, keyType], ... ]
dataStreamSize = sizeof(uint64_t) + count * (sizeof(uint32_t) + sizeof(KeyType));
// Allocate memory for data stream.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Initialize the data stream.
uint64_t offset = 0;
std::memcpy(dataStream, reinterpret_cast<char*>(&count), sizeof(uint64_t));
offset += sizeof(uint64_t);
// Allocate memory for data stream.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Initialize the data stream.
uint64_t offset = 0;
std::memcpy(dataStream, reinterpret_cast<char*>(&count), sizeof(uint64_t));
offset += sizeof(uint64_t);
// For each pair [oid, key type].
for (const auto& p : keyTypes)
{
uint32_t oid = p.first;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&oid), sizeof(uint32_t));
offset += sizeof(uint32_t);
// For each pair [oid, key type].
for (const auto& p : keyTypes)
{
uint32_t oid = p.first;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&oid), sizeof(uint32_t));
offset += sizeof(uint32_t);
KeyType keyType = p.second;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&keyType), sizeof(KeyType));
offset += sizeof(KeyType);
}
KeyType keyType = p.second;
std::memcpy(&dataStream[offset], reinterpret_cast<char*>(&keyType), sizeof(KeyType));
offset += sizeof(KeyType);
}
return dataStreamSmartPtr;
return dataStreamSmartPtr;
}
void StatisticsManager::saveToFile()
{
std::lock_guard<std::mutex> lock(mut);
std::lock_guard<std::mutex> lock(mut);
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> out(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "wb", 1));
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> out(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "wb", 1));
if (!out)
{
BRM::log_errno("StatisticsManager::saveToFile(): open");
throw ios_base::failure("StatisticsManager::saveToFile(): open failed.");
}
if (!out)
{
BRM::log_errno("StatisticsManager::saveToFile(): open");
throw ios_base::failure("StatisticsManager::saveToFile(): open failed.");
}
// Compute hash.
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
utils::Hasher128 hasher;
// Compute hash.
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
utils::Hasher128 hasher;
// Prepare a statistics file header.
const uint32_t headerSize = sizeof(StatisticsFileHeader);
StatisticsFileHeader fileHeader;
std::memset(&fileHeader, 0, headerSize);
fileHeader.version = version;
fileHeader.epoch = epoch;
fileHeader.dataSize = dataStreamSize;
// Compute hash from the data.
fileHeader.dataHash = hasher(dataStreamSmartPtr.get(), dataStreamSize);
// Prepare a statistics file header.
const uint32_t headerSize = sizeof(StatisticsFileHeader);
StatisticsFileHeader fileHeader;
std::memset(&fileHeader, 0, headerSize);
fileHeader.version = version;
fileHeader.epoch = epoch;
fileHeader.dataSize = dataStreamSize;
// Compute hash from the data.
fileHeader.dataHash = hasher(dataStreamSmartPtr.get(), dataStreamSize);
// Write statistics file header.
uint64_t size = out->write(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
// Write statistics file header.
uint64_t size = out->write(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
// Write data.
size = out->write(dataStreamSmartPtr.get(), dataStreamSize);
if (size != dataStreamSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
// Write data.
size = out->write(dataStreamSmartPtr.get(), dataStreamSize);
if (size != dataStreamSize)
{
auto rc = IDBPolicy::remove(fileName);
if (rc == -1)
std::cerr << "Cannot remove file " << fileName << std::endl;
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
throw ios_base::failure("StatisticsManager::saveToFile(): write failed. ");
}
}
void StatisticsManager::loadFromFile()
{
std::lock_guard<std::mutex> lock(mut);
// Check that stats file does exist.
if (!boost::filesystem::exists(statsFile))
return;
std::lock_guard<std::mutex> lock(mut);
// Check that stats file does exist.
if (!boost::filesystem::exists(statsFile))
return;
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> in(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", 1));
const char* fileName = statsFile.c_str();
std::unique_ptr<IDBDataFile> in(
IDBDataFile::open(IDBPolicy::getType(fileName, IDBPolicy::WRITEENG), fileName, "rb", 1));
if (!in)
{
BRM::log_errno("StatisticsManager::loadFromFile(): open");
throw ios_base::failure(
"StatisticsManager::loadFromFile(): open failed. Check the error log.");
}
if (!in)
{
BRM::log_errno("StatisticsManager::loadFromFile(): open");
throw ios_base::failure("StatisticsManager::loadFromFile(): open failed. Check the error log.");
}
// Read the file header.
StatisticsFileHeader fileHeader;
const uint32_t headerSize = sizeof(StatisticsFileHeader);
int64_t size = in->read(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
// Read the file header.
StatisticsFileHeader fileHeader;
const uint32_t headerSize = sizeof(StatisticsFileHeader);
int64_t size = in->read(reinterpret_cast<char*>(&fileHeader), headerSize);
if (size != headerSize)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
// Initialize fields from the file header.
version = fileHeader.version;
epoch = fileHeader.epoch;
const auto dataHash = fileHeader.dataHash;
const auto dataStreamSize = fileHeader.dataSize;
// Initialize fields from the file header.
version = fileHeader.version;
epoch = fileHeader.epoch;
const auto dataHash = fileHeader.dataHash;
const auto dataStreamSize = fileHeader.dataSize;
// Allocate the memory for the file data.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Allocate the memory for the file data.
std::unique_ptr<char[]> dataStreamSmartPtr(new char[dataStreamSize]);
auto* dataStream = dataStreamSmartPtr.get();
// Read the data.
uint64_t dataOffset = 0;
auto sizeToRead = dataStreamSize;
size = in->read(dataStream, sizeToRead);
sizeToRead -= size;
dataOffset += size;
while (sizeToRead > 0)
{
size = in->read(dataStream + dataOffset, sizeToRead);
if (size < 0)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
// Read the data.
uint64_t dataOffset = 0;
auto sizeToRead = dataStreamSize;
size = in->read(dataStream, sizeToRead);
sizeToRead -= size;
dataOffset += size;
}
while (sizeToRead > 0)
{
size = in->read(dataStream + dataOffset, sizeToRead);
if (size < 0)
throw ios_base::failure("StatisticsManager::loadFromFile(): read failed. ");
utils::Hasher128 hasher;
auto computedDataHash = hasher(dataStream, dataStreamSize);
if (dataHash != computedDataHash)
throw ios_base::failure("StatisticsManager::loadFromFile(): invalid file hash. ");
sizeToRead -= size;
dataOffset += size;
}
uint64_t count = 0;
std::memcpy(reinterpret_cast<char*>(&count), dataStream, sizeof(uint64_t));
uint64_t offset = sizeof(uint64_t);
utils::Hasher128 hasher;
auto computedDataHash = hasher(dataStream, dataStreamSize);
if (dataHash != computedDataHash)
throw ios_base::failure("StatisticsManager::loadFromFile(): invalid file hash. ");
uint64_t count = 0;
std::memcpy(reinterpret_cast<char*>(&count), dataStream, sizeof(uint64_t));
uint64_t offset = sizeof(uint64_t);
// For each pair.
for (uint64_t i = 0; i < count; ++i)
{
uint32_t oid;
KeyType keyType;
std::memcpy(reinterpret_cast<char*>(&oid), &dataStream[offset], sizeof(uint32_t));
offset += sizeof(uint32_t);
std::memcpy(reinterpret_cast<char*>(&keyType), &dataStream[offset], sizeof(KeyType));
offset += sizeof(KeyType);
// Insert pair.
keyTypes[oid] = keyType;
}
// For each pair.
for (uint64_t i = 0; i < count; ++i)
{
uint32_t oid;
KeyType keyType;
std::memcpy(reinterpret_cast<char*>(&oid), &dataStream[offset], sizeof(uint32_t));
offset += sizeof(uint32_t);
std::memcpy(reinterpret_cast<char*>(&keyType), &dataStream[offset], sizeof(KeyType));
offset += sizeof(KeyType);
// Insert pair.
keyTypes[oid] = keyType;
}
}
uint64_t StatisticsManager::computeHashFromStats()
{
utils::Hasher128 hasher;
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
return hasher(dataStreamSmartPtr.get(), dataStreamSize);
utils::Hasher128 hasher;
uint64_t dataStreamSize = 0;
std::unique_ptr<char[]> dataStreamSmartPtr = convertStatsToDataStream(dataStreamSize);
return hasher(dataStreamSmartPtr.get(), dataStreamSize);
}
void StatisticsManager::serialize(messageqcpp::ByteStream& bs)
{
uint64_t count = keyTypes.size();
bs << version;
bs << epoch;
bs << count;
uint64_t count = keyTypes.size();
bs << version;
bs << epoch;
bs << count;
for (const auto& keyType : keyTypes)
{
bs << keyType.first;
bs << (uint32_t) keyType.second;
}
for (const auto& keyType : keyTypes)
{
bs << keyType.first;
bs << (uint32_t)keyType.second;
}
}
void StatisticsManager::unserialize(messageqcpp::ByteStream& bs)
{
uint64_t count;
bs >> version;
bs >> epoch;
bs >> count;
uint64_t count;
bs >> version;
bs >> epoch;
bs >> count;
for (uint32_t i = 0; i < count; ++i)
{
uint32_t oid, keyType;
bs >> oid;
bs >> keyType;
keyTypes[oid] = static_cast<KeyType>(keyType);
}
for (uint32_t i = 0; i < count; ++i)
{
uint32_t oid, keyType;
bs >> oid;
bs >> keyType;
keyTypes[oid] = static_cast<KeyType>(keyType);
}
}
bool StatisticsManager::hasKey(uint32_t oid) { return keyTypes.count(oid) > 0 ? true : false; }
bool StatisticsManager::hasKey(uint32_t oid)
{
return keyTypes.count(oid) > 0 ? true : false;
}
KeyType StatisticsManager::getKeyType(uint32_t oid) { return keyTypes[oid]; }
KeyType StatisticsManager::getKeyType(uint32_t oid)
{
return keyTypes[oid];
}
StatisticsDistributor* StatisticsDistributor::instance()
{
static StatisticsDistributor* sd = new StatisticsDistributor();
return sd;
static StatisticsDistributor* sd = new StatisticsDistributor();
return sd;
}
void StatisticsDistributor::distributeStatistics()
{
countClients();
countClients();
{
std::lock_guard<std::mutex> lock(mut);
// No clients.
if (clientsCount == 0)
return;
#ifdef DEBUG_STATISTICS
std::cout << "Distribute statistics from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
#endif
messageqcpp::ByteStream msg, statsHash, statsBs;
// Current hash.
statsHash << statistics::StatisticsManager::instance()->computeHashFromStats();
// Statistics.
statistics::StatisticsManager::instance()->serialize(statsBs);
for (uint32_t i = 0; i < clientsCount; ++i)
{
std::lock_guard<std::mutex> lock(mut);
// No clients.
if (clientsCount == 0)
return;
try
{
messageqcpp::ByteStream::quadbyte qb = ANALYZE_TABLE_REC_STATS;
msg << qb;
auto exeMgrID = "ExeMgr" + std::to_string(i + 2);
// Create a client.
std::unique_ptr<messageqcpp::MessageQueueClient> exemgrClient(
new messageqcpp::MessageQueueClient(exeMgrID));
#ifdef DEBUG_STATISTICS
std::cout << "Distribute statistics from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
std::cout << "Try to connect to " << exeMgrID << std::endl;
#endif
messageqcpp::ByteStream msg, statsHash, statsBs;
// Current hash.
statsHash << statistics::StatisticsManager::instance()->computeHashFromStats();
// Statistics.
statistics::StatisticsManager::instance()->serialize(statsBs);
for (uint32_t i = 0; i < clientsCount; ++i)
// Try to connect to the client.
if (!exemgrClient->connect())
{
try
{
messageqcpp::ByteStream::quadbyte qb = ANALYZE_TABLE_REC_STATS;
msg << qb;
auto exeMgrID = "ExeMgr" + std::to_string(i + 2);
// Create a client.
std::unique_ptr<messageqcpp::MessageQueueClient> exemgrClient(
new messageqcpp::MessageQueueClient(exeMgrID));
msg.restart();
#ifdef DEBUG_STATISTICS
std::cout << "Try to connect to " << exeMgrID << std::endl;
std::cout << "Unable to connect to " << exeMgrID << std::endl;
#endif
// Try to connect to the client.
if (!exemgrClient->connect())
{
msg.restart();
#ifdef DEBUG_STATISTICS
std::cout << "Unable to connect to " << exeMgrID << std::endl;
#endif
continue;
}
#ifdef DEBUG_STATISTICS
std::cout
<< "Write flag ANALYZE_TABLE_REC_STATS from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a flag to client ExeMgr.
exemgrClient->write(msg);
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics hash from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a hash of the stats.
exemgrClient->write(statsHash);
// Read the state from Client.
msg.restart();
msg = exemgrClient->read();
msg >> qb;
// Do not need a stats.
if (qb == ANALYZE_TABLE_SUCCESS)
{
msg.restart();
continue;
}
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics bytestream from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a statistics to client ExeMgr.
exemgrClient->write(statsBs);
// Read the flag back from the client ExeMgr.
msg.restart();
msg = exemgrClient->read();
if (msg.length() == 0)
throw runtime_error("Lost conection to ExeMgr.");
#ifdef DEBUG_STATISTICS
std::cout << "Read flag on ExeMgr(Server) from ExeMgr(Client) " << std::endl;
#endif
msg.restart();
}
catch (std::exception& e)
{
msg.restart();
std::cerr << "distributeStatistics() failed with error: " << e.what() << std::endl;
}
catch (...)
{
msg.restart();
std::cerr << "distributeStatistics() failed with unknown error." << std::endl;
}
continue;
}
#ifdef DEBUG_STATISTICS
std::cout << "Write flag ANALYZE_TABLE_REC_STATS from ExeMgr(Server) to ExeMgr(Clients) "
<< std::endl;
#endif
// Write a flag to client ExeMgr.
exemgrClient->write(msg);
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics hash from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
#endif
// Write a hash of the stats.
exemgrClient->write(statsHash);
// Read the state from Client.
msg.restart();
msg = exemgrClient->read();
msg >> qb;
// Do not need a stats.
if (qb == ANALYZE_TABLE_SUCCESS)
{
msg.restart();
continue;
}
#ifdef DEBUG_STATISTICS
std::cout << "Write statistics bytestream from ExeMgr(Server) to ExeMgr(Clients) " << std::endl;
#endif
// Write a statistics to client ExeMgr.
exemgrClient->write(statsBs);
// Read the flag back from the client ExeMgr.
msg.restart();
msg = exemgrClient->read();
if (msg.length() == 0)
throw runtime_error("Lost conection to ExeMgr.");
#ifdef DEBUG_STATISTICS
std::cout << "Read flag on ExeMgr(Server) from ExeMgr(Client) " << std::endl;
#endif
msg.restart();
}
catch (std::exception& e)
{
msg.restart();
std::cerr << "distributeStatistics() failed with error: " << e.what() << std::endl;
}
catch (...)
{
msg.restart();
std::cerr << "distributeStatistics() failed with unknown error." << std::endl;
}
}
}
}
void StatisticsDistributor::countClients()
{
#ifdef DEBUG_STATISTICS
std::cout << "count clients to distribute statistics " << std::endl;
std::cout << "count clients to distribute statistics " << std::endl;
#endif
auto* config = config::Config::makeConfig();
// Starting from the ExeMgr2, since the Server starts on the ExeMgr1.
std::atomic<uint32_t> exeMgrNumber(2);
auto* config = config::Config::makeConfig();
// Starting from the ExeMgr2, since the Server starts on the ExeMgr1.
std::atomic<uint32_t> exeMgrNumber(2);
try
try
{
while (true)
{
while (true)
{
auto exeMgrID = "ExeMgr" + std::to_string(exeMgrNumber);
auto exeMgrIP = config->getConfig(exeMgrID, "IPAddr");
if (exeMgrIP == "")
break;
auto exeMgrID = "ExeMgr" + std::to_string(exeMgrNumber);
auto exeMgrIP = config->getConfig(exeMgrID, "IPAddr");
if (exeMgrIP == "")
break;
#ifdef DEBUG_STATISTICS
std::cout << "Client: " << exeMgrID << std::endl;
std::cout << "Client: " << exeMgrID << std::endl;
#endif
++exeMgrNumber;
}
}
catch (std::exception& e)
{
std::cerr << "countClients() failed with error: " << e.what() << std::endl;
}
catch (...)
{
std::cerr << "countClients() failed with unknown error: ";
++exeMgrNumber;
}
}
catch (std::exception& e)
{
std::cerr << "countClients() failed with error: " << e.what() << std::endl;
}
catch (...)
{
std::cerr << "countClients() failed with unknown error: ";
}
clientsCount = exeMgrNumber - 2;
clientsCount = exeMgrNumber - 2;
#ifdef DEBUG_STATISTICS
std::cout << "Number of clients: " << clientsCount << std::endl;
std::cout << "Number of clients: " << clientsCount << std::endl;
#endif
}
} // namespace statistics
} // namespace statistics

111
utils/common/statistics.h
View File

@ -37,31 +37,30 @@ using namespace idbdatafile;
namespace statistics
{
// Represents a column key type:
// PK - primary key.
// FK - foreign key.
enum class KeyType : uint32_t
{
PK,
FK
PK,
FK
};
// Rerpresents types of statistics CS supports.
enum class StatisticsType : uint32_t
{
// A special statistics type, made to solve circular inner join problem.
PK_FK
// A special statistics type, made to solve circular inner join problem.
PK_FK
};
// Represetns a header for the statistics file.
struct StatisticsFileHeader
{
uint64_t version;
uint64_t epoch;
uint64_t dataHash;
uint64_t dataSize;
uint8_t offset[1024];
uint64_t version;
uint64_t epoch;
uint64_t dataHash;
uint64_t dataSize;
uint8_t offset[1024];
};
// This class is responsible for processing and storing statistics.
@ -69,56 +68,64 @@ struct StatisticsFileHeader
// the updated statistics into the special file.
class StatisticsManager
{
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsManager* instance();
// Analyzes the given `rowGroup` by processing it row by row and searching for foreign key.
void analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace);
// Ouputs stats to out stream.
void output(StatisticsType statisticsType = StatisticsType::PK_FK);
// Saves stats to the file.
void saveToFile();
// Loads stats from the file.
void loadFromFile();
void incEpoch() { ++epoch; }
// Serialize stats to the given `bs`.
void serialize(messageqcpp::ByteStream& bs);
// Unserialize stats from the given `bs`.
void unserialize(messageqcpp::ByteStream& bs);
// Computes hash from the current statistics data.
uint64_t computeHashFromStats();
// Checks whether statistics is available for the given `oid`.
bool hasKey(uint32_t oid);
// Returns a KeyType for the given `oid`.
KeyType getKeyType(uint32_t oid);
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsManager* instance();
// Analyzes the given `rowGroup` by processing it row by row and searching for foreign key.
void analyzeColumnKeyTypes(const rowgroup::RowGroup& rowGroup, bool trace);
// Ouputs stats to out stream.
void output(StatisticsType statisticsType = StatisticsType::PK_FK);
// Saves stats to the file.
void saveToFile();
// Loads stats from the file.
void loadFromFile();
void incEpoch()
{
++epoch;
}
// Serialize stats to the given `bs`.
void serialize(messageqcpp::ByteStream& bs);
// Unserialize stats from the given `bs`.
void unserialize(messageqcpp::ByteStream& bs);
// Computes hash from the current statistics data.
uint64_t computeHashFromStats();
// Checks whether statistics is available for the given `oid`.
bool hasKey(uint32_t oid);
// Returns a KeyType for the given `oid`.
KeyType getKeyType(uint32_t oid);
private:
std::map<uint32_t, KeyType> keyTypes;
StatisticsManager() : epoch(0), version(1) { IDBPolicy::init(true, false, "", 0); }
std::unique_ptr<char[]> convertStatsToDataStream(uint64_t& dataStreamSize);
private:
std::map<uint32_t, KeyType> keyTypes;
StatisticsManager() : epoch(0), version(1)
{
IDBPolicy::init(true, false, "", 0);
}
std::unique_ptr<char[]> convertStatsToDataStream(uint64_t& dataStreamSize);
std::mutex mut;
uint32_t epoch;
uint32_t version;
std::string statsFile = "/var/lib/columnstore/local/statistics";
std::mutex mut;
uint32_t epoch;
uint32_t version;
std::string statsFile = "/var/lib/columnstore/local/statistics";
};
// This class is responsible for distributing the statistics across all `ExeMgr` in a cluster.
class StatisticsDistributor
{
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsDistributor* instance();
public:
// Returns the instance of this class, static initialization happens only once.
static StatisticsDistributor* instance();
// Distribute stats across all `ExeMgr` in cluster by connecting to them using config file.
void distributeStatistics();
// Distribute stats across all `ExeMgr` in cluster by connecting to them using config file.
void distributeStatistics();
private:
StatisticsDistributor() : clientsCount(0) {}
// Count the number of clients by reading config file and evaluating `ExeMgr` fields.
void countClients();
uint32_t clientsCount;
std::mutex mut;
private:
StatisticsDistributor() : clientsCount(0)
{
}
// Count the number of clients by reading config file and evaluating `ExeMgr` fields.
void countClients();
uint32_t clientsCount;
std::mutex mut;
};
} // namespace statistics
} // namespace statistics

145
utils/common/stlpoolallocator.h
View File

@ -16,9 +16,9 @@
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
* $Id$
*
******************************************************************************************/
/* Makes PoolAllocator STL-compliant */
@ -33,152 +33,151 @@
namespace utils
{
/* If using the pool allocator with a boost smart ptr, use an instance of this
as the deleter. */
struct BoostPoolDeallocator
{
inline void operator()(void* ptr) { };
inline void operator()(void* ptr){};
};
/* This is an STL-compliant wrapper for PoolAllocator + an optimization for containers
* that aren't entirely node based (ex: vectors and hash tables)
*/
template<class T>
template <class T>
class STLPoolAllocator
{
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template<class U> struct rebind
{
typedef STLPoolAllocator<U> other;
};
public:
typedef size_t size_type;
typedef ptrdiff_t difference_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef T value_type;
template <class U>
struct rebind
{
typedef STLPoolAllocator<U> other;
};
STLPoolAllocator() throw();
STLPoolAllocator(const STLPoolAllocator&) throw();
STLPoolAllocator(uint32_t capacity) throw();
template<class U> STLPoolAllocator(const STLPoolAllocator<U>&) throw();
~STLPoolAllocator();
STLPoolAllocator() throw();
STLPoolAllocator(const STLPoolAllocator&) throw();
STLPoolAllocator(uint32_t capacity) throw();
template <class U>
STLPoolAllocator(const STLPoolAllocator<U>&) throw();
~STLPoolAllocator();
STLPoolAllocator<T>& operator=(const STLPoolAllocator<T>&);
STLPoolAllocator<T>& operator=(const STLPoolAllocator<T>&);
void usePoolAllocator(boost::shared_ptr<PoolAllocator> b);
boost::shared_ptr<utils::PoolAllocator> getPoolAllocator();
void usePoolAllocator(boost::shared_ptr<PoolAllocator> b);
boost::shared_ptr<utils::PoolAllocator> getPoolAllocator();
pointer allocate(size_type, const void* hint = 0);
void deallocate(pointer p, size_type n);
size_type max_size() const throw();
inline uint64_t getMemUsage() const
{
return pa->getMemUsage();
}
pointer allocate(size_type, const void* hint = 0);
void deallocate(pointer p, size_type n);
size_type max_size() const throw();
inline uint64_t getMemUsage() const
{
return pa->getMemUsage();
}
void construct(pointer p, const T& val);
void destroy(pointer p);
void construct(pointer p, const T& val);
void destroy(pointer p);
static const uint32_t DEFAULT_SIZE = 32768 * sizeof(T);
static const uint32_t DEFAULT_SIZE = 32768 * sizeof(T);
boost::shared_ptr<utils::PoolAllocator> pa;
boost::shared_ptr<utils::PoolAllocator> pa;
};
template<class T>
template <class T>
STLPoolAllocator<T>::STLPoolAllocator() throw()
{
pa.reset(new PoolAllocator(DEFAULT_SIZE));
pa.reset(new PoolAllocator(DEFAULT_SIZE));
}
template<class T>
template <class T>
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<T>& s) throw()
{
pa = s.pa;
pa = s.pa;
}
template<class T>
template <class T>
STLPoolAllocator<T>::STLPoolAllocator(uint32_t capacity) throw()
{
pa.reset(new PoolAllocator(capacity));
pa.reset(new PoolAllocator(capacity));
}
template<class T>
template<class U>
template <class T>
template <class U>
STLPoolAllocator<T>::STLPoolAllocator(const STLPoolAllocator<U>& s) throw()
{
pa = s.pa;
pa = s.pa;
}
template<class T>
template <class T>
STLPoolAllocator<T>::~STLPoolAllocator()
{
}
template<class T>
template <class T>
void STLPoolAllocator<T>::usePoolAllocator(boost::shared_ptr<PoolAllocator> p)
{
pa = p;
pa = p;
}
template<class T>
template <class T>
boost::shared_ptr<utils::PoolAllocator> STLPoolAllocator<T>::getPoolAllocator()
{
return pa;
return pa;
}
template<class T>
typename STLPoolAllocator<T>::pointer
STLPoolAllocator<T>::allocate(typename STLPoolAllocator<T>::size_type s,
typename std::allocator<void>::const_pointer hint)
template <class T>
typename STLPoolAllocator<T>::pointer STLPoolAllocator<T>::allocate(
typename STLPoolAllocator<T>::size_type s, typename std::allocator<void>::const_pointer hint)
{
return (pointer) pa->allocate(s * sizeof(T));
return (pointer)pa->allocate(s * sizeof(T));
}
template<class T>
template <class T>
void STLPoolAllocator<T>::deallocate(typename STLPoolAllocator<T>::pointer p,
typename STLPoolAllocator<T>::size_type n)
{
pa->deallocate((void*) p);
pa->deallocate((void*)p);
}
template<class T>
template <class T>
typename STLPoolAllocator<T>::size_type STLPoolAllocator<T>::max_size() const throw()
{
return std::numeric_limits<uint64_t>::max() / sizeof(T);
return std::numeric_limits<uint64_t>::max() / sizeof(T);
}
template<class T>
template <class T>
void STLPoolAllocator<T>::construct(typename STLPoolAllocator<T>::pointer p, const T& val)
{
new ((void*)p) T(val);
new ((void*)p) T(val);
}
template<class T>
template <class T>
void STLPoolAllocator<T>::destroy(typename STLPoolAllocator<T>::pointer p)
{
p->T::~T();
p->T::~T();
}
template<class T>
template <class T>
STLPoolAllocator<T>& STLPoolAllocator<T>::operator=(const STLPoolAllocator<T>& c)
{
pa = c.pa;
return *this;
pa = c.pa;
return *this;
}
template<typename T>
template <typename T>
bool operator==(const STLPoolAllocator<T>&, const STLPoolAllocator<T>&)
{
return true;
return true;
}
template<typename T>
template <typename T>
bool operator!=(const STLPoolAllocator<T>&, const STLPoolAllocator<T>&)
{
return false;
}
return false;
}
} // namespace utils

159
utils/common/syncstream.h
View File

@ -30,28 +30,28 @@
void copyStream(istream& iss, ostream& oss)
{
string line;
getline(iss, line);
while (iss.good())
{
oss << line << endl;
getline(iss, line);
}
string line;
getline(iss, line);
while (iss.good())
{
oss << line << endl;
getline(iss, line);
}
}
main()
{
FILE* ifp;
FILE* ofp;
FILE* ifp;
FILE* ofp;
...
...
isyncstream iss(ifp);
osyncstream oss(ofp);
isyncstream iss(ifp);
osyncstream oss(ofp);
copyStream(iss, oss);
copyStream(iss, oss);
...
...
}
*/
@ -62,89 +62,96 @@ main()
namespace syncstream
{
/** A streambuf implementation for C stdio FILE* streams.
*
* Adapted from http://www.drdobbs.com/184401305
*/
class syncbuf : public std::streambuf
{
public:
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f) {}
public:
/** ctor */
syncbuf(FILE* f) : std::streambuf(), fptr(f)
{
}
protected:
/** Write character in the case of overflow */
virtual int overflow(int c = EOF)
{
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow()
{
int c = getc(fptr);
protected:
/** Write character in the case of overflow */
virtual int overflow(int c = EOF)
{
return (c != EOF ? fputc(c, fptr) : EOF);
}
/** Get character in the case of overflow */
virtual int underflow()
{
int c = getc(fptr);
if (c != EOF)
ungetc(c, fptr);
if (c != EOF)
ungetc(c, fptr);
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow()
{
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF)
{
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync()
{
return fflush(fptr);
}
return c;
}
/** Get character in the case of overflow and advance get pointer */
virtual int uflow()
{
return getc(fptr);
}
/** put character back in the case of backup underflow */
virtual int pbackfail(int c = EOF)
{
return (c != EOF ? ungetc(c, fptr) : EOF);
}
/** Synchronize stream buffer */
virtual int sync()
{
return fflush(fptr);
}
private:
FILE* fptr;
private:
FILE* fptr;
};
/** An istream adaptor for input FILE* streams */
class isyncstream : public std::istream
{
public:
/** ctor */
isyncstream() : istream(&buf), buf(0) {}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
public:
/** ctor */
isyncstream() : istream(&buf), buf(0)
{
}
/** ctor */
isyncstream(FILE* fptr) : istream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
private:
syncbuf buf;
};
/** An ostream adaptor for output FILE* streams */
class osyncstream : public std::ostream
{
public:
/** ctor */
osyncstream() : ostream(&buf), buf(0) {}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr) {}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
public:
/** ctor */
osyncstream() : ostream(&buf), buf(0)
{
}
/** ctor */
osyncstream(FILE* fptr) : ostream(&buf), buf(fptr)
{
}
/** const streambuf accessor */
const syncbuf* rdbuf() const
{
return &buf;
}
private:
syncbuf buf;
private:
syncbuf buf;
};
}
} // namespace syncstream

22
utils/common/threadnaming.cpp
View File

@ -20,15 +20,15 @@
namespace utils
{
void setThreadName(const char *threadName)
{
prctl(PR_SET_NAME, threadName, 0, 0, 0);
}
void setThreadName(const char* threadName)
{
prctl(PR_SET_NAME, threadName, 0, 0, 0);
}
std::string getThreadName()
{
char buf[32];
prctl(PR_GET_NAME, buf, 0, 0, 0);
return std::string(buf);
}
} // end of namespace
std::string getThreadName()
{
char buf[32];
prctl(PR_GET_NAME, buf, 0, 0, 0);
return std::string(buf);
}
} // namespace utils

6
utils/common/threadnaming.h
View File

@ -20,6 +20,6 @@
namespace utils
{
void setThreadName(const char *threadName);
std::string getThreadName();
} // end of namespace
void setThreadName(const char* threadName);
std::string getThreadName();
} // namespace utils

15
utils/common/utils_utf8.cpp
View File

@ -17,21 +17,16 @@
#include "mariadb_my_sys.h"
namespace datatypes
{
static inline CHARSET_INFO & get_charset_or_bin(int32_t charsetNumber)
static inline CHARSET_INFO& get_charset_or_bin(int32_t charsetNumber)
{
CHARSET_INFO *cs= get_charset(charsetNumber, MYF(MY_WME));
return cs ? *cs : my_charset_bin;
CHARSET_INFO* cs = get_charset(charsetNumber, MYF(MY_WME));
return cs ? *cs : my_charset_bin;
}
Charset::Charset(uint32_t charsetNumber)
:mCharset(&get_charset_or_bin(charsetNumber))
Charset::Charset(uint32_t charsetNumber) : mCharset(&get_charset_or_bin(charsetNumber))
{
}
}
} // namespace datatypes

139
utils/common/utils_utf8.h
View File

@ -18,11 +18,8 @@
// $Id$
#pragma once
#include <string>
#if defined(_MSC_VER)
#include <malloc.h>
@ -40,120 +37,116 @@
// Change the name from utf8. Even change the file name to something resembling char helper
namespace utf8
{
const int MAX_UTF8_BYTES_PER_CHAR = 4;
// BUG 5241
// Infinidb specific mbstowcs(). This will handle both windows and unix platforms
// Params dest and max should have enough length to accomodate NULL
inline
size_t idb_mbstowcs(wchar_t* dest, const char* src, size_t max)
inline size_t idb_mbstowcs(wchar_t* dest, const char* src, size_t max)
{
#ifdef _MSC_VER
// 4th param (-1) denotes to convert till hit NULL char
// if 6th param max = 0, will return the required buffer size
size_t strwclen = MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)max);
// decrement the count of NULL; will become -1 on failure
return --strwclen;
// 4th param (-1) denotes to convert till hit NULL char
// if 6th param max = 0, will return the required buffer size
size_t strwclen = MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)max);
// decrement the count of NULL; will become -1 on failure
return --strwclen;
#else
return mbstowcs(dest, src, max);
return mbstowcs(dest, src, max);
#endif
}
// BUG 5241
// Infinidb specific wcstombs(). This will handle both windows and unix platforms
// Params dest and max should have enough length to accomodate NULL
inline
size_t idb_wcstombs(char* dest, const wchar_t* src, size_t max)
inline size_t idb_wcstombs(char* dest, const wchar_t* src, size_t max)
{
#ifdef _MSC_VER
// 4th param (-1) denotes to convert till hit NULL char
//if 6th param max = 0, will return the required buffer size
size_t strmblen = WideCharToMultiByte( CP_UTF8, 0, src, -1, dest, (int)max, NULL, NULL);
// decrement the count of NULL; will become -1 on failure
return --strmblen;
// 4th param (-1) denotes to convert till hit NULL char
// if 6th param max = 0, will return the required buffer size
size_t strmblen = WideCharToMultiByte(CP_UTF8, 0, src, -1, dest, (int)max, NULL, NULL);
// decrement the count of NULL; will become -1 on failure
return --strmblen;
#else
return wcstombs(dest, src, max);
return wcstombs(dest, src, max);
#endif
}
// convert UTF-8 string to wstring
inline
std::wstring utf8_to_wstring (const std::string& str)
inline std::wstring utf8_to_wstring(const std::string& str)
{
size_t bufsize = str.length() + 1;
size_t bufsize = str.length() + 1;
// Convert to wide characters. Do all further work in wide characters
wchar_t* wcbuf = new wchar_t[bufsize];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strwclen = idb_mbstowcs(wcbuf, str.c_str(), str.length() + 1);
// Convert to wide characters. Do all further work in wide characters
wchar_t* wcbuf = new wchar_t[bufsize];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strwclen = idb_mbstowcs(wcbuf, str.c_str(), str.length() + 1);
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if ( strwclen == static_cast<size_t>(-1) )
strwclen = 0;
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if (strwclen == static_cast<size_t>(-1))
strwclen = 0;
std::wstring ret(wcbuf, strwclen);
std::wstring ret(wcbuf, strwclen);
delete [] wcbuf;
return ret;
delete[] wcbuf;
return ret;
}
// convert wstring to UTF-8 string
inline
std::string wstring_to_utf8 (const std::wstring& str)
inline std::string wstring_to_utf8(const std::wstring& str)
{
char* outbuf = new char[(str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strmblen = idb_wcstombs(outbuf, str.c_str(), str.length() * MAX_UTF8_BYTES_PER_CHAR + 1);
char* outbuf = new char[(str.length() * MAX_UTF8_BYTES_PER_CHAR) + 1];
// Passing +1 so that windows is happy to see extra position to place NULL
size_t strmblen = idb_wcstombs(outbuf, str.c_str(), str.length() * MAX_UTF8_BYTES_PER_CHAR + 1);
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if ( strmblen == static_cast<size_t>(-1) )
strmblen = 0;
// if result is -1 it means bad characters which may happen if locale is wrong.
// return an empty string
if (strmblen == static_cast<size_t>(-1))
strmblen = 0;
std::string ret(outbuf, strmblen);
std::string ret(outbuf, strmblen);
delete [] outbuf;
return ret;
delete[] outbuf;
return ret;
}
inline
uint8_t utf8_truncate_point(const char* input, size_t length)
inline uint8_t utf8_truncate_point(const char* input, size_t length)
{
// Find the beginning of a multibyte char to truncate at and return the
// number of bytes to truncate1`
if (length < 3)
{
return 0;
}
const unsigned char* b = (const unsigned char*)(input) + length - 3;
if (b[2] & 0x80)
{
// First byte in a new multi-byte sequence
if (b[2] & 0x40) return 1;
// 3 byte sequence
else if ((b[1] & 0xe0) == 0xe0) return 2;
// 4 byte sequence
else if ((b[0] & 0xf0) == 0xf0) return 3;
}
// Find the beginning of a multibyte char to truncate at and return the
// number of bytes to truncate1`
if (length < 3)
{
return 0;
}
const unsigned char* b = (const unsigned char*)(input) + length - 3;
if (b[2] & 0x80)
{
// First byte in a new multi-byte sequence
if (b[2] & 0x40)
return 1;
// 3 byte sequence
else if ((b[1] & 0xe0) == 0xe0)
return 2;
// 4 byte sequence
else if ((b[0] & 0xf0) == 0xf0)
return 3;
}
return 0;
}
int mcs_strcoll(const char* str1, const char* str2, const uint32_t charsetNumber);
int mcs_strcoll(const char* str1, const uint32_t l1, const char* str2, const uint32_t l2, const uint32_t charsetNumber);
int mcs_strcoll(const char* str1, const uint32_t l1, const char* str2, const uint32_t l2,
const uint32_t charsetNumber);
int mcs_strcoll(const std::string* str1, const std::string* str2, const uint32_t charsetNumber);
int mcs_strcoll(const std::string& str1, const std::string& str2, const uint32_t charsetNumber);
int mcs_strcollsp(const char* str1, const char* str2, const uint32_t charsetNumber);
int mcs_strcollsp(const char* str1, uint32_t l1, const char* str2, const uint32_t l2, const uint32_t charsetNumber);
int mcs_strcollsp(const char* str1, uint32_t l1, const char* str2, const uint32_t l2,
const uint32_t charsetNumber);
int mcs_strcollsp(const std::string* str1, const std::string* str2, const uint32_t charsetNumber);
int mcs_strcollsp(const std::string& str1, const std::string& str2, const uint32_t charsetNumber);
} //namespace utf8
} // namespace utf8

154
utils/common/vlarray.h
View File

@ -16,83 +16,109 @@
MA 02110-1301, USA. */
#pragma once
namespace utils {
template <typename T, size_t SIZE=64>
namespace utils
{
template <typename T, size_t SIZE = 64>
class VLArray
{
public:
VLArray(size_t sz) :
sz(sz),
stack_storage(NULL),
dyn_storage(NULL),
ptr(NULL)
public:
VLArray(size_t sz) : sz(sz), stack_storage(NULL), dyn_storage(NULL), ptr(NULL)
{
if (sz > SIZE)
{
if (sz > SIZE) {
dyn_storage = new T[sz];
ptr = dyn_storage;
} else {
stack_storage = new (stack) T[sz];
ptr = stack_storage;
}
dyn_storage = new T[sz];
ptr = dyn_storage;
}
VLArray(size_t sz, const T& initval) : VLArray(sz)
else
{
for (size_t i= 0; i < sz; ++i)
ptr[i]= initval;
stack_storage = new (stack) T[sz];
ptr = stack_storage;
}
}
VLArray(const VLArray&) = delete;
VLArray(VLArray&&) = delete;
VLArray& operator=(const VLArray&) = delete;
VLArray& operator=(VLArray&&) = delete;
VLArray(size_t sz, const T& initval) : VLArray(sz)
{
for (size_t i = 0; i < sz; ++i)
ptr[i] = initval;
}
~VLArray() {
if (dyn_storage) {
delete [] dyn_storage;
} else {
// we cannot use `delete [] stack_storage` here so call d-tors explicitly
if (!std::is_trivially_destructible<T>::value) {
for (size_t i = 0; i < sz; ++i)
stack_storage[i].~T();
}
}
VLArray(const VLArray&) = delete;
VLArray(VLArray&&) = delete;
VLArray& operator=(const VLArray&) = delete;
VLArray& operator=(VLArray&&) = delete;
~VLArray()
{
if (dyn_storage)
{
delete[] dyn_storage;
}
size_t size() const { return sz; }
const T* data() const { return ptr; }
T* data() { return ptr; }
const T& operator[](size_t i) const { return ptr[i]; }
T& operator[](size_t i) { return ptr[i]; }
const T& at(size_t i) const {
if (i >= sz) {
throw std::out_of_range("index out of range: " + std::to_string(i) +
" >= size " + std::to_string(sz));
}
return ptr[i];
else
{
// we cannot use `delete [] stack_storage` here so call d-tors explicitly
if (!std::is_trivially_destructible<T>::value)
{
for (size_t i = 0; i < sz; ++i)
stack_storage[i].~T();
}
}
}
T& at(size_t i) {
if (i >= sz) {
throw std::out_of_range("index out of range: " + std::to_string(i) +
" >= size " + std::to_string(sz));
}
return ptr[i];
size_t size() const
{
return sz;
}
const T* data() const
{
return ptr;
}
T* data()
{
return ptr;
}
const T& operator[](size_t i) const
{
return ptr[i];
}
T& operator[](size_t i)
{
return ptr[i];
}
const T& at(size_t i) const
{
if (i >= sz)
{
throw std::out_of_range("index out of range: " + std::to_string(i) + " >= size " + std::to_string(sz));
}
return ptr[i];
}
operator const T* () const { return ptr; }
operator T* () { return ptr; }
T& at(size_t i)
{
if (i >= sz)
{
throw std::out_of_range("index out of range: " + std::to_string(i) + " >= size " + std::to_string(sz));
}
return ptr[i];
}
private:
const size_t sz;
alignas(T) char stack[SIZE * sizeof(T)];
T* stack_storage;
T* dyn_storage;
T* ptr;
operator const T*() const
{
return ptr;
}
operator T*()
{
return ptr;
}
private:
const size_t sz;
alignas(T) char stack[SIZE * sizeof(T)];
T* stack_storage;
T* dyn_storage;
T* ptr;
};
} // namespace utils
} // namespace utils

39
utils/common/widedecimalutils.h
View File

@ -22,26 +22,25 @@
namespace utils
{
const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64;
const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL;
const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64;
const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL;
inline void int128Max(int128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0x7FFFFFFFFFFFFFFF;
}
inline void int128Min(int128_t& val)
{
val = int128_t(0x8000000000000000LL) << 64;
}
inline void uint128Max(uint128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0xFFFFFFFFFFFFFFFF;
}
inline void int128Max(int128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0x7FFFFFFFFFFFFFFF;
}
inline void int128Min(int128_t& val)
{
val = int128_t(0x8000000000000000LL) << 64;
}
inline void uint128Max(uint128_t& val)
{
uint64_t* ptr = reinterpret_cast<uint64_t*>(&val);
ptr[0] = 0xFFFFFFFFFFFFFFFF;
ptr[1] = 0xFFFFFFFFFFFFFFFF;
}
} // namespace utils