database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
utils/idbdatafile/IDBPolicy.cpp
View File

@ -41,71 +41,73 @@ namespace idbdatafile
bool IDBPolicy::s_usehdfs = false;
bool IDBPolicy::s_bUseRdwrMemBuffer = false;
int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0;
int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0;
std::string IDBPolicy::s_hdfsRdwrScratch;
bool IDBPolicy::s_configed = false;
boost::mutex IDBPolicy::s_mutex;
void IDBPolicy::init( bool bEnableLogging, bool bUseRdwrMemBuffer, const string& hdfsRdwrScratch, int64_t hdfsRdwrBufferMaxSize )
{
IDBFactory::installDefaultPlugins();
IDBFactory::installDefaultPlugins();
IDBLogger::enable( bEnableLogging );
IDBLogger::enable( bEnableLogging );
s_bUseRdwrMemBuffer = bUseRdwrMemBuffer;
s_hdfsRdwrBufferMaxSize = hdfsRdwrBufferMaxSize;
s_hdfsRdwrScratch = hdfsRdwrScratch;
s_bUseRdwrMemBuffer = bUseRdwrMemBuffer;
s_hdfsRdwrBufferMaxSize = hdfsRdwrBufferMaxSize;
s_hdfsRdwrScratch = hdfsRdwrScratch;
// Create our scratch directory
if( hdfsRdwrScratch.length() > 0 )
{
// TODO-check to make sure this directory has sufficient space, whatever that means.
boost::filesystem::path tmpfilepath( hdfsRdwrScratch );
if (boost::filesystem::exists(tmpfilepath))
{
if (!boost::filesystem::is_directory(tmpfilepath) && useHdfs())
{
// We found a file named what we want our scratch directory to be named.
// Major issue, since we can't assume anything about where to put our tmp files
ostringstream oss;
oss << "IDBPolicy::init: scratch diretory setting " << hdfsRdwrScratch.c_str()
<< " exists as a file. Can't create hdfs buffer files.";
throw runtime_error(oss.str());
}
}
else
{
if (!boost::filesystem::create_directory(tmpfilepath))
{
// We failed to create the scratch directory
ostringstream oss;
oss << "IDBPolicy::init: failed to create hdfs scratch directory "
<< hdfsRdwrScratch.c_str() << ". Can't create hdfs buffer files.";
throw runtime_error(oss.str());
}
}
}
// Create our scratch directory
if ( hdfsRdwrScratch.length() > 0 )
{
// TODO-check to make sure this directory has sufficient space, whatever that means.
boost::filesystem::path tmpfilepath( hdfsRdwrScratch );
if (boost::filesystem::exists(tmpfilepath))
{
if (!boost::filesystem::is_directory(tmpfilepath) && useHdfs())
{
// We found a file named what we want our scratch directory to be named.
// Major issue, since we can't assume anything about where to put our tmp files
ostringstream oss;
oss << "IDBPolicy::init: scratch diretory setting " << hdfsRdwrScratch.c_str()
<< " exists as a file. Can't create hdfs buffer files.";
throw runtime_error(oss.str());
}
}
else
{
if (!boost::filesystem::create_directory(tmpfilepath))
{
// We failed to create the scratch directory
ostringstream oss;
oss << "IDBPolicy::init: failed to create hdfs scratch directory "
<< hdfsRdwrScratch.c_str() << ". Can't create hdfs buffer files.";
throw runtime_error(oss.str());
}
}
}
}
bool IDBPolicy::installPlugin(const std::string& plugin)
{
bool ret = IDBFactory::installPlugin(plugin);
// this is a cheesy way to do this, but it seems as good as anything for
// now. At some point, this policy class needs to be data driven - some
// type of specification to drive the logic here.
try
{
// see if there is an HDFS plugin
IDBFactory::name(IDBDataFile::HDFS);
s_usehdfs = true;
}
catch (std::exception& )
{
// nothing to do - this just means the plugin was not HDFS
;
}
bool ret = IDBFactory::installPlugin(plugin);
return ret;
// this is a cheesy way to do this, but it seems as good as anything for
// now. At some point, this policy class needs to be data driven - some
// type of specification to drive the logic here.
try
{
// see if there is an HDFS plugin
IDBFactory::name(IDBDataFile::HDFS);
s_usehdfs = true;
}
catch (std::exception& )
{
// nothing to do - this just means the plugin was not HDFS
;
}
return ret;
}
bool IDBPolicy::isLocalFile( const std::string& path )
@ -113,119 +115,125 @@ bool IDBPolicy::isLocalFile( const std::string& path )
boost::filesystem::path filepath( path );
#ifdef _MSC_VER
size_t strmblen = funcexp::utf8::idb_wcstombs(0, filepath.extension().c_str(), 0) + 1;
char* outbuf = (char*)alloca(strmblen * sizeof(char));
strmblen = funcexp::utf8::idb_wcstombs(outbuf, filepath.extension().c_str(), strmblen);
char* outbuf = (char*)alloca(strmblen * sizeof(char));
strmblen = funcexp::utf8::idb_wcstombs(outbuf, filepath.extension().c_str(), strmblen);
string fileExt(outbuf, strmblen);
#else
string fileExt = filepath.extension().c_str();
#endif
bool isXml = (fileExt == ".xml");
bool isXml = (fileExt == ".xml");
bool isVb = path.find("versionbuffer") != string::npos;
bool isInDbroot = path.find("/Calpont/data") != string::npos;
bool isScratch = path.find(s_hdfsRdwrScratch) == 0;
bool isVb = path.find("versionbuffer") != string::npos;
bool isInDbroot = path.find("/Calpont/data") != string::npos;
bool isScratch = path.find(s_hdfsRdwrScratch) == 0;
return !isInDbroot || isXml || isVb || isScratch;
return !isInDbroot || isXml || isVb || isScratch;
}
IDBDataFile::Types IDBPolicy::getType( const std::string& path, Contexts ctxt )
{
bool isLocal = isLocalFile( path );
bool isLocal = isLocalFile( path );
if( ctxt == PRIMPROC )
{
if( isLocal || !useHdfs() )
return IDBDataFile::UNBUFFERED;
else
return IDBDataFile::HDFS;
}
else
{
if( isLocal || !useHdfs() )
return IDBDataFile::BUFFERED;
else
return IDBDataFile::HDFS;
}
if ( ctxt == PRIMPROC )
{
if ( isLocal || !useHdfs() )
return IDBDataFile::UNBUFFERED;
else
return IDBDataFile::HDFS;
}
else
{
if ( isLocal || !useHdfs() )
return IDBDataFile::BUFFERED;
else
return IDBDataFile::HDFS;
}
}
IDBFileSystem& IDBPolicy::getFs( const std::string& path )
{
// for now context doesn't actually matter so just pass PRIMPROC
// later the whole logic around file -> filesystem type mapping
// needs to be data driven
return IDBFactory::getFs( getType( path, PRIMPROC) );
// for now context doesn't actually matter so just pass PRIMPROC
// later the whole logic around file -> filesystem type mapping
// needs to be data driven
return IDBFactory::getFs( getType( path, PRIMPROC) );
}
// ported from we_config.cpp
void IDBPolicy::configIDBPolicy()
{
// make sure this is done once.
boost::mutex::scoped_lock lk(s_mutex);
// make sure this is done once.
boost::mutex::scoped_lock lk(s_mutex);
if (s_configed)
return;
if (s_configed)
return;
config::Config* cf = config::Config::makeConfig();
config::Config* cf = config::Config::makeConfig();
//--------------------------------------------------------------------------
// IDBDataFile logging
//--------------------------------------------------------------------------
bool idblog = false;
string idblogstr = cf->getConfig("SystemConfig", "DataFileLog");
if ( idblogstr.length() != 0 )
{
boost::to_upper(idblogstr);
idblog = ( idblogstr == "ON" );
}
//--------------------------------------------------------------------------
// IDBDataFile logging
//--------------------------------------------------------------------------
bool idblog = false;
string idblogstr = cf->getConfig("SystemConfig", "DataFileLog");
//--------------------------------------------------------------------------
// Optional File System Plugin - if a HDFS type plugin is loaded
// then the system will use HDFS for all IDB data files
//--------------------------------------------------------------------------
string fsplugin = cf->getConfig("SystemConfig", "DataFilePlugin");
if ( fsplugin.length() != 0 )
{
IDBPolicy::installPlugin(fsplugin);
}
if ( idblogstr.length() != 0 )
{
boost::to_upper(idblogstr);
idblog = ( idblogstr == "ON" );
}
//--------------------------------------------------------------------------
// HDFS file buffering
//--------------------------------------------------------------------------
// Maximum amount of memory to use for hdfs buffering.
bool bUseRdwrMemBuffer = true; // If true, use in-memory buffering, else use file buffering
int64_t hdfsRdwrBufferMaxSize = 0;
string strBufferMaxSize = cf->getConfig("SystemConfig", "hdfsRdwrBufferMaxSize");
// Default is use membuf with no maximum size unless changed by config file.
if (strBufferMaxSize.length() > 0)
{
hdfsRdwrBufferMaxSize = static_cast<int64_t>(cf->uFromText(strBufferMaxSize));
if ( hdfsRdwrBufferMaxSize == 0 )
{
// If we're given a size of 0, turn off membuffering.
bUseRdwrMemBuffer = false;
}
}
//--------------------------------------------------------------------------
// Optional File System Plugin - if a HDFS type plugin is loaded
// then the system will use HDFS for all IDB data files
//--------------------------------------------------------------------------
string fsplugin = cf->getConfig("SystemConfig", "DataFilePlugin");
// Directory in which to place file buffer temporary files.
string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch");
if ( hdfsRdwrScratch.length() == 0 )
{
if ( fsplugin.length() != 0 )
{
IDBPolicy::installPlugin(fsplugin);
}
//--------------------------------------------------------------------------
// HDFS file buffering
//--------------------------------------------------------------------------
// Maximum amount of memory to use for hdfs buffering.
bool bUseRdwrMemBuffer = true; // If true, use in-memory buffering, else use file buffering
int64_t hdfsRdwrBufferMaxSize = 0;
string strBufferMaxSize = cf->getConfig("SystemConfig", "hdfsRdwrBufferMaxSize");
// Default is use membuf with no maximum size unless changed by config file.
if (strBufferMaxSize.length() > 0)
{
hdfsRdwrBufferMaxSize = static_cast<int64_t>(cf->uFromText(strBufferMaxSize));
if ( hdfsRdwrBufferMaxSize == 0 )
{
// If we're given a size of 0, turn off membuffering.
bUseRdwrMemBuffer = false;
}
}
// Directory in which to place file buffer temporary files.
string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch");
if ( hdfsRdwrScratch.length() == 0 )
{
string tmpPath = cf->getConfig("SystemConfig", "TempDiskPath");
if ( tmpPath.length() == 0 )
{
hdfsRdwrScratch = "/tmp/hdfsscratch";
hdfsRdwrScratch = "/tmp/hdfsscratch";
}
else
{
hdfsRdwrScratch = tmpPath;
hdfsRdwrScratch = tmpPath;
hdfsRdwrScratch += "/hdfsscratch";
}
}
}
IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize );
IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize );
s_configed = true;
s_configed = true;
}
}