database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-3606 Make ColumnStore use generic paths

Browse Source 
ColumnStore now uses standard bin/lib paths for pretty much everything.
Data path is now hard-coded to /var/lib/columnstore.

This patch also:

* Removes v1 decompression
* Removes a bunch of unneeded files
* Removes COLUMNSTORE_INSTALL_DIR / $INSTALLDIR
* Makes my.cnf.d work for all platforms (MCOL-3558)
* Changes configcpp to use recursive mutex (fixes possible config write deadlock)
* Fixes MCOL-3599 Fix regr functions, The library was installed in the wrong location
* Fixes a bunch of Ubuntu packaging issues
* Changes the binary names of several of the executables so as not to
clash with potential executables from other packages
This commit is contained in:
Andrew Hutchings
2019-11-04 11:06:33 +00:00
parent dea1dec507
commit 8ab9ebb0f4
178 changed files with 1310 additions and 18366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
writeengine/redistribute/we_redistributecontrol.cpp
View File

@ -89,8 +89,7 @@ RedistributeControl* RedistributeControl::instance()
RedistributeControl::RedistributeControl() : fInfoFilePtr(NULL), fPlanFilePtr(NULL)
{
// default path /usr/local/mariadb/columnstore/data1/systemFiles/redistribute
string installDir = startup::StartUp::installDir();
fRedistributeDir = installDir + RedistributeDir;
fRedistributeDir = "/var/lib/columnstore/" + RedistributeDir;
fInfoFilePath = fRedistributeDir + InfoFileName;
fPlanFilePath = fRedistributeDir + PlanFileName;

83
writeengine/server/we_dataloader.cpp
View File

@ -286,7 +286,7 @@ bool WEDataLoader::setupCpimport() // fork the cpimport
for (int i = aStartFD; i < aEndFD; i++) close(i);
errno = 0;
int aRet = execv(Cmds[0], &Cmds[0]); //NOTE - works with full Path
int aRet = execvp(Cmds[0], &Cmds[0]); //NOTE - works with full Path
//int aRet = execvp(Cmds[0], &Cmds[0]); //NOTE - works if $PATH has cpimport
int execvErrno = errno;
@ -563,85 +563,6 @@ void WEDataLoader::str2Argv(std::string CmdLine, std::vector<char*>& V)
V.push_back(0); //null terminate
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
std::string WEDataLoader::getCalpontHome()
{
string calpontDir = config::Config::makeConfig()->getConfig(
"SystemConfig", "CalpontHome");
if (0 == calpontDir.length())
{
calpontDir = startup::StartUp::installDir() + "/bin";
}
else
{
calpontDir += "/bin";
}
return calpontDir;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
std::string WEDataLoader::getPrgmPath(std::string& PrgmName)
{
std::string cpimportPath = getCalpontHome();
cpimportPath += "/";
cpimportPath += PrgmName;
return cpimportPath;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
void WEDataLoader::updateCmdLineWithPath(string& CmdLine)
{
std::istringstream iSs(CmdLine);
std::ostringstream oSs;
std::string aArg;
int aCount = 0;
while (iSs >> aArg)
{
if (0 == aCount)
{
string aPrgmPath = getPrgmPath(aArg);
oSs << aPrgmPath;
}
else
{
oSs << " ";
oSs << aArg;
}
++aCount;
}
CmdLine = oSs.str();
}
//-----------------------------------------------------------------------------
/**
*
@ -1207,8 +1128,6 @@ void WEDataLoader::onReceiveCmdLineArgs(ByteStream& Ibs)
{
Ibs >> fCmdLineStr;
cout << "CMD LINE ARGS came in " << fCmdLineStr << endl;
updateCmdLineWithPath(fCmdLineStr);
cout << "Updated CmdLine : " << fCmdLineStr << endl;
if (fpSysLog)
{

3
writeengine/server/we_dataloader.h
View File

@ -65,9 +65,6 @@ public:
void pushData2Cpimport(ByteStream& Ibs); // push data to cpimport from the queue
void closeWritePipe();
void str2Argv(std::string CmdLine, std::vector<char*>& V);
std::string getCalpontHome();
std::string getPrgmPath(std::string& PrgmName);
void updateCmdLineWithPath(std::string& CmdLine);
public:

71
writeengine/shared/we_config.cpp
View File

@ -120,7 +120,7 @@ void Config::checkReload( )
if ( m_bulkRoot.length() == 0 )
{
m_bulkRoot = startup::StartUp::installDir();
m_bulkRoot = "/var/lib/columnstore";
#ifndef _MSC_VER
m_bulkRoot += "/data";
#endif
@ -214,68 +214,6 @@ void Config::checkReload( )
if ( ncpb.length() != 0 )
m_NumCompressedPadBlks = cf->uFromText(ncpb);
#if 0 // common code, moved to IDBPolicy
//--------------------------------------------------------------------------
// IDBDataFile logging
//--------------------------------------------------------------------------
bool idblog = false;
string idblogstr = cf->getConfig("SystemConfig", "DataFileLog");
if ( idblogstr.length() != 0 )
{
boost::to_upper(idblogstr);
idblog = ( idblogstr == "ON" );
}
//--------------------------------------------------------------------------
// Optional File System Plugin - if a HDFS type plugin is loaded
// then the system will use HDFS for all IDB data files
//--------------------------------------------------------------------------
string fsplugin = cf->getConfig("SystemConfig", "DataFilePlugin");
if ( fsplugin.length() != 0 )
{
IDBPolicy::installPlugin(fsplugin);
}
//--------------------------------------------------------------------------
// HDFS file buffering
//--------------------------------------------------------------------------
// Maximum amount of memory to use for hdfs buffering.
bool bUseRdwrMemBuffer = true; // If true, use in-memory buffering, else use file buffering
int64_t hdfsRdwrBufferMaxSize = 0;
string strBufferMaxSize = cf->getConfig("SystemConfig", "hdfsRdwrBufferMaxSize");
if (strBufferMaxSize.length() == 0)
{
// Default is use membuf with no maximum size.
bUseRdwrMemBuffer = true;
}
else
{
hdfsRdwrBufferMaxSize = static_cast<int64_t>(cf->uFromText(strBufferMaxSize));
if ( hdfsRdwrBufferMaxSize == 0 )
{
// If we're given a size of 0, turn off membuffering.
bUseRdwrMemBuffer = false;
}
}
// Directory in which to place file buffer temporary files.
string TmpFileDir = startup::StartUp::tmpDir();
string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch");
hdfsRdwrScratch = TmpFileDir + hdfsRdwrScratch;
if ( hdfsRdwrScratch.length() == 0 )
{
hdfsRdwrScratch = TmpFileDir + "/hdfsscratch";
}
IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize );
#endif
IDBPolicy::configIDBPolicy();
//--------------------------------------------------------------------------
@ -307,12 +245,7 @@ void Config::checkReload( )
if ( m_VersionBufferDir.length() == 0 )
{
#ifdef _MSC_VER
m_VersionBufferDir = startup::StartUp::installDir() + "\\version";
#else
m_VersionBufferDir =
startup::StartUp::installDir() + "/data1/systemFiles/dbrm/BRM_saves";
#endif
m_VersionBufferDir = "/var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves";
}
//--------------------------------------------------------------------------

4
writeengine/splitter/we_cmdargs.cpp
View File

@ -1466,8 +1466,8 @@ void WECmdArgs::checkJobIdCase()
snprintf(aBuff, sizeof(aBuff), "%s/Job_%s.xml", fJobPath.c_str(),
fJobId.c_str());
else // for time being
snprintf(aBuff, sizeof(aBuff), "%s/data/bulk/job/Job_%s.xml",
startup::StartUp::installDir().c_str(), fJobId.c_str());
snprintf(aBuff, sizeof(aBuff), "/var/lib/columnstore/data/bulk/job/Job_%s.xml",
fJobId.c_str());
std::string aJobFileName(aBuff);

87
writeengine/splitter/we_splitterapp.cpp
View File

@ -469,17 +469,9 @@ void WESplitterApp::invokeCpimport()
boost::uuids::uuid u = boost::uuids::random_generator()();
fCmdArgs.setJobUUID(u);
//BUG 4361 - check cpimport.bin is available or not
std::string aCpiBinFile = getCalpontHome() + "/cpimport.bin"; //BUG 4361
if (access(aCpiBinFile.c_str(), X_OK) != 0)
throw runtime_error("Error: Missing File " + aCpiBinFile);
fCmdArgs.setMode(3);
std::string aCmdLineStr = fCmdArgs.getCpImportCmdLine();
updateCmdLineWithPath(aCmdLineStr);
if (fDh.getDebugLvl())
cout << "CPI CmdLineArgs : " << aCmdLineStr << endl;
@ -503,90 +495,13 @@ void WESplitterApp::invokeCpimport()
Cmds.push_back(0); //null terminate
int aRet = execv(Cmds[0], &Cmds[0]); //NOTE - works with full Path
int aRet = execvp(Cmds[0], &Cmds[0]); //NOTE - works with full Path
if (fDh.getDebugLvl())
cout << "Return status of cpimport is " << aRet << endl;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
std::string WESplitterApp::getCalpontHome()
{
string calpontDir = config::Config::makeConfig()->getConfig(
"SystemConfig", "CalpontHome");
if (0 == calpontDir.length())
{
calpontDir = startup::StartUp::installDir() + "/bin";
}
else
{
calpontDir += "/bin";
}
return calpontDir;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
std::string WESplitterApp::getPrgmPath(std::string& PrgmName)
{
std::string cpimportPath = getCalpontHome();
cpimportPath += "/";
cpimportPath += PrgmName;
return cpimportPath;
}
//-----------------------------------------------------------------------------
/**
*
* @brief Include the absolute path to prgm name, which is
* @brief the first element in the vector
* @param V vector which contains each element of argv
*
**/
void WESplitterApp::updateCmdLineWithPath(string& CmdLine)
{
std::istringstream iSs(CmdLine);
std::ostringstream oSs;
std::string aArg;
int aCount = 0;
while (iSs >> aArg)
{
if (0 == aCount)
{
string aPrgmPath = getPrgmPath(aArg);
oSs << aPrgmPath;
}
else
{
oSs << " ";
oSs << aArg;
}
++aCount;
}
CmdLine = oSs.str();
}
//-----------------------------------------------------------------------------
void WESplitterApp::updateWithJobFile(int aIdx)
{

3
writeengine/splitter/we_splitterapp.h
View File

@ -78,9 +78,6 @@ public:
static void onSigHup(int aInt);
void invokeCpimport();
std::string getCalpontHome();
std::string getPrgmPath(std::string& PrgmName);
void updateCmdLineWithPath(string& CmdLine);
private:
public: // for multi table support