From 81e35e35eb6c962f9d6d6503652262e021c1e2e2 Mon Sep 17 00:00:00 2001 From: David Hill Date: Fri, 21 Sep 2018 11:23:43 -0500 Subject: [PATCH] MCOL-520 --- dbcon/joblist/largedatalist.h | 9 --------- oam/etc/Columnstore.xml | 8 +++++--- oam/install_scripts/post-install | 5 ++++- oam/install_scripts/pre-uninstall | 16 ++++++---------- oamapps/serverMonitor/main.cpp | 5 +---- utils/idbdatafile/IDBPolicy.cpp | 4 +++- utils/startup/installdir.cpp | 18 ++++++++++++++++++ writeengine/shared/we_config.cpp | 3 +++ writeengine/splitter/we_cmdargs.cpp | 2 +- 9 files changed, 41 insertions(+), 29 deletions(-) diff --git a/dbcon/joblist/largedatalist.h b/dbcon/joblist/largedatalist.h index 924b0b200..7aedead4b 100644 --- a/dbcon/joblist/largedatalist.h +++ b/dbcon/joblist/largedatalist.h @@ -204,7 +204,6 @@ template LargeDataList::LargeDataList(uint32_t nc, uint32_t elementSaveSize1st, uint32_t elementSaveSize2nd, const ResourceManager* rm): base(nc), path(rm.getScTempDiskPath()), fTraceOn(false), fReUse(false), fSaveForReuse(false), fRestoreInfo(NULL) { -// config::Config *config = config::Config::makeConfig(); loadedSet = 0; setCount = 1; @@ -214,14 +213,6 @@ LargeDataList::LargeDataList(uint32_t nc, uint32_t eleme multipleProducers = false; fLoadedSetCount = 0; setDiskElemSize ( elementSaveSize1st, elementSaveSize2nd ); -// try { -// path = config->getConfig("SystemConfig", "TempDiskPath"); -// } -// catch (...) { -// } -// if (path.length() == 0) -// path = defaultTempDiskPath; - //pthread_cond_init(&consumePhase, NULL); } template diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index 8474b67d0..2196fe7b9 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -249,13 +249,14 @@ 90 80 70 - /tmp + 10 0.0.0.0 128M 10 - /tmp/columnstore_tmp_files $INSTALLDIR 10 120 @@ -266,7 +267,8 @@ - /tmp/rdwrscratch + rdwrscratch + /columnstore_tmp_files dm diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index c18b0913e..9b2b939a8 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -174,8 +174,11 @@ if [ $user != "root" ]; then mkdir $tmpDir >/dev/null 2>&1 fi +TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` +tmpDir=$tmpDir+$TempFileDir + #create columnstore temp file directory -mkdir -p $tmpDir/columnstore_tmp_files >/dev/null 2>&1 +mkdir -p $tmpDir >/dev/null 2>&1 #create mount directories mkdir /mnt/tmp > /dev/null 2>&1 diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 5b82a1275..52571aacf 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -69,6 +69,9 @@ if [ $user == "root" ]; then fi rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1\ +# delete Mariab Columnstore shared memory segments +$installdir/bin/clearShm > /dev/null 2>&1 + #get temp directory tmpDir="/tmp" if [ $user != "root" ]; then @@ -76,21 +79,14 @@ if [ $user != "root" ]; then mkdir $tmpDir >/dev/null 2>&1 fi -# delete Mariab Columnstore shared memory segments -$installdir/bin/clearShm > /dev/null 2>&1 +TempFileDir=`$installdir/bin/getConfig SystemConfig TempFileDir` +tmpDir=$tmpDir+$TempFileDir # delete tmp files +rm -rf $tmpDir rm -f $installdir/local/*.columnstore rm -rf $installdir/local/etc/ -rm -rf $tmpDir/bucketreuse -rm -f $tmpDir/columnstore.txt -rm -f $tmpDir/dbbuilder.* -rm -f $tmpDir/dbrmfiles -rm -f $tmpDir/pkgcheck -rm -f $tmpDir/upgrade-status.log.* -rm -f $tmpDir/mount.log rm -f $installdir/data/bulk/tmpjob/* >/dev/null 2>&1 -rm -rf $tmpDir/columnstore_tmp_files rm -f $installdir/local/moveDbrootTransactionLog lockdir=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation LockFileDirectory` diff --git a/oamapps/serverMonitor/main.cpp b/oamapps/serverMonitor/main.cpp index 6908cbe86..e3fb8e511 100644 --- a/oamapps/serverMonitor/main.cpp +++ b/oamapps/serverMonitor/main.cpp @@ -42,11 +42,8 @@ int main (int argc, char** argv) { ServerMonitor serverMonitor; Oam oam; - - string TempFileDir; - oam.getSystemConfig("TempFileDir", TempFileDir); - tmpDir = startup::StartUp::tmpDir() + TempFileDir; + tmpDir = startup::StartUp::tmpDir(); struct sigaction ign; diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 7822bf891..4992a1d15 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -217,7 +217,9 @@ void IDBPolicy::configIDBPolicy() // Directory in which to place file buffer temporary files. string tmpDir = startup::StartUp::tmpDir(); - string hdfsRdwrScratch = tmpDir + "/rdwrscratch"; + + string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); + hdfsRdwrScratch = tmpDir + hdfsRdwrScratch; IDBPolicy::init( idblog, bUseRdwrMemBuffer, hdfsRdwrScratch, hdfsRdwrBufferMaxSize ); diff --git a/utils/startup/installdir.cpp b/utils/startup/installdir.cpp index 19583c6e5..663ac74b3 100644 --- a/utils/startup/installdir.cpp +++ b/utils/startup/installdir.cpp @@ -31,6 +31,10 @@ using namespace boost; #include "idbregistry.h" #endif #include "installdir.h" +#include "configcpp.h" + +using namespace config; + namespace startup { @@ -92,6 +96,18 @@ const string StartUp::tmpDir() *fTmpDirp = cfStr; #else + + Config* sysConfig = Config::makeConfig(); + + string TempFileDir; + + try + { + TempFileDir = sysConfig->getConfig("SystemConfig", "TempFileDir"); + } + catch (...) + {} + fTmpDirp = new string("/tmp"); //See if we can figure out the tmp dir in Linux... //1. env var COLUMNSTORE_INSTALL_DIR @@ -108,6 +124,8 @@ const string StartUp::tmpDir() *fTmpDirp = homedir + "/.tmp"; } + + *fTmpDirp = *fTmpDirp + TempFileDir; #endif diff --git a/writeengine/shared/we_config.cpp b/writeengine/shared/we_config.cpp index 07554283e..63393cd9c 100644 --- a/writeengine/shared/we_config.cpp +++ b/writeengine/shared/we_config.cpp @@ -263,7 +263,10 @@ void Config::checkReload( ) } // Directory in which to place file buffer temporary files. + string TmpFileDir = startup::StartUp::tmpDir(); + string hdfsRdwrScratch = cf->getConfig("SystemConfig", "hdfsRdwrScratch"); + hdfsRdwrScratch = TmpFileDir + hdfsRdwrScratch if ( hdfsRdwrScratch.length() == 0 ) { diff --git a/writeengine/splitter/we_cmdargs.cpp b/writeengine/splitter/we_cmdargs.cpp index e64d9a4f9..83044fa45 100644 --- a/writeengine/splitter/we_cmdargs.cpp +++ b/writeengine/splitter/we_cmdargs.cpp @@ -1418,7 +1418,7 @@ std::string WECmdArgs::getTmpFileDir() { if (!fTmpFileDir.empty()) return fTmpFileDir; - fTmpFileDir = startup::StartUp::tmpDir() + "columnstore_tmp_files"; + fTmpFileDir = startup::StartUp::tmpDir(); if (fTmpFileDir.empty()) throw( runtime_error("Config ERROR: TmpFileDir not found!!"));